parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject incubator-parquet-mr git commit: PARQUET-204: add parquet-schema directory support
Date Tue, 24 Mar 2015 23:06:39 GMT
Repository: incubator-parquet-mr
Updated Branches:
  refs/heads/master cb7f6a8cb -> fd3085ed3


PARQUET-204: add parquet-schema directory support

Author: Neville Li <neville@spotify.com>

Closes #136 from nevillelyh/neville/PARQUET-204 and squashes the following commits:

633829b [Neville Li] PARQUET-204: add parquet-schema directory support
7aa8581 [Neville Li] PARQUET-203: consolidate PathFilter for hidden files


Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/fd3085ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/fd3085ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/fd3085ed

Branch: refs/heads/master
Commit: fd3085ed31d920e8ca6bba391e75d1423ed8b607
Parents: cb7f6a8
Author: Neville Li <neville@spotify.com>
Authored: Tue Mar 24 16:06:26 2015 -0700
Committer: Ryan Blue <blue@apache.org>
Committed: Tue Mar 24 16:06:26 2015 -0700

----------------------------------------------------------------------
 .../java/parquet/hadoop/ParquetFileReader.java  |  9 ++----
 .../java/parquet/hadoop/ParquetInputFormat.java | 10 ++----
 .../main/java/parquet/hadoop/ParquetReader.java |  9 ++----
 .../main/java/parquet/hadoop/PrintFooter.java   |  9 ++----
 .../parquet/hadoop/util/HiddenFileFilter.java   | 33 ++++++++++++++++++++
 .../parquet/hadoop/TestParquetFileWriter.java   |  9 ++----
 .../tools/command/ShowSchemaCommand.java        | 19 ++++++++++-
 7 files changed, 61 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/fd3085ed/parquet-hadoop/src/main/java/parquet/hadoop/ParquetFileReader.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/parquet/hadoop/ParquetFileReader.java
index de94b12..99f52c8 100644
--- a/parquet-hadoop/src/main/java/parquet/hadoop/ParquetFileReader.java
+++ b/parquet-hadoop/src/main/java/parquet/hadoop/ParquetFileReader.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 
 import parquet.Log;
 import parquet.bytes.BytesInput;
@@ -75,6 +74,7 @@ import parquet.hadoop.ColumnChunkPageReadStore.ColumnChunkPageReader;
 import parquet.hadoop.metadata.BlockMetaData;
 import parquet.hadoop.metadata.ColumnChunkMetaData;
 import parquet.hadoop.metadata.ParquetMetadata;
+import parquet.hadoop.util.HiddenFileFilter;
 import parquet.hadoop.util.counters.BenchmarkCounter;
 import parquet.io.ParquetDecodingException;
 
@@ -299,12 +299,7 @@ public class ParquetFileReader implements Closeable {
   private static List<FileStatus> listFiles(Configuration conf, FileStatus fileStatus)
throws IOException {
     if (fileStatus.isDir()) {
       FileSystem fs = fileStatus.getPath().getFileSystem(conf);
-      FileStatus[] list = fs.listStatus(fileStatus.getPath(), new PathFilter() {
-        @Override
-        public boolean accept(Path p) {
-          return !p.getName().startsWith("_") && !p.getName().startsWith(".");
-        }
-      });
+      FileStatus[] list = fs.listStatus(fileStatus.getPath(), HiddenFileFilter.INSTANCE);
       List<FileStatus> result = new ArrayList<FileStatus>();
       for (FileStatus sub : list) {
         result.addAll(listFiles(conf, sub));

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/fd3085ed/parquet-hadoop/src/main/java/parquet/hadoop/ParquetInputFormat.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/parquet/hadoop/ParquetInputFormat.java b/parquet-hadoop/src/main/java/parquet/hadoop/ParquetInputFormat.java
index 8728965..6b7b9b0 100644
--- a/parquet-hadoop/src/main/java/parquet/hadoop/ParquetInputFormat.java
+++ b/parquet-hadoop/src/main/java/parquet/hadoop/ParquetInputFormat.java
@@ -65,6 +65,7 @@ import parquet.hadoop.metadata.GlobalMetaData;
 import parquet.hadoop.metadata.ParquetMetadata;
 import parquet.hadoop.util.ConfigurationUtil;
 import parquet.hadoop.util.ContextUtil;
+import parquet.hadoop.util.HiddenFileFilter;
 import parquet.hadoop.util.SerializationUtil;
 import parquet.io.ParquetDecodingException;
 import parquet.schema.MessageType;
@@ -346,7 +347,7 @@ public class ParquetInputFormat<T> extends FileInputFormat<Void,
T> {
       if (file.isDir()) {
         Path p = file.getPath();
         FileSystem fs = p.getFileSystem(conf);
-        staticAddInputPathRecursively(result, fs, p, hiddenFileFilter);
+        staticAddInputPathRecursively(result, fs, p, HiddenFileFilter.INSTANCE);
       } else {
         result.add(file);
       }
@@ -367,13 +368,6 @@ public class ParquetInputFormat<T> extends FileInputFormat<Void,
T> {
     }
   }
 
-  private static final PathFilter hiddenFileFilter = new PathFilter(){
-    public boolean accept(Path p){
-      String name = p.getName();
-      return !name.startsWith("_") && !name.startsWith(".");
-    }
-  };
-
   /**
    * @param jobContext the current job context
    * @return the footers for the files

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/fd3085ed/parquet-hadoop/src/main/java/parquet/hadoop/ParquetReader.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/parquet/hadoop/ParquetReader.java b/parquet-hadoop/src/main/java/parquet/hadoop/ParquetReader.java
index 4231ae4..7c3ecbb 100644
--- a/parquet-hadoop/src/main/java/parquet/hadoop/ParquetReader.java
+++ b/parquet-hadoop/src/main/java/parquet/hadoop/ParquetReader.java
@@ -31,13 +31,13 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
-import org.apache.hadoop.fs.PathFilter;
 import parquet.filter.UnboundRecordFilter;
 import parquet.filter2.compat.FilterCompat;
 import parquet.filter2.compat.FilterCompat.Filter;
 import parquet.filter2.compat.RowGroupFilter;
 import parquet.hadoop.api.ReadSupport;
 import parquet.hadoop.metadata.BlockMetaData;
+import parquet.hadoop.util.HiddenFileFilter;
 import parquet.schema.MessageType;
 
 /**
@@ -110,12 +110,7 @@ public class ParquetReader<T> implements Closeable {
     this.conf = conf;
 
     FileSystem fs = file.getFileSystem(conf);
-    List<FileStatus> statuses = Arrays.asList(fs.listStatus(file, new PathFilter()
{
-      @Override
-      public boolean accept(Path p) {
-        return !p.getName().startsWith("_") && !p.getName().startsWith(".");
-      }
-    }));
+    List<FileStatus> statuses = Arrays.asList(fs.listStatus(file, HiddenFileFilter.INSTANCE));
     List<Footer> footers = ParquetFileReader.readAllFootersInParallelUsingSummaryFiles(conf,
statuses, false);
     this.footersIterator = footers.iterator();
   }

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/fd3085ed/parquet-hadoop/src/main/java/parquet/hadoop/PrintFooter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/parquet/hadoop/PrintFooter.java b/parquet-hadoop/src/main/java/parquet/hadoop/PrintFooter.java
index 5f1d271..0728f8f 100644
--- a/parquet-hadoop/src/main/java/parquet/hadoop/PrintFooter.java
+++ b/parquet-hadoop/src/main/java/parquet/hadoop/PrintFooter.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 
 import parquet.column.ColumnDescriptor;
 import parquet.column.Encoding;
@@ -50,6 +49,7 @@ import parquet.column.statistics.Statistics;
 import parquet.hadoop.metadata.BlockMetaData;
 import parquet.hadoop.metadata.ColumnChunkMetaData;
 import parquet.hadoop.metadata.ParquetMetadata;
+import parquet.hadoop.util.HiddenFileFilter;
 import parquet.io.ParquetDecodingException;
 import parquet.schema.MessageType;
 
@@ -82,12 +82,7 @@ public class PrintFooter {
       List<FileStatus> statuses;
       if (fileStatus.isDir()) {
         System.out.println("listing files in " + fileStatus.getPath());
-        statuses = Arrays.asList(fs.listStatus(fileStatus.getPath(), new PathFilter() {
-          @Override
-          public boolean accept(Path path) {
-            return !path.getName().startsWith("_");
-          }
-        }));
+        statuses = Arrays.asList(fs.listStatus(fileStatus.getPath(), HiddenFileFilter.INSTANCE));
       } else {
         statuses = new ArrayList<FileStatus>();
         statuses.add(fileStatus);

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/fd3085ed/parquet-hadoop/src/main/java/parquet/hadoop/util/HiddenFileFilter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/parquet/hadoop/util/HiddenFileFilter.java b/parquet-hadoop/src/main/java/parquet/hadoop/util/HiddenFileFilter.java
new file mode 100644
index 0000000..0bc2de6
--- /dev/null
+++ b/parquet-hadoop/src/main/java/parquet/hadoop/util/HiddenFileFilter.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package parquet.hadoop.util;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+
+public class HiddenFileFilter implements PathFilter {
+  public static final HiddenFileFilter INSTANCE = new HiddenFileFilter();
+
+  private HiddenFileFilter() {}
+
+  @Override
+  public boolean accept(Path p) {
+    return !p.getName().startsWith("_") && !p.getName().startsWith(".");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/fd3085ed/parquet-hadoop/src/test/java/parquet/hadoop/TestParquetFileWriter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/parquet/hadoop/TestParquetFileWriter.java b/parquet-hadoop/src/test/java/parquet/hadoop/TestParquetFileWriter.java
index 4a81b85..e991bd2 100644
--- a/parquet-hadoop/src/test/java/parquet/hadoop/TestParquetFileWriter.java
+++ b/parquet-hadoop/src/test/java/parquet/hadoop/TestParquetFileWriter.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.junit.Test;
 import parquet.Log;
 import parquet.bytes.BytesInput;
@@ -37,6 +36,7 @@ import parquet.column.statistics.BinaryStatistics;
 import parquet.column.statistics.LongStatistics;
 import parquet.format.Statistics;
 import parquet.hadoop.metadata.*;
+import parquet.hadoop.util.HiddenFileFilter;
 import parquet.io.api.Binary;
 import parquet.schema.MessageType;
 import parquet.schema.MessageTypeParser;
@@ -325,12 +325,7 @@ public class TestParquetFileWriter {
 
     validateFooters(metadata);
 
-    footers = ParquetFileReader.readAllFootersInParallelUsingSummaryFiles(configuration,
Arrays.asList(fs.listStatus(testDirPath, new PathFilter() {
-      @Override
-      public boolean accept(Path p) {
-        return !p.getName().startsWith("_");
-      }
-    })), false);
+    footers = ParquetFileReader.readAllFootersInParallelUsingSummaryFiles(configuration,
Arrays.asList(fs.listStatus(testDirPath, HiddenFileFilter.INSTANCE)), false);
     validateFooters(footers);
 
     fs.delete(metadataFile.getPath(), false);

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/fd3085ed/parquet-tools/src/main/java/parquet/tools/command/ShowSchemaCommand.java
----------------------------------------------------------------------
diff --git a/parquet-tools/src/main/java/parquet/tools/command/ShowSchemaCommand.java b/parquet-tools/src/main/java/parquet/tools/command/ShowSchemaCommand.java
index a933aea..11c4304 100644
--- a/parquet-tools/src/main/java/parquet/tools/command/ShowSchemaCommand.java
+++ b/parquet-tools/src/main/java/parquet/tools/command/ShowSchemaCommand.java
@@ -25,10 +25,13 @@ import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
 import parquet.hadoop.ParquetFileReader;
 import parquet.hadoop.metadata.ParquetMetadata;
+import parquet.hadoop.util.HiddenFileFilter;
 import parquet.schema.MessageType;
 import parquet.tools.Main;
 import parquet.tools.util.MetadataUtils;
@@ -71,7 +74,21 @@ public class ShowSchemaCommand extends ArgsOnlyCommand {
     String input = args[0];
 
     Configuration conf = new Configuration();
-    ParquetMetadata metaData = ParquetFileReader.readFooter(conf, new Path(input));
+    ParquetMetadata metaData;
+
+    Path path = new Path(input);
+    FileSystem fs = path.getFileSystem(conf);
+    Path file;
+    if (fs.isDirectory(path)) {
+      FileStatus[] statuses = fs.listStatus(path, HiddenFileFilter.INSTANCE);
+      if (statuses.length == 0) {
+        throw new RuntimeException("Directory " + path.toString() + " is empty");
+      }
+      file = statuses[0].getPath();
+    } else {
+      file = path;
+    }
+    metaData = ParquetFileReader.readFooter(conf, file);
     MessageType schema = metaData.getFileMetaData().getSchema();
 
     Main.out.println(schema);


Mime
View raw message