crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-597: Upgrade to Parquet 1.8.1
Date Mon, 28 Mar 2016 15:12:03 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 0d31415e4 -> 1252e7f91


CRUNCH-597: Upgrade to Parquet 1.8.1


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/1252e7f9
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/1252e7f9
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/1252e7f9

Branch: refs/heads/master
Commit: 1252e7f9181053132d35d13b0137f997786c79ff
Parents: 0d31415
Author: Josh Wills <jwills@apache.org>
Authored: Thu Mar 24 09:55:16 2016 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Thu Mar 24 09:55:16 2016 -0700

----------------------------------------------------------------------
 crunch-core/pom.xml                                          | 2 +-
 .../crunch/io/parquet/AvroParquetFileSourceTargetIT.java     | 8 ++++----
 .../org/apache/crunch/io/parquet/AvroParquetPipelineIT.java  | 4 ++--
 .../src/main/java/org/apache/crunch/PipelineCallable.java    | 2 +-
 crunch-core/src/main/java/org/apache/crunch/io/Compress.java | 2 +-
 .../crunch/io/parquet/AvroParquetFileReaderFactory.java      | 6 +++---
 .../org/apache/crunch/io/parquet/AvroParquetFileSource.java  | 8 ++++----
 .../org/apache/crunch/io/parquet/AvroParquetFileTarget.java  | 4 ++--
 .../crunch/io/parquet/AvroParquetFileReaderFactoryTest.java  | 2 +-
 .../java/org/apache/crunch/SparkAvroParquetPipelineIT.java   | 4 ++--
 pom.xml                                                      | 4 ++--
 11 files changed, 23 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml
index 75f7208..603c231 100644
--- a/crunch-core/pom.xml
+++ b/crunch-core/pom.xml
@@ -46,7 +46,7 @@ under the License.
     </dependency>
 
     <dependency>
-      <groupId>com.twitter</groupId>
+      <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-avro</artifactId>
     </dependency>
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
index d75d9da..a14a5e3 100644
--- a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
@@ -51,12 +51,12 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
-import parquet.avro.AvroParquetWriter;
+import org.apache.parquet.avro.AvroParquetWriter;
 
 import com.google.common.collect.Lists;
-import parquet.column.ColumnReader;
-import parquet.filter.RecordFilter;
-import parquet.filter.UnboundRecordFilter;
+import org.apache.parquet.column.ColumnReader;
+import org.apache.parquet.filter.RecordFilter;
+import org.apache.parquet.filter.UnboundRecordFilter;
 
 @SuppressWarnings("serial")
 public class AvroParquetFileSourceTargetIT implements Serializable {

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java
b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java
index 055d0d7..62ab1c2 100644
--- a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetPipelineIT.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.fs.Path;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
-import parquet.avro.AvroParquetReader;
-import parquet.avro.AvroParquetWriter;
+import org.apache.parquet.avro.AvroParquetReader;
+import org.apache.parquet.avro.AvroParquetWriter;
 
 import static org.hamcrest.core.Is.is;
 import static org.junit.Assert.assertThat;

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java b/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java
index 9f6829e..4ab6df1 100644
--- a/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java
+++ b/crunch-core/src/main/java/org/apache/crunch/PipelineCallable.java
@@ -17,13 +17,13 @@
  */
 package org.apache.crunch;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import org.apache.hadoop.conf.Configuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import parquet.Preconditions;
 
 import java.util.Map;
 import java.util.concurrent.Callable;

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/io/Compress.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/Compress.java b/crunch-core/src/main/java/org/apache/crunch/io/Compress.java
index 881b017..bbf09e5 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/Compress.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/Compress.java
@@ -53,7 +53,7 @@ public class Compress {
   public static <T extends Target> T snappy(T target) {
     Class<? extends CompressionCodec> snappyCodec = org.apache.hadoop.io.compress.SnappyCodec.class;
     if (target instanceof AvroParquetFileTarget || target instanceof AvroParquetFileSourceTarget)
{
-      snappyCodec = parquet.hadoop.codec.SnappyCodec.class;
+      snappyCodec = org.apache.parquet.hadoop.codec.SnappyCodec.class;
     }
     return (T) compress(target, snappyCodec)
         .outputConf(AvroJob.OUTPUT_CODEC, DataFileConstants.SNAPPY_CODEC);

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java
b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java
index c193563..0b631a8 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactory.java
@@ -30,9 +30,9 @@ import org.apache.crunch.types.avro.AvroType;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import parquet.avro.AvroReadSupport;
-import parquet.hadoop.ParquetReader;
-import parquet.schema.MessageType;
+import org.apache.parquet.avro.AvroReadSupport;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.schema.MessageType;
 
 class AvroParquetFileReaderFactory<T> implements FileReaderFactory<T> {
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
index 18b2928..09703bd 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
@@ -40,10 +40,10 @@ import org.apache.hadoop.fs.Path;
 
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
 import org.apache.hadoop.util.ReflectionUtils;
-import parquet.avro.AvroParquetInputFormat;
-import parquet.avro.AvroReadSupport;
-import parquet.filter.UnboundRecordFilter;
-import parquet.hadoop.ParquetInputSplit;
+import org.apache.parquet.avro.AvroParquetInputFormat;
+import org.apache.parquet.avro.AvroReadSupport;
+import org.apache.parquet.filter.UnboundRecordFilter;
+import org.apache.parquet.hadoop.ParquetInputSplit;
 
 public class AvroParquetFileSource<T extends IndexedRecord> extends FileSourceImpl<T>
implements ReadableSource<T> {
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java
b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java
index 34ee14b..2698053 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileTarget.java
@@ -33,8 +33,8 @@ import org.apache.crunch.types.avro.AvroType;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.Job;
-import parquet.avro.AvroWriteSupport;
-import parquet.hadoop.ParquetOutputFormat;
+import org.apache.parquet.avro.AvroWriteSupport;
+import org.apache.parquet.hadoop.ParquetOutputFormat;
 
 import java.util.Map;
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
index 9f5ff70..d2ad5bc 100644
--- a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
@@ -41,7 +41,7 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
-import parquet.avro.AvroParquetWriter;
+import org.apache.parquet.avro.AvroParquetWriter;
 
 public class AvroParquetFileReaderFactoryTest {
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java
----------------------------------------------------------------------
diff --git a/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java b/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java
index f5e2c25..a86718e 100644
--- a/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java
+++ b/crunch-spark/src/it/java/org/apache/crunch/SparkAvroParquetPipelineIT.java
@@ -37,8 +37,8 @@ import org.apache.hadoop.fs.Path;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
-import parquet.avro.AvroParquetReader;
-import parquet.avro.AvroParquetWriter;
+import org.apache.parquet.avro.AvroParquetReader;
+import org.apache.parquet.avro.AvroParquetWriter;
 
 import java.io.File;
 import java.io.FileOutputStream;

http://git-wip-us.apache.org/repos/asf/crunch/blob/1252e7f9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7389c76..78ad9b7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -86,7 +86,7 @@ under the License.
     <commons-cli.version>1.2</commons-cli.version>
     <avro.version>1.7.7</avro.version>
     <hive.version>0.13.1</hive.version>
-    <parquet.version>1.4.3</parquet.version>
+    <parquet.version>1.8.1</parquet.version>
     <javassist.version>3.16.1-GA</javassist.version>
     <jackson.version>1.8.8</jackson.version>
     <protobuf-java.version>2.5.0</protobuf-java.version>
@@ -242,7 +242,7 @@ under the License.
       </dependency>
 
       <dependency>
-        <groupId>com.twitter</groupId>
+        <groupId>org.apache.parquet</groupId>
         <artifactId>parquet-avro</artifactId>
         <version>${parquet.version}</version>
         <exclusions>


Mime
View raw message