hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x..@apache.org
Subject [49/50] [abbrv] hive git commit: HIVE-11277: Merge master to parquet 06/16/2015 [Parquet branch]
Date Thu, 16 Jul 2015 07:53:07 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
index f7adb39,d82e93c..e1b6dd8
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
@@@ -17,10 -17,9 +17,9 @@@ import java.io.IOException
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.hive.ql.exec.Utilities;
- import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
  import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport;
  import org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper;
 -import org.apache.hadoop.io.ArrayWritable;
 +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
  import org.apache.hadoop.io.NullWritable;
  import org.apache.hadoop.mapred.FileInputFormat;
  import org.apache.hadoop.mapred.RecordReader;

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
index 3cd48f8,3261e4b..cb25a4f
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java
@@@ -13,11 -13,12 +13,11 @@@
   */
  package org.apache.hadoop.hive.ql.io.parquet.convert;
  
 -import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport;
 -import org.apache.hadoop.io.ArrayWritable;
 +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
- import parquet.io.api.GroupConverter;
- import parquet.io.api.RecordMaterializer;
- import parquet.schema.GroupType;
- import parquet.schema.MessageType;
+ import org.apache.parquet.io.api.GroupConverter;
+ import org.apache.parquet.io.api.RecordMaterializer;
+ import org.apache.parquet.schema.GroupType;
 -import org.apache.parquet.schema.MessageTypeParser;
++import org.apache.parquet.schema.MessageType;
  
  import java.util.Map;
  

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
index 7915111,06f3d32..99d3643
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
@@@ -20,13 -20,13 +20,15 @@@ package org.apache.hadoop.hive.ql.io.pa
  
  import com.google.common.base.Preconditions;
  import java.util.ArrayList;
 +import java.util.Arrays;
  import java.util.List;
  
 +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
- import parquet.io.api.Converter;
- import parquet.schema.GroupType;
- import parquet.schema.Type;
+ import org.apache.hadoop.io.ArrayWritable;
+ import org.apache.hadoop.io.Writable;
+ import org.apache.parquet.io.api.Converter;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.Type;
  
  public class HiveCollectionConverter extends HiveGroupConverter {
    private final GroupType collectionType;

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
index c22c045,b1ca85a..30001cb
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
@@@ -13,14 -13,14 +13,13 @@@
   */
  package org.apache.hadoop.hive.ql.io.parquet.convert;
  
--import org.apache.hadoop.io.Writable;
- import parquet.io.api.Converter;
- import parquet.io.api.GroupConverter;
- import parquet.io.api.PrimitiveConverter;
- import parquet.schema.GroupType;
- import parquet.schema.OriginalType;
- import parquet.schema.PrimitiveType;
- import parquet.schema.Type;
+ import org.apache.parquet.io.api.Converter;
+ import org.apache.parquet.io.api.GroupConverter;
+ import org.apache.parquet.io.api.PrimitiveConverter;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.OriginalType;
+ import org.apache.parquet.schema.PrimitiveType;
+ import org.apache.parquet.schema.Type;
  
  import java.util.Map;
  

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java
index 4d06c36,9c35a9f..bd706fa
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java
@@@ -18,10 -17,11 +18,10 @@@ import java.util.Arrays
  import java.util.List;
  import java.util.Map;
  
 -import org.apache.hadoop.io.ArrayWritable;
 -import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
- import parquet.io.api.Converter;
- import parquet.schema.GroupType;
- import parquet.schema.Type;
+ import org.apache.parquet.io.api.Converter;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.Type;
  
  /**
   *

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
index c84caec,c0af291..3b36eaf
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
@@@ -22,13 -22,15 +22,13 @@@ import java.util.ArrayList
  import java.util.List;
  import java.util.Map;
  
 -import org.apache.hadoop.io.ArrayWritable;
 -import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
- import parquet.column.Dictionary;
- import parquet.io.api.Binary;
- import parquet.io.api.Converter;
- import parquet.io.api.PrimitiveConverter;
- import parquet.schema.GroupType;
- import parquet.schema.PrimitiveType;
+ import org.apache.parquet.column.Dictionary;
+ import org.apache.parquet.io.api.Binary;
+ import org.apache.parquet.io.api.Converter;
+ import org.apache.parquet.io.api.PrimitiveConverter;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.PrimitiveType;
  
  /**
   * Converters for repeated fields need to know when the parent field starts and

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
index a5856bf,8b8b92d..c5ad201
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
@@@ -29,17 -29,17 +29,17 @@@ import org.apache.hadoop.hive.serde2.ob
  import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
  import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
  import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 -import org.apache.hadoop.io.ArrayWritable;
 +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
  import org.apache.hadoop.util.StringUtils;
  
- import parquet.hadoop.api.InitContext;
- import parquet.hadoop.api.ReadSupport;
- import parquet.io.api.RecordMaterializer;
- import parquet.schema.GroupType;
- import parquet.schema.MessageType;
- import parquet.schema.Type;
- import parquet.schema.Types;
- import parquet.schema.PrimitiveType.PrimitiveTypeName;
+ import org.apache.parquet.hadoop.api.InitContext;
+ import org.apache.parquet.hadoop.api.ReadSupport;
+ import org.apache.parquet.io.api.RecordMaterializer;
+ import org.apache.parquet.schema.GroupType;
+ import org.apache.parquet.schema.MessageType;
+ import org.apache.parquet.schema.Type;
+ import org.apache.parquet.schema.Types;
+ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
  
  /**
   *
@@@ -246,9 -245,9 +246,9 @@@ public class DataWritableReadSupport ex
     * @return Record Materialize for Hive
     */
    @Override
 -  public RecordMaterializer<ArrayWritable> prepareForRead(final Configuration configuration,
 +  public RecordMaterializer<ObjectArrayWritable> prepareForRead(final Configuration
configuration,
        final Map<String, String> keyValueMetaData, final MessageType fileSchema,
-           final parquet.hadoop.api.ReadSupport.ReadContext readContext) {
+           final org.apache.parquet.hadoop.api.ReadSupport.ReadContext readContext) {
      final Map<String, String> metadata = readContext.getReadSupportMetadata();
      if (metadata == null) {
        throw new IllegalStateException("ReadContext not initialized properly. " +

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
index dc4f896,e1bf8e2..58eabf0
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
@@@ -34,10 -33,11 +34,10 @@@ import org.apache.hadoop.hive.serde2.ty
  import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
  import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
  import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 -import org.apache.hadoop.io.ArrayWritable;
  import org.apache.hadoop.io.Text;
  import org.apache.hadoop.io.Writable;
- import parquet.hadoop.ParquetOutputFormat;
- import parquet.hadoop.ParquetWriter;
+ import org.apache.parquet.hadoop.ParquetOutputFormat;
+ import org.apache.parquet.hadoop.ParquetWriter;
  
  /**
   *

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
index 17d8feb,4b69642..e3e875e
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
@@@ -20,18 -20,22 +20,18 @@@ package org.apache.hadoop.hive.ql.io.pa
  
  import java.util.List;
  import org.apache.hadoop.fs.Path;
 -import org.apache.hadoop.hive.serde2.io.DoubleWritable;
 -import org.apache.hadoop.io.ArrayWritable;
 -import org.apache.hadoop.io.FloatWritable;
 -import org.apache.hadoop.io.IntWritable;
 -import org.apache.hadoop.io.LongWritable;
 +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
  import org.junit.Assert;
  import org.junit.Test;
- import parquet.io.api.RecordConsumer;
- import parquet.schema.MessageType;
- import parquet.schema.Types;
+ import org.apache.parquet.io.api.RecordConsumer;
+ import org.apache.parquet.schema.MessageType;
+ import org.apache.parquet.schema.Types;
  
- import static parquet.schema.OriginalType.LIST;
- import static parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
- import static parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
- import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
- import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
+ import static org.apache.parquet.schema.OriginalType.LIST;
+ import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
+ import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
+ import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
+ import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
  
  public class TestArrayCompatibility extends AbstractTestParquetDirect {
  

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
index 5f586e4,7049139..71f4acf
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
@@@ -353,21 -358,19 +353,19 @@@ public class TestDataWritableWriter 
          + "  }\n"
          + "}\n";
  
 -    ArrayWritable hiveRecord = createGroup(
 +    ObjectArrayWritable hiveRecord = createGroup(
          createGroup(
              createArray(
-                 createArray(
-                     createString("key1"),
-                     createInt(1)
-                 ),
-                 createArray(
-                     createString("key2"),
-                     createInt(2)
-                 ),
-                 createArray(
-                     createString("key3"),
-                     createNull()
-                 )
+                 createString("key1"),
+                 createInt(1)
+             ),
+             createArray(
+                 createString("key2"),
+                 createInt(2)
+             ),
+             createArray(
+                 createString("key3"),
+                 createNull()
              )
          )
      );
@@@ -424,15 -427,11 +422,11 @@@
          + "  }\n"
          + "}\n";
  
 -    ArrayWritable hiveRecord = createGroup(
 -        createArray(
 +    ObjectArrayWritable hiveRecord = createGroup(
 +        createGroup(
              createArray(
-                 createGroup(
-                     createArray(
-                         createInt(1),
-                         createInt(2)
-                     )
-                 )
+                 createInt(1),
+                 createInt(2)
              )
          )
      );

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java
index 1b98cd3,518eefd..fdf4b0f
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java
@@@ -15,10 -15,10 +15,10 @@@ package org.apache.hadoop.hive.ql.io.pa
  
  import static org.mockito.Mockito.mock;
  
 -import org.apache.hadoop.io.ArrayWritable;
 +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
  import org.junit.Test;
  
- import parquet.hadoop.ParquetInputFormat;
+ import org.apache.parquet.hadoop.ParquetInputFormat;
  
  public class TestMapredParquetInputFormat {
    @Test

http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetRowGroupFilter.java
----------------------------------------------------------------------


Mime
View raw message