parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alexleven...@apache.org
Subject parquet-mr git commit: PARQUET-337 handle binary fields in set/map/list in parquet-scrooge
Date Thu, 16 Jul 2015 23:41:02 GMT
Repository: parquet-mr
Updated Branches:
  refs/heads/master 8a2c61866 -> f79c9365d


PARQUET-337 handle binary fields in set/map/list in parquet-scrooge

https://issues.apache.org/jira/browse/PARQUET-337

Author: Jake Donham <jdonham@twitter.com>

Closes #243 from jaked/PARQUET-337 and squashes the following commits:

8129fe5 [Jake Donham] parquet-scrooge: handle binary fields in set/map/list


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/f79c9365
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/f79c9365
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/f79c9365

Branch: refs/heads/master
Commit: f79c9365d0ee89cb407b90cc084eece8fcf9a8a2
Parents: 8a2c618
Author: Jake Donham <jdonham@twitter.com>
Authored: Thu Jul 16 16:41:04 2015 -0700
Committer: Alex Levenson <alexlevenson@twitter.com>
Committed: Thu Jul 16 16:41:04 2015 -0700

----------------------------------------------------------------------
 .../apache/parquet/scrooge/ScroogeStructConverter.java  |  9 ++++++---
 .../parquet/scrooge/ScroogeStructConverterTest.java     | 12 +++++++++---
 parquet-scrooge/src/test/thrift/test.thrift             |  4 ++++
 3 files changed, 19 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/f79c9365/parquet-scrooge/src/main/java/org/apache/parquet/scrooge/ScroogeStructConverter.java
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/main/java/org/apache/parquet/scrooge/ScroogeStructConverter.java
b/parquet-scrooge/src/main/java/org/apache/parquet/scrooge/ScroogeStructConverter.java
index ca2716b..c7448e3 100644
--- a/parquet-scrooge/src/main/java/org/apache/parquet/scrooge/ScroogeStructConverter.java
+++ b/parquet-scrooge/src/main/java/org/apache/parquet/scrooge/ScroogeStructConverter.java
@@ -1,4 +1,4 @@
-/* 
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,6 +22,7 @@ import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.ParameterizedType;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
@@ -284,6 +285,8 @@ public class ScroogeStructConverter {
       return new ThriftType.I64Type();
     } else if (typeClass == String.class) {
       return new ThriftType.StringType();
+    } else if (typeClass == ByteBuffer.class) {
+      return new ThriftType.StringType();
     } else if (typeClass == scala.collection.Seq.class) {
       Manifest<?> a = typeManifest.typeArguments().apply(0);
       return convertListTypeField(name, a, requirement);

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/f79c9365/parquet-scrooge/src/test/java/org/apache/parquet/scrooge/ScroogeStructConverterTest.java
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/test/java/org/apache/parquet/scrooge/ScroogeStructConverterTest.java
b/parquet-scrooge/src/test/java/org/apache/parquet/scrooge/ScroogeStructConverterTest.java
index 5037934..648634c 100644
--- a/parquet-scrooge/src/test/java/org/apache/parquet/scrooge/ScroogeStructConverterTest.java
+++ b/parquet-scrooge/src/test/java/org/apache/parquet/scrooge/ScroogeStructConverterTest.java
@@ -1,4 +1,4 @@
-/* 
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -36,6 +36,7 @@ import org.apache.parquet.scrooge.test.SetNestSet;
 import org.apache.parquet.scrooge.test.StringAndBinary;
 import org.apache.parquet.scrooge.test.TestFieldOfEnum;
 import org.apache.parquet.scrooge.test.TestListPrimitive;
+import org.apache.parquet.scrooge.test.TestMapBinary;
 import org.apache.parquet.scrooge.test.TestMapComplex;
 import org.apache.parquet.scrooge.test.TestMapPrimitiveKey;
 import org.apache.parquet.scrooge.test.TestMapPrimitiveValue;
@@ -107,6 +108,11 @@ public class ScroogeStructConverterTest {
   }
 
   @Test
+  public void testMapBinary() throws Exception {
+    shouldConvertConsistentlyWithThriftStructConverter(TestMapBinary.class);
+  }
+
+  @Test
   public void testMapComplex() throws Exception {
     shouldConvertConsistentlyWithThriftStructConverter(TestMapComplex.class);
   }

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/f79c9365/parquet-scrooge/src/test/thrift/test.thrift
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/test/thrift/test.thrift b/parquet-scrooge/src/test/thrift/test.thrift
index fd9eed4..6db7dc1 100644
--- a/parquet-scrooge/src/test/thrift/test.thrift
+++ b/parquet-scrooge/src/test/thrift/test.thrift
@@ -101,6 +101,10 @@ struct TestMapComplex{
   1: required map<Phone,Address> phone_address_map
 }
 
+struct TestMapBinary{
+  1: required map<string,binary> string_binary_map
+}
+
 struct TestMapPrimitiveKey {
   1: required map<i16,string> short_map,
   2: required map<i32,string> int_map,


Mime
View raw message