parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tians...@apache.org
Subject parquet-mr git commit: Revert "PARQUET-252 : Support nests container types for scrooge support"
Date Mon, 27 Apr 2015 22:38:55 GMT
Repository: parquet-mr
Updated Branches:
  refs/heads/master f28aa7104 -> 720b988fd


Revert "PARQUET-252 : Support nests container types for scrooge support"

This reverts commit f28aa71041181867a720134e26b64b03cbccb6ec.


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/720b988f
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/720b988f
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/720b988f

Branch: refs/heads/master
Commit: 720b988fd8d7a50fbe922e6c73a3681b1c566331
Parents: f28aa71
Author: Tianshuo Deng <tdeng@twitter.com>
Authored: Mon Apr 27 15:37:21 2015 -0700
Committer: Tianshuo Deng <tdeng@twitter.com>
Committed: Mon Apr 27 15:37:21 2015 -0700

----------------------------------------------------------------------
 .../parquet/scrooge/ScroogeStructConverter.java | 316 ++++++++-----------
 .../scrooge/ScroogeStructConverterTest.java     | 151 +++------
 parquet-scrooge/src/test/thrift/test.thrift     |  93 +-----
 3 files changed, 188 insertions(+), 372 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/720b988f/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java b/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java
index 2c3b761..c5bb72f 100644
--- a/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java
+++ b/parquet-scrooge/src/main/java/parquet/scrooge/ScroogeStructConverter.java
@@ -18,31 +18,24 @@
  */
 package parquet.scrooge;
 
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.ParameterizedType;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import scala.collection.JavaConversions;
-import scala.collection.JavaConversions$;
-import scala.collection.Seq;
-import scala.reflect.Manifest;
-
 import com.twitter.scrooge.ThriftStructCodec;
 import com.twitter.scrooge.ThriftStructFieldInfo;
-
 import parquet.thrift.struct.ThriftField;
 import parquet.thrift.struct.ThriftType;
 import parquet.thrift.struct.ThriftType.StructType.StructOrUnionType;
 import parquet.thrift.struct.ThriftTypeID;
-import static parquet.thrift.struct.ThriftField.Requirement;
-import static parquet.thrift.struct.ThriftField.Requirement.DEFAULT;
-import static parquet.thrift.struct.ThriftField.Requirement.REQUIRED;
-import static parquet.thrift.struct.ThriftField.Requirement.OPTIONAL;
+import scala.collection.JavaConversions;
+import scala.collection.JavaConversions$;
+import scala.collection.Seq;
+import scala.reflect.Manifest;
 
+import java.lang.reflect.*;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import static parquet.thrift.struct.ThriftField.Requirement;
+import static parquet.thrift.struct.ThriftField.Requirement.*;
 
 /**
  * Class to convert a scrooge generated class to {@link ThriftType.StructType}. {@link ScroogeReadSupport
} uses this
@@ -54,30 +47,18 @@ public class ScroogeStructConverter {
 
   /**
    * convert a given scrooge generated class to {@link ThriftType.StructType}
+   *
+   * @param scroogeClass
+   * @return
+   * @throws Exception
    */
   public ThriftType.StructType convert(Class scroogeClass) {
     return convertStructFromClass(scroogeClass);
   }
 
-  private static String mapKeyName(String fieldName) {
-    return fieldName + "_map_key";
-  }
-
-  private static String mapValueName(String fieldName) {
-    return fieldName + "_map_value";
-  }
-
-  private static String listElemName(String fieldName) {
-    return fieldName + "_list_elem";
-  }
-
-  private static String setElemName(String fieldName) {
-    return fieldName + "_set_elem";
-  }
-
   private Class getCompanionClass(Class klass) {
     try {
-      return Class.forName(klass.getName() + "$");
+     return Class.forName(klass.getName() + "$");
     } catch (ClassNotFoundException e) {
       throw new ScroogeSchemaConversionException("Can not find companion object for scrooge
class " + klass, e);
     }
@@ -88,9 +69,9 @@ public class ScroogeStructConverter {
   }
 
   private ThriftType.StructType convertCompanionClassToStruct(Class<?> companionClass)
{
-    ThriftStructCodec<?> companionObject;
+    ThriftStructCodec companionObject = null;
     try {
-      companionObject = (ThriftStructCodec<?>) companionClass.getField("MODULE$").get(null);
+      companionObject = (ThriftStructCodec<?>)companionClass.getField("MODULE$").get(null);
     } catch (ReflectiveOperationException e) {
       throw new ScroogeSchemaConversionException("Can not get ThriftStructCodec from companion
object of " + companionClass.getName(), e);
     }
@@ -107,9 +88,9 @@ public class ScroogeStructConverter {
     return new ThriftType.StructType(children, structOrUnionType);
   }
 
-  private Iterable<ThriftStructFieldInfo> getFieldInfos(ThriftStructCodec<?>
c) {
+  private Iterable<ThriftStructFieldInfo> getFieldInfos(ThriftStructCodec c) {
     Class<? extends ThriftStructCodec> klass = c.getClass();
-    if (isUnion(klass)) {
+    if (isUnion(klass)){
       // Union needs special treatment since currently scrooge does not generates the fieldInfos
       // field in the parent union class
       return getFieldInfosForUnion(klass);
@@ -117,14 +98,9 @@ public class ScroogeStructConverter {
       //each struct has a generated fieldInfos method to provide metadata to its fields
       try {
         Object r = klass.getMethod("fieldInfos").invoke(c);
-        return JavaConversions$.MODULE$.asJavaIterable((scala.collection.Iterable<ThriftStructFieldInfo>)
r);
-      } catch (ClassCastException e) {
-        throw new ScroogeSchemaConversionException("can not get field Info from: " + c.toString(),
e);
-      } catch (InvocationTargetException e) {
-        throw new ScroogeSchemaConversionException("can not get field Info from: " + c.toString(),
e);
-      } catch (NoSuchMethodException e) {
-        throw new ScroogeSchemaConversionException("can not get field Info from: " + c.toString(),
e);
-      } catch (IllegalAccessException e) {
+        Iterable<ThriftStructFieldInfo> a = JavaConversions$.MODULE$.asJavaIterable((scala.collection.Iterable<ThriftStructFieldInfo>)r);
+        return a;
+      } catch (ReflectiveOperationException e) {
         throw new ScroogeSchemaConversionException("can not get field Info from: " + c.toString(),
e);
       }
     }
@@ -133,25 +109,57 @@ public class ScroogeStructConverter {
 
   private Iterable<ThriftStructFieldInfo> getFieldInfosForUnion(Class klass) {
     ArrayList<ThriftStructFieldInfo> fields = new ArrayList<ThriftStructFieldInfo>();
-    for (Field f : klass.getDeclaredFields()) {
-      if (f.getType().equals(Manifest.class)) {
-        Class unionClass = (Class) ((ParameterizedType) f.getGenericType()).getActualTypeArguments()[0];
-        Class companionUnionClass = getCompanionClass(unionClass);
-        try {
-          Object companionUnionObj = companionUnionClass.getField("MODULE$").get(null);
-          ThriftStructFieldInfo info = (ThriftStructFieldInfo) companionUnionClass.getMethod("fieldInfo").invoke(companionUnionObj);
-          fields.add(info);
-        } catch (ReflectiveOperationException e) {
-          throw new ScroogeSchemaConversionException("can not find fiedInfo for " + unionClass,
e);
-        }
+    for(Field f: klass.getDeclaredFields()){
+       if (f.getType().equals(Manifest.class)) {
+         Class unionClass = (Class)((ParameterizedType)f.getGenericType()).getActualTypeArguments()[0];
+         Class companionUnionClass = getCompanionClass(unionClass);
+         try {
+           Object companionUnionObj = companionUnionClass.getField("MODULE$").get(null);
+           ThriftStructFieldInfo info = (ThriftStructFieldInfo)companionUnionClass.getMethod("fieldInfo").invoke(companionUnionObj);
+           fields.add(info);
+         } catch (ReflectiveOperationException e) {
+           throw new ScroogeSchemaConversionException("can not find fiedInfo for " + unionClass,
e);
+         }
       }
     }
     return fields;
   }
 
+  private boolean isUnion(Class klass){
+    for(Field f: klass.getDeclaredFields()) {
+         if (f.getName().equals("Union"))
+           return true;
+    }
+    return false;
+  }
+
+
+  private Requirement getRequirementType(ThriftStructFieldInfo f) {
+    if (f.isOptional() && !f.isRequired()) {
+      return OPTIONAL;
+    } else if (f.isRequired() && !f.isOptional()) {
+      return REQUIRED;
+    } else if (!f.isOptional() && !f.isRequired()) {
+      return DEFAULT;
+    } else {
+      throw new ScroogeSchemaConversionException("can not determine requirement type for
: " + f.toString()
+              + ", isOptional=" + f.isOptional() + ", isRequired=" + f.isRequired());
+    }
+  }
 
   /**
-   * Convert a field in scrooge to ThriftField in parquet
+   * Convert thrift field in scrooge to ThriftField in parquet
+   * Use reflection to detect if a field is optional or required since scrooge does not provide
requirement information
+   * in generated classes.
+   * This will not correctly recognize fields that are not specified with a requirement type
eg.
+   * struct Address {
+   * 1: string street
+   * }
+   * street will be identified as "REQUIRED"
+   *
+   * @param scroogeField
+   * @return
+   * @throws Exception
    */
   public ThriftField toThriftField(ThriftStructFieldInfo scroogeField) {
     Requirement requirement = getRequirementType(scroogeField);
@@ -161,87 +169,70 @@ public class ScroogeStructConverter {
     ThriftTypeID typeId = ThriftTypeID.fromByte(thriftTypeByte);
     ThriftType thriftType;
     switch (typeId) {
-      case BOOL:
-        thriftType = new ThriftType.BoolType();
-        break;
-      case BYTE:
-        thriftType = new ThriftType.ByteType();
-        break;
-      case DOUBLE:
-        thriftType = new ThriftType.DoubleType();
-        break;
-      case I16:
-        thriftType = new ThriftType.I16Type();
-        break;
-      case I32:
-        thriftType = new ThriftType.I32Type();
-        break;
-      case I64:
-        thriftType = new ThriftType.I64Type();
-        break;
-      case STRING:
-        thriftType = new ThriftType.StringType();
-        break;
-      case STRUCT:
-        thriftType = convertStructTypeField(scroogeField);
-        break;
-      case MAP:
-        thriftType = convertMapTypeField(scroogeField, requirement);
-        break;
-      case SET:
-        thriftType = convertSetTypeField(scroogeField, requirement);
-        break;
-      case LIST:
-        thriftType = convertListTypeField(scroogeField, requirement);
-        break;
-      case ENUM:
-        thriftType = convertEnumTypeField(scroogeField);
-        break;
-      case STOP:
-      case VOID:
-      default:
-        throw new IllegalArgumentException("can't convert type " + typeId);
+    case BOOL:
+      thriftType = new ThriftType.BoolType();
+      break;
+    case BYTE:
+      thriftType = new ThriftType.ByteType();
+      break;
+    case DOUBLE:
+      thriftType = new ThriftType.DoubleType();
+      break;
+    case I16:
+      thriftType = new ThriftType.I16Type();
+      break;
+    case I32:
+      thriftType = new ThriftType.I32Type();
+      break;
+    case I64:
+      thriftType = new ThriftType.I64Type();
+      break;
+    case STRING:
+      thriftType = new ThriftType.StringType();
+      break;
+    case STRUCT:
+      thriftType = convertStructTypeField(scroogeField);
+      break;
+    case MAP:
+      thriftType = convertMapTypeField(scroogeField, requirement);
+      break;
+    case SET:
+      thriftType = convertSetTypeField(scroogeField, requirement);
+      break;
+    case LIST:
+      thriftType = convertListTypeField(scroogeField, requirement);
+      break;
+    case ENUM:
+      thriftType = convertEnumTypeField(scroogeField);
+      break;
+    case STOP:
+    case VOID:
+    default:
+      throw new IllegalArgumentException("can't convert type " + typeId);
     }
     return new ThriftField(fieldName, fieldId, requirement, thriftType);
   }
 
   private ThriftType convertSetTypeField(ThriftStructFieldInfo f, Requirement requirement)
{
-    return convertSetTypeField(f.tfield().name, f.valueManifest().get(), requirement);
-  }
-
-  private ThriftType convertSetTypeField(String fieldName, Manifest<?> valueManifest,
Requirement requirement) {
-    String elemName = setElemName(fieldName);
-    ThriftType elementType = convertClassToThriftType(elemName, requirement, valueManifest);
+    ThriftType elementType = convertClassToThriftType(f.valueManifest().get().runtimeClass());
     //Set only has one sub-field as element field, therefore using hard-coded 1 as fieldId,
     //it's the same as the solution used in ElephantBird
-    ThriftField elementField = generateFieldWithoutId(elemName, requirement, elementType);
+    ThriftField elementField = generateFieldWithoutId(f.tfield().name, requirement, elementType);
     return new ThriftType.SetType(elementField);
   }
 
   private ThriftType convertListTypeField(ThriftStructFieldInfo f, Requirement requirement)
{
-    return convertListTypeField(f.tfield().name, f.valueManifest().get(), requirement);
-  }
-
-  private ThriftType convertListTypeField(String fieldName, Manifest<?> valueManifest,
Requirement requirement) {
-    String elemName = listElemName(fieldName);
-    ThriftType elementType = convertClassToThriftType(elemName, requirement, valueManifest);
-    ThriftField elementField = generateFieldWithoutId(elemName, requirement, elementType);
+    ThriftType elementType = convertClassToThriftType(f.valueManifest().get().runtimeClass());
+    ThriftField elementField = generateFieldWithoutId(f.tfield().name, requirement, elementType);
     return new ThriftType.ListType(elementField);
   }
 
   private ThriftType convertMapTypeField(ThriftStructFieldInfo f, Requirement requirement)
{
-    return convertMapTypeField(f.tfield().name, f.keyManifest().get(), f.valueManifest().get(),
requirement);
-  }
+    ThriftType keyType = convertClassToThriftType(f.keyManifest().get().runtimeClass());
+    ThriftField keyField = generateFieldWithoutId(f.tfield().name + "_map_key", requirement,
keyType);
 
-  private ThriftType convertMapTypeField(String fieldName, Manifest<?> keyManifest,
Manifest<?> valueManifest, Requirement requirement) {
-
-    String keyName = mapKeyName(fieldName);
-    String valueName = mapValueName(fieldName);
-    ThriftType keyType = convertClassToThriftType(keyName, requirement, keyManifest);
-    ThriftField keyField = generateFieldWithoutId(keyName, requirement, keyType);
-
-    ThriftType valueType = convertClassToThriftType(valueName, requirement, valueManifest);
-    ThriftField valueField = generateFieldWithoutId(valueName, requirement, valueType);
+    ThriftType valueType = convertClassToThriftType(f.valueManifest().get().runtimeClass());
+    ThriftField valueField = generateFieldWithoutId(f.tfield().name + "_map_value", requirement,
valueType);
 
     return new ThriftType.MapType(keyField, valueField);
   }
@@ -249,20 +240,26 @@ public class ScroogeStructConverter {
   /**
    * Generate artificial field, this kind of fields do not have a field ID.
    * To be consistent with the behavior in ElephantBird, here uses 1 as the field ID
+   *
+   * @param fieldName
+   * @param requirement
+   * @param thriftType
+   * @return
    */
   private ThriftField generateFieldWithoutId(String fieldName, Requirement requirement, ThriftType
thriftType) {
-    return new ThriftField(fieldName, (short) 1, requirement, thriftType);
+    return new ThriftField(fieldName, (short)1, requirement, thriftType);
   }
 
   /**
    * In composite types,  such as the type of the key in a map, since we use reflection to
get the type class, this method
    * does conversion based on the class provided.
    *
-   * @return converted ThriftType
+   * @param typeClass
+   * @return
+   * @throws Exception
    */
-  private ThriftType convertClassToThriftType(String name, Requirement requirement, Manifest<?>
typeManifest) {
-    Class typeClass = typeManifest.runtimeClass();
-    if (typeManifest.runtimeClass() == boolean.class) {
+  private ThriftType convertClassToThriftType(Class typeClass) {
+    if (typeClass == boolean.class) {
       return new ThriftType.BoolType();
     } else if (typeClass == byte.class) {
       return new ThriftType.ByteType();
@@ -276,19 +273,6 @@ public class ScroogeStructConverter {
       return new ThriftType.I64Type();
     } else if (typeClass == String.class) {
       return new ThriftType.StringType();
-    } else if (typeClass == scala.collection.Seq.class) {
-      Manifest<?> a = typeManifest.typeArguments().apply(0);
-      return convertListTypeField(name, a, requirement);
-    } else if (typeClass == scala.collection.Set.class) {
-      Manifest<?> setElementManifest = typeManifest.typeArguments().apply(0);
-      return convertSetTypeField(name, setElementManifest, requirement);
-    } else if (typeClass == scala.collection.Map.class) {
-      List<Manifest<?>> ms = JavaConversions.seqAsJavaList(typeManifest.typeArguments());
-      Manifest keyManifest = ms.get(0);
-      Manifest valueManifest = ms.get(1);
-      return convertMapTypeField(name, keyManifest, valueManifest, requirement);
-    } else if (com.twitter.scrooge.ThriftEnum.class.isAssignableFrom(typeClass)) {
-      return convertEnumTypeField(typeClass, name);
     } else {
       return convertStructFromClass(typeClass);
     }
@@ -307,65 +291,41 @@ public class ScroogeStructConverter {
     Object cObject = companionObjectClass.getField("MODULE$").get(null);
     Method listMethod = companionObjectClass.getMethod("list", new Class[]{});
     Object result = listMethod.invoke(cObject, null);
-    return JavaConversions.seqAsJavaList((Seq) result);
+    return JavaConversions.seqAsJavaList((Seq)result);
   }
 
   public ThriftType convertEnumTypeField(ThriftStructFieldInfo f) {
-    return convertEnumTypeField(f.manifest().runtimeClass(), f.tfield().name);
-  }
-
-  private ThriftType convertEnumTypeField(Class enumClass, String fieldName) {
     List<ThriftType.EnumValue> enumValues = new ArrayList<ThriftType.EnumValue>();
-    String enumName = enumClass.getName();
+
+    String enumName = f.manifest().runtimeClass().getName();
     try {
       List enumCollection = getEnumList(enumName);
       for (Object enumObj : enumCollection) {
-        ScroogeEnumDesc enumDesc = ScroogeEnumDesc.fromEnum(enumObj);
+        ScroogeEnumDesc enumDesc = ScroogeEnumDesc.getEnumDesc(enumObj);
+        //be compatible with thrift generated enum which have capitalized name
         enumValues.add(new ThriftType.EnumValue(enumDesc.id, enumDesc.originalName));
       }
       return new ThriftType.EnumType(enumValues);
     } catch (ReflectiveOperationException e) {
-      throw new ScroogeSchemaConversionException("Can not convert enum field " + fieldName,
e);
-    } catch (RuntimeException e) {
-      throw new ScroogeSchemaConversionException("Can not convert enum field " + fieldName,
e);
-    }
-
-  }
-
-  //In scrooge generated class, if a class is a union, then it must have a field called "Union"
-  private boolean isUnion(Class klass) {
-    for (Field f : klass.getDeclaredFields()) {
-      if (f.getName().equals("Union"))
-        return true;
+      throw new ScroogeSchemaConversionException("Can not convert enum field " + f, e);
     }
-    return false;
-  }
 
-
-  private Requirement getRequirementType(ThriftStructFieldInfo f) {
-    if (f.isOptional() && !f.isRequired()) {
-      return OPTIONAL;
-    } else if (f.isRequired() && !f.isOptional()) {
-      return REQUIRED;
-    } else if (!f.isOptional() && !f.isRequired()) {
-      return DEFAULT;
-    } else {
-      throw new ScroogeSchemaConversionException("can not determine requirement type for
: " + f.toString()
-          + ", isOptional=" + f.isOptional() + ", isRequired=" + f.isRequired());
-    }
   }
 
   private static class ScroogeEnumDesc {
     private int id;
+    private String name;
     private String originalName;
 
-    public static ScroogeEnumDesc fromEnum(Object rawScroogeEnum) throws NoSuchMethodException,
InvocationTargetException, IllegalAccessException {
+    public static ScroogeEnumDesc getEnumDesc(Object rawScroogeEnum) throws NoSuchMethodException,
InvocationTargetException, IllegalAccessException {
       Class enumClass = rawScroogeEnum.getClass();
       Method valueMethod = enumClass.getMethod("value", new Class[]{});
+      Method nameMethod = enumClass.getMethod("name", new Class[]{});
       Method originalNameMethod = enumClass.getMethod("originalName", new Class[]{});
       ScroogeEnumDesc result = new ScroogeEnumDesc();
-      result.id = (Integer) valueMethod.invoke(rawScroogeEnum, null);
-      result.originalName = (String) originalNameMethod.invoke(rawScroogeEnum, null);
+      result.id = (Integer)valueMethod.invoke(rawScroogeEnum, null);
+      result.name = (String)nameMethod.invoke(rawScroogeEnum, null);
+      result.originalName = (String)originalNameMethod.invoke(rawScroogeEnum, null);
       return result;
     }
   }

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/720b988f/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java
b/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java
index a20cc5b..3dc5369 100644
--- a/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java
+++ b/parquet-scrooge/src/test/java/parquet/scrooge/ScroogeStructConverterTest.java
@@ -18,22 +18,9 @@
  */
 package parquet.scrooge;
 
-import org.apache.thrift.TBase;
 import org.junit.Test;
 
-import parquet.schema.MessageType;
 import parquet.scrooge.test.AddressWithStreetWithDefaultRequirement;
-import parquet.scrooge.test.ListNestEnum;
-import parquet.scrooge.test.ListNestMap;
-import parquet.scrooge.test.ListNestSet;
-import parquet.scrooge.test.MapNestList;
-import parquet.scrooge.test.MapNestMap;
-import parquet.scrooge.test.MapNestSet;
-import parquet.scrooge.test.NestedList;
-import parquet.scrooge.test.SetNestList;
-import parquet.scrooge.test.SetNestMap;
-import parquet.scrooge.test.SetNestSet;
-import parquet.scrooge.test.StringAndBinary;
 import parquet.scrooge.test.TestFieldOfEnum;
 import parquet.scrooge.test.TestListPrimitive;
 import parquet.scrooge.test.TestMapComplex;
@@ -43,6 +30,7 @@ import parquet.scrooge.test.TestOptionalMap;
 import parquet.scrooge.test.TestPersonWithAllInformation;
 import parquet.scrooge.test.TestSetPrimitive;
 import parquet.scrooge.test.TestUnion;
+import parquet.scrooge.test.StringAndBinary;
 import parquet.thrift.ThriftSchemaConverter;
 import parquet.thrift.struct.ThriftType;
 import static org.junit.Assert.assertEquals;
@@ -51,128 +39,85 @@ import static org.junit.Assert.assertEquals;
  * Test convert scrooge schema to Parquet Schema
  */
 public class ScroogeStructConverterTest {
-
-  /**
-   * Convert ThriftStructs from a thrift class and a scrooge class, assert
-   * they are the same
-   * @param scroogeClass
-   */
-  private void shouldConvertConsistentlyWithThriftStructConverter(Class scroogeClass) throws
ClassNotFoundException {
-      Class<? extends TBase<?, ?>> thriftClass = (Class<? extends TBase<?,
?>>)Class.forName(scroogeClass.getName().replaceFirst("parquet.scrooge.test", "parquet.thrift.test"));
-      ThriftType.StructType structFromThriftSchemaConverter = new ThriftSchemaConverter().toStructType(thriftClass);
-      ThriftType.StructType structFromScroogeSchemaConverter = new ScroogeStructConverter().convert(scroogeClass);
-
-      assertEquals(toParquetSchema(structFromThriftSchemaConverter), toParquetSchema(structFromScroogeSchemaConverter));
-  }
-
-  private MessageType toParquetSchema(ThriftType.StructType struct) {
-    ThriftSchemaConverter sc = new ThriftSchemaConverter();
-    return sc.convert(struct);
-  }
-
   @Test
-  public void testConvertPrimitiveMapKey() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(TestMapPrimitiveKey.class);
+  public void testConvertPrimitiveMapKey() throws Exception{
+    ThriftType.StructType scroogeMap = new ScroogeStructConverter().convert(TestMapPrimitiveKey.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestMapPrimitiveKey.class);
+    assertEquals(expected,scroogeMap);
   }
 
   @Test
   public void testBinary() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(StringAndBinary.class);
+    ThriftType.StructType scroogeBinary = new ScroogeStructConverter().convert(StringAndBinary.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.StringAndBinary.class);
+    assertEquals(expected, scroogeBinary);
   }
 
   @Test
   public void testUnion() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(TestUnion.class);
-  }
-
-  @Test
-  public void testConvertPrimitiveMapValue() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(TestMapPrimitiveValue.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestUnion.class);
+    ThriftType.StructType scroogeUnion = new ScroogeStructConverter().convert(TestUnion.class);
+    assertEquals(expected, scroogeUnion);
   }
 
   @Test
-  public void testConvertPrimitiveList() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(TestListPrimitive.class);
+  public void testConvertPrimitiveMapValue() throws Exception{
+    ThriftType.StructType scroogeMap = new ScroogeStructConverter().convert(TestMapPrimitiveValue.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestMapPrimitiveValue.class);
+    assertEquals(expected,scroogeMap);
   }
 
   @Test
-  public void testConvertPrimitiveSet() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(TestSetPrimitive.class);
+  public void testConvertPrimitiveList() throws Exception{
+    ThriftType.StructType scroogeList = new ScroogeStructConverter().convert(TestListPrimitive.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestListPrimitive.class);
+    assertEquals(expected, scroogeList);
   }
 
   @Test
-  public void testConvertEnum() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(TestFieldOfEnum.class);
+     public void testConvertPrimitiveSet() throws Exception{
+    ThriftType.StructType scroogeList = new ScroogeStructConverter().convert(TestSetPrimitive.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestSetPrimitive.class);
+    assertEquals(expected, scroogeList);
   }
 
   @Test
-  public void testMapComplex() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(TestMapComplex.class);
+  public void testConvertEnum() throws Exception{
+    ThriftType.StructType scroogeList = new ScroogeStructConverter().convert(TestFieldOfEnum.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestFieldOfEnum.class);
+    assertEquals(expected, scroogeList);
   }
 
   @Test
-  public void testConvertStruct() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(TestPersonWithAllInformation.class);
+  public void testMapComplex() throws Exception{
+    ThriftType.StructType scroogePerson = new ScroogeStructConverter().convert(TestMapComplex.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestMapComplex.class);
+    assertEquals(expected, scroogePerson);
   }
 
   @Test
-  public void testDefaultFields() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(AddressWithStreetWithDefaultRequirement.class);
-  }
-
-  @Test
-  public void testConvertOptionalPrimitiveMap() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(TestOptionalMap.class);
-  }
-
-  @Test
-  public void testConvertNestedList() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(NestedList.class);
-  }
-
-  @Test
-  public void testConvertListNestMap() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(ListNestMap.class);
-  }
-
-  @Test
-  public void testConvertListNestEnum() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(ListNestEnum.class);
-  }
-
-  @Test
-  public void testConvertMapNestList() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(MapNestList.class);
-  }
-
-  @Test
-  public void testConvertMapNestMap() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(MapNestMap.class);
-  }
-
-  @Test
-  public void testConvertMapNestSet() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(MapNestSet.class);
-  }
-
-  @Test
-  public void testConvertListNestSet() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(ListNestSet.class);
-  }
-
-  @Test
-  public void testConvertSetNestSet() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(SetNestSet.class);
+  public void testConvertStruct() throws Exception{
+    ThriftType.StructType scroogePerson = new ScroogeStructConverter().convert(TestPersonWithAllInformation.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestPersonWithAllInformation.class);
+    assertEquals(expected, scroogePerson);
   }
 
+/**
+ * TODO: DEFAULT requirement can not be identified, since scrooge does not store the requirement
type in generated class
+ * Current solution uses reflection based on following rules:
+ * if the getter returns option, then it's optional, otherwise it's required
+ */
   @Test
-  public void testConvertSetNestList() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(SetNestList.class);
+  public void testDefaultFields() throws Exception{
+    ThriftType.StructType scroogePerson = new ScroogeStructConverter().convert(AddressWithStreetWithDefaultRequirement.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.AddressWithStreetWithDefaultRequirement.class);
+    assertEquals(expected.toJSON(), scroogePerson.toJSON());
   }
 
   @Test
-  public void testConvertSetNestMap() throws Exception {
-    shouldConvertConsistentlyWithThriftStructConverter(SetNestMap.class);
+  public void testConvertOptionalPrimitiveMap() throws Exception{
+    ThriftType.StructType scroogeMap = new ScroogeStructConverter().convert(TestOptionalMap.class);
+    ThriftType.StructType expected = new ThriftSchemaConverter().toStructType(parquet.thrift.test.TestOptionalMap.class);
+    assertEquals(expected,scroogeMap);
   }
-
 }

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/720b988f/parquet-scrooge/src/test/thrift/test.thrift
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/test/thrift/test.thrift b/parquet-scrooge/src/test/thrift/test.thrift
index f721193..a80bbb0 100644
--- a/parquet-scrooge/src/test/thrift/test.thrift
+++ b/parquet-scrooge/src/test/thrift/test.thrift
@@ -169,95 +169,6 @@ struct TestFieldOfEnum{
 }
 
 struct StringAndBinary {
-  1: required string s
-  2: required binary b
-}
-
-#fixture fox nested structures
-struct NestedList {
-  1: required list<list<Address>> rll
-  2: required list<list<list<Address>>> rlll
-  3: optional list<list<Address>> oll
-  4: optional list<list<list<Address>>> olll
-  5: list<list<Address>> ll
-  6: list<list<list<Address>>> lll
-}
-
-struct ListNestMap {
-  1: required list<map<Phone, Address>> rlm
-  2: required list<list<map<Phone, Address>>> rllm
-  3: optional list<map<Phone, Address>> olm
-  4: optional list<list<map<Phone, Address>>> ollm
-  5: list<map<Phone, Address>> lm
-  6: list<list<map<Phone, Address>>> llm
-}
-
-struct ListNestSet {
-   1: required list<set<Address>> rls
-   2: required list<list<set<Address>>> rlls
-   3: optional list<set<Address>> ols
-   4: optional list<list<set<Address>>> olls
-   5: list<set<Address>> ls
-   6: list<list<set<Address>>> lls
-}
-
-struct ListNestEnum {
-   1: required list<Operation> rle
-}
-
-struct MapNestMap {
-  1: required map<map<Phone, Address>, map<Address, Phone>> rmm
-  2: required map<map<map<Phone,Address>, Address>, map<Address, Phone>>
rmmm
-  3: optional map<map<Phone, Address>, map<Address, Phone>> omm
-  4: optional map<map<map<Phone,Address>, Address>, map<Address, Phone>>
ommm
-  5: map<map<Phone, Address>, map<Address, Phone>> mm
-  6: map<map<map<Phone,Address>, Address>, map<Address, Phone>> mmm
-}
-
-struct MapNestList {
-  1: required map<list<Phone>, list<Address>> rml
-  2: required map<list<list<Phone>>, list<list<Address>>> rmll
-  3: optional map<list<Phone>, list<Address>> oml
-  4: optional map<list<list<Phone>>, list<list<Address>>> omll
-  5: map<list<Phone>, list<Address>> ml
-  6: map<list<list<Phone>>, list<list<Address>>> mll
-}
-
-struct MapNestSet {
-  1: required map<set<Phone>, set<Address>> rms
-  2: required map<set<set<Phone>>, set<set<Address>>> rmss
-  3: optional map<set<Phone>, set<Address>> oms
-  4: optional map<set<set<Phone>>, set<set<Address>>> omss
-  5: map<set<Phone>, set<Address>> ms
-  6: map<set<set<Phone>>, set<set<Address>>> mss
-}
-
-struct SetNestSet {
-  1: required set<set<Address>> rss
-  2: required set<set<set<Address>>> rsss
-  3: optional set<set<Address>> oss
-  4: optional set<set<set<Address>>> osss
-  5: set<set<Address>> ss
-  6: set<set<set<Address>>> sss
-}
-
-struct SetNestList {
-   1: required set<list<Address>> rsl
-   2: required set<set<list<Address>>> rssl
-   3: optional set<list<Address>> osl
-   4: optional set<set<list<Address>>> ossl
-   5: set<list<Address>> sl
-   6: set<set<list<Address>>> ssl
-}
-
-struct SetNestMap {
-  1: required set<map<Phone, Address>> rsm
-  2: required set<set<map<Phone, Address>>> rssm
-  3: required set<set<list<list<map<Phone, Address>>>>> rssllm
-  4: optional set<map<Phone, Address>> osm
-  5: optional set<set<map<Phone, Address>>> ossm
-  6: optional set<set<list<list<map<Phone, Address>>>>> ossllm
-  7: set<map<Phone, Address>> sm
-  8: set<set<map<Phone, Address>>> ssm
-  9: set<set<list<list<map<Phone, Address>>>>> ssllm
+  1: required string s;
+  2: required binary b;
 }


Mime
View raw message