drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From meh...@apache.org
Subject [1/2] drill git commit: DRILL-2848: Hygiene - ParquetToDrillConverter cleanup
Date Fri, 08 May 2015 03:27:36 GMT
Repository: drill
Updated Branches:
  refs/heads/master 79a712aae -> 7abd7cf4e


DRILL-2848: Hygiene - ParquetToDrillConverter cleanup


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a8fa4023
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a8fa4023
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a8fa4023

Branch: refs/heads/master
Commit: a8fa402365831c89f1a5b7fbd9e68f862104a026
Parents: 79a712a
Author: Mehant Baid <mehantr@gmail.com>
Authored: Fri May 1 14:12:09 2015 -0700
Committer: Mehant Baid <mehantr@gmail.com>
Committed: Thu May 7 16:31:01 2015 -0700

----------------------------------------------------------------------
 .../drill/common/util/CoreDecimalUtility.java   |   5 +-
 .../ParquetToDrillTypeConverter.java            | 258 +++++--------------
 2 files changed, 73 insertions(+), 190 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/a8fa4023/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
index 302652e..9f61ccf 100644
--- a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
+++ b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
@@ -78,7 +78,10 @@ public class CoreDecimalUtility {
    * Helper function to detect if the given data type is Decimal
    */
   public static boolean isDecimalType(TypeProtos.MajorType type) {
-    TypeProtos.MinorType minorType = type.getMinorType();
+    return isDecimalType(type.getMinorType());
+  }
+
+  public static boolean isDecimalType(TypeProtos.MinorType minorType) {
     if (minorType == TypeProtos.MinorType.DECIMAL9 || minorType == TypeProtos.MinorType.DECIMAL18
||
         minorType == TypeProtos.MinorType.DECIMAL28SPARSE || minorType == TypeProtos.MinorType.DECIMAL38SPARSE)
{
       return true;

http://git-wip-us.apache.org/repos/asf/drill/blob/a8fa4023/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
index 7c3eeb8..53f059e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
@@ -20,10 +20,9 @@ package org.apache.drill.exec.store.parquet.columnreaders;
 import static parquet.Preconditions.checkArgument;
 
 import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
 
+import org.apache.drill.common.util.CoreDecimalUtility;
 import parquet.format.ConvertedType;
 import parquet.format.SchemaElement;
 import parquet.schema.PrimitiveType;
@@ -34,202 +33,83 @@ public class ParquetToDrillTypeConverter {
     return schemaElement.getPrecision() <= 28 ? TypeProtos.MinorType.DECIMAL28SPARSE :
MinorType.DECIMAL38SPARSE;
   }
 
-  public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName primitiveTypeName,
int length,
-                                          TypeProtos.DataMode mode, SchemaElement schemaElement)
{
+  private static TypeProtos.MinorType getMinorType(PrimitiveType.PrimitiveTypeName primitiveTypeName,
int length,
+                                                   SchemaElement schemaElement) {
+
     ConvertedType convertedType = schemaElement.getConverted_type();
-    switch (mode) {
 
-      case OPTIONAL:
-        switch (primitiveTypeName) {
-          case BINARY:
-            if (convertedType == null) {
-              return Types.optional(TypeProtos.MinorType.VARBINARY);
-            }
-            switch (convertedType) {
-              case UTF8:
-                return Types.optional(TypeProtos.MinorType.VARCHAR);
-              case DECIMAL:
-                return Types.withScaleAndPrecision(getDecimalType(schemaElement), TypeProtos.DataMode.OPTIONAL,
schemaElement.getScale(), schemaElement.getPrecision());
-              default:
-                return Types.optional(TypeProtos.MinorType.VARBINARY);
-            }
-          case INT64:
-            if (convertedType == null) {
-              return Types.optional(TypeProtos.MinorType.BIGINT);
-            }
-            switch(convertedType) {
-              case DECIMAL:
-                return Types.withScaleAndPrecision(TypeProtos.MinorType.DECIMAL18, DataMode.OPTIONAL,
schemaElement.getScale(), schemaElement.getPrecision());
-              // TODO - add this back if it is decided to be added upstream, was removed
form our pull request July 2014
-//              case TIME_MICROS:
-//                throw new UnsupportedOperationException();
-              case TIMESTAMP_MILLIS:
-                return Types.optional(MinorType.TIMESTAMP);
-              default:
-                throw new UnsupportedOperationException(String.format("unsupported type:
%s %s", primitiveTypeName, convertedType));
-            }
-          case INT32:
-            if (convertedType == null) {
-              return Types.optional(TypeProtos.MinorType.INT);
-            }
-            switch(convertedType) {
-              case DECIMAL:
-                return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.OPTIONAL,
schemaElement.getScale(), schemaElement.getPrecision());
-              case DATE:
-                return Types.optional(MinorType.DATE);
-              case TIME_MILLIS:
-                return Types.optional(MinorType.TIME);
-              default:
-                throw new UnsupportedOperationException(String.format("unsupported type:
%s %s", primitiveTypeName, convertedType));
-            }
-          case BOOLEAN:
-            return Types.optional(TypeProtos.MinorType.BIT);
-          case FLOAT:
-            return Types.optional(TypeProtos.MinorType.FLOAT4);
-          case DOUBLE:
-            return Types.optional(TypeProtos.MinorType.FLOAT8);
-          // TODO - Both of these are not supported by the parquet library yet (7/3/13),
-          // but they are declared here for when they are implemented
-          case INT96:
-            return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
-                .setMode(mode).build();
-          case FIXED_LEN_BYTE_ARRAY:
-            if (convertedType == null) {
-              checkArgument(length > 0, "A length greater than zero must be provided for
a FixedBinary type.");
-              return TypeProtos.MajorType.newBuilder().setMinorType(MinorType.VARBINARY).setMode(mode).build();
-            } else if (convertedType == ConvertedType.DECIMAL) {
-              return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.OPTIONAL,
schemaElement.getScale(), schemaElement.getPrecision());
-            }
+    switch (primitiveTypeName) {
+      case BINARY:
+        if (convertedType == null) {
+          return (TypeProtos.MinorType.VARBINARY);
+        }
+        switch (convertedType) {
+          case UTF8:
+            return (TypeProtos.MinorType.VARCHAR);
+          case DECIMAL:
+            return (getDecimalType(schemaElement));
           default:
-            throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+            return (TypeProtos.MinorType.VARBINARY);
+        }
+      case INT64:
+        if (convertedType == null) {
+          return (TypeProtos.MinorType.BIGINT);
         }
-      case REQUIRED:
-        switch (primitiveTypeName) {
-          case BINARY:
-            if (convertedType == null) {
-              return Types.required(TypeProtos.MinorType.VARBINARY);
-            }
-            switch (convertedType) {
-              case UTF8:
-                return Types.required(MinorType.VARCHAR);
-              case DECIMAL:
-                return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.REQUIRED,
schemaElement.getScale(), schemaElement.getPrecision());
-              default:
-                return Types.required(TypeProtos.MinorType.VARBINARY);
-            }
-          case INT64:
-            if (convertedType == null) {
-              return Types.required(MinorType.BIGINT);
-            }
-            switch(convertedType) {
-              case DECIMAL:
-                return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REQUIRED,
schemaElement.getScale(), schemaElement.getPrecision());
-//              case FINETIME:
+        switch(convertedType) {
+          case DECIMAL:
+            return TypeProtos.MinorType.DECIMAL18;
+          // TODO - add this back if it is decided to be added upstream, was removed form
our pull request July 2014
+//              case TIME_MICROS:
 //                throw new UnsupportedOperationException();
-              case TIMESTAMP_MILLIS:
-                return Types.required(MinorType.TIMESTAMP);
-              default:
-                throw new UnsupportedOperationException(String.format("unsupported type:
%s %s", primitiveTypeName, convertedType));
-            }
-          case INT32:
-            if (convertedType == null) {
-              return Types.required(MinorType.INT);
-            }
-            switch(convertedType) {
-              case DECIMAL:
-                return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REQUIRED,
schemaElement.getScale(), schemaElement.getPrecision());
-              case DATE:
-                return Types.required(MinorType.DATE);
-              case TIME_MILLIS:
-                return Types.required(MinorType.TIME);
-              default:
-                throw new UnsupportedOperationException(String.format("unsupported type:
%s %s", primitiveTypeName, convertedType));
-            }
-          case BOOLEAN:
-            return Types.required(TypeProtos.MinorType.BIT);
-          case FLOAT:
-            return Types.required(TypeProtos.MinorType.FLOAT4);
-          case DOUBLE:
-            return Types.required(TypeProtos.MinorType.FLOAT8);
-          // Both of these are not supported by the parquet library yet (7/3/13),
-          // but they are declared here for when they are implemented
-          case INT96:
-            return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
-                .setMode(mode).build();
-          case FIXED_LEN_BYTE_ARRAY:
-            if (convertedType == null) {
-              checkArgument(length > 0, "A length greater than zero must be provided for
a FixedBinary type.");
-              return TypeProtos.MajorType.newBuilder().setMinorType(MinorType.VARBINARY).setMode(mode).build();
-            } else if (convertedType == ConvertedType.DECIMAL) {
-              return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.REQUIRED,
schemaElement.getScale(), schemaElement.getPrecision());
-            }
+          case TIMESTAMP_MILLIS:
+            return TypeProtos.MinorType.TIMESTAMP;
           default:
-            throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+            throw new UnsupportedOperationException(String.format("unsupported type: %s %s",
primitiveTypeName, convertedType));
         }
-      case REPEATED:
-        switch (primitiveTypeName) {
-          case BINARY:
-            if (convertedType == null) {
-              return Types.repeated(TypeProtos.MinorType.VARBINARY);
-            }
-            switch (schemaElement.getConverted_type()) {
-              case UTF8:
-                return Types.repeated(MinorType.VARCHAR);
-              case DECIMAL:
-                return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.REPEATED,
schemaElement.getScale(), schemaElement.getPrecision());
-              default:
-                return Types.repeated(TypeProtos.MinorType.VARBINARY);
-            }
-          case INT64:
-            if (convertedType == null) {
-              return Types.repeated(MinorType.BIGINT);
-            }
-            switch(convertedType) {
-              case DECIMAL:
-                return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REPEATED,
schemaElement.getScale(), schemaElement.getPrecision());
-//              case FINETIME:
-//                throw new UnsupportedOperationException();
-              case TIMESTAMP_MILLIS:
-                return Types.repeated(MinorType.TIMESTAMP);
-              default:
-                throw new UnsupportedOperationException(String.format("unsupported type:
%s %s", primitiveTypeName, convertedType));
-            }
-          case INT32:
-            if (convertedType == null) {
-              return Types.repeated(MinorType.INT);
-            }
-            switch(convertedType) {
-              case DECIMAL:
-                return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REPEATED,
schemaElement.getScale(), schemaElement.getPrecision());
-              case DATE:
-                return Types.repeated(MinorType.DATE);
-              case TIME_MILLIS:
-                return Types.repeated(MinorType.TIME);
-              default:
-                throw new UnsupportedOperationException(String.format("unsupported type:
%s %s", primitiveTypeName, convertedType));
-            }
-          case BOOLEAN:
-            return Types.repeated(TypeProtos.MinorType.BIT);
-          case FLOAT:
-            return Types.repeated(TypeProtos.MinorType.FLOAT4);
-          case DOUBLE:
-            return Types.repeated(TypeProtos.MinorType.FLOAT8);
-          // Both of these are not supported by the parquet library yet (7/3/13),
-          // but they are declared here for when they are implemented
-          case INT96:
-            return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
-                .setMode(mode).build();
-          case FIXED_LEN_BYTE_ARRAY:
-            if (convertedType == null) {
-              checkArgument(length > 0, "A length greater than zero must be provided for
a FixedBinary type.");
-              return TypeProtos.MajorType.newBuilder().setMinorType(MinorType.VARBINARY).setMode(mode).build();
-            } else if (convertedType == ConvertedType.DECIMAL) {
-              return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.REPEATED,
schemaElement.getScale(), schemaElement.getPrecision());
-            }
+      case INT32:
+        if (convertedType == null) {
+          return TypeProtos.MinorType.INT;
+        }
+        switch(convertedType) {
+          case DECIMAL:
+            return TypeProtos.MinorType.DECIMAL9;
+          case DATE:
+            return TypeProtos.MinorType.DATE;
+          case TIME_MILLIS:
+            return TypeProtos.MinorType.TIME;
           default:
-            throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+            throw new UnsupportedOperationException(String.format("unsupported type: %s %s",
primitiveTypeName, convertedType));
+        }
+      case BOOLEAN:
+        return TypeProtos.MinorType.BIT;
+      case FLOAT:
+        return TypeProtos.MinorType.FLOAT4;
+      case DOUBLE:
+        return TypeProtos.MinorType.FLOAT8;
+      // TODO - Both of these are not supported by the parquet library yet (7/3/13),
+      // but they are declared here for when they are implemented
+      case INT96:
+        return TypeProtos.MinorType.FIXEDBINARY;
+      case FIXED_LEN_BYTE_ARRAY:
+        if (convertedType == null) {
+          checkArgument(length > 0, "A length greater than zero must be provided for a
FixedBinary type.");
+          return TypeProtos.MinorType.VARBINARY;
+        } else if (convertedType == ConvertedType.DECIMAL) {
+          return getDecimalType(schemaElement);
         }
+      default:
+        throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+    }
+  }
+
+  public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName primitiveTypeName,
int length,
+                                          TypeProtos.DataMode mode, SchemaElement schemaElement)
{
+    MinorType minorType = getMinorType(primitiveTypeName, length, schemaElement);
+    TypeProtos.MajorType.Builder typeBuilder = TypeProtos.MajorType.newBuilder().setMinorType(minorType).setMode(mode);
+
+    if (CoreDecimalUtility.isDecimalType(minorType)) {
+      typeBuilder.setPrecision(schemaElement.getPrecision()).setScale(schemaElement.getScale());
     }
-    throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName +
" Mode: " + mode);
+    return typeBuilder.build();
   }
 }


Mime
View raw message