spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject spark git commit: [SPARK-16632][SQL] Revert PR #14272: Respect Hive schema when merging parquet schema
Date Thu, 21 Jul 2016 14:08:40 GMT
Repository: spark
Updated Branches:
  refs/heads/master 6203668d5 -> 69626addd


[SPARK-16632][SQL] Revert PR #14272: Respect Hive schema when merging parquet schema

## What changes were proposed in this pull request?

PR #14278 is a more general and simpler fix for SPARK-16632 than PR #14272. After merging
#14278, we no longer need changes made in #14272. So here I revert them.

This PR targets both master and branch-2.0.

## How was this patch tested?

Existing tests.

Author: Cheng Lian <lian@databricks.com>

Closes #14300 from liancheng/revert-pr-14272.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/69626add
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/69626add
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/69626add

Branch: refs/heads/master
Commit: 69626adddc0441a4834b70a32e2d95b11d69a219
Parents: 6203668
Author: Cheng Lian <lian@databricks.com>
Authored: Thu Jul 21 22:08:34 2016 +0800
Committer: Cheng Lian <lian@databricks.com>
Committed: Thu Jul 21 22:08:34 2016 +0800

----------------------------------------------------------------------
 .../parquet/ParquetReadSupport.scala            | 18 ---------
 .../parquet/ParquetSchemaSuite.scala            | 39 --------------------
 2 files changed, 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/69626add/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
index 0bee874..8a2e0d7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
@@ -26,8 +26,6 @@ import org.apache.parquet.hadoop.api.{InitContext, ReadSupport}
 import org.apache.parquet.hadoop.api.ReadSupport.ReadContext
 import org.apache.parquet.io.api.RecordMaterializer
 import org.apache.parquet.schema._
-import org.apache.parquet.schema.OriginalType._
-import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._
 import org.apache.parquet.schema.Type.Repetition
 
 import org.apache.spark.internal.Logging
@@ -123,12 +121,6 @@ private[parquet] object ParquetReadSupport {
   }
 
   private def clipParquetType(parquetType: Type, catalystType: DataType): Type = {
-    val primName = if (parquetType.isPrimitive()) {
-      parquetType.asPrimitiveType().getPrimitiveTypeName()
-    } else {
-      null
-    }
-
     catalystType match {
       case t: ArrayType if !isPrimitiveCatalystType(t.elementType) =>
         // Only clips array types with nested type as element type.
@@ -143,16 +135,6 @@ private[parquet] object ParquetReadSupport {
       case t: StructType =>
         clipParquetGroup(parquetType.asGroupType(), t)
 
-      case _: ByteType if primName == INT32 =>
-        // SPARK-16632: Handle case where Hive stores bytes in a int32 field without specifying
-        // the original type.
-        Types.primitive(INT32, parquetType.getRepetition()).as(INT_8).named(parquetType.getName())
-
-      case _: ShortType if primName == INT32 =>
-        // SPARK-16632: Handle case where Hive stores shorts in a int32 field without specifying
-        // the original type.
-        Types.primitive(INT32, parquetType.getRepetition()).as(INT_16).named(parquetType.getName())
-
       case _ =>
         // UDTs and primitive types are not clipped.  For UDTs, a clipped version might not
be able
         // to be mapped to desired user-space types.  So UDTs shouldn't participate schema
merging.

http://git-wip-us.apache.org/repos/asf/spark/blob/69626add/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
index 31ebec0..8a980a7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
@@ -1581,43 +1581,4 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
         |  }
         |}
       """.stripMargin)
-
-  testSchemaClipping(
-    "int32 parquet field with byte schema field",
-
-    parquetSchema =
-      """message root {
-        |  optional int32 value;
-        |}
-      """.stripMargin,
-
-    catalystSchema =
-      new StructType()
-        .add("value", ByteType, nullable = true),
-
-    expectedSchema =
-      """message root {
-        |  optional int32 value (INT_8);
-        |}
-      """.stripMargin)
-
-  testSchemaClipping(
-    "int32 parquet field with short schema field",
-
-    parquetSchema =
-      """message root {
-        |  optional int32 value;
-        |}
-      """.stripMargin,
-
-    catalystSchema =
-      new StructType()
-        .add("value", ShortType, nullable = true),
-
-    expectedSchema =
-      """message root {
-        |  optional int32 value (INT_16);
-        |}
-      """.stripMargin)
-
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message