spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wenc...@apache.org
Subject spark git commit: [SPARK-15253][SQL] Support old table schema config key "spark.sql.sources.schema" for DESCRIBE TABLE
Date Mon, 16 May 2016 02:41:54 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-2.0 d305f7227 -> 4f2f96f5e


[SPARK-15253][SQL] Support old table schema config key "spark.sql.sources.schema" for DESCRIBE
TABLE

## What changes were proposed in this pull request?

"DESCRIBE table" is broken when table schema is stored at key "spark.sql.sources.schema".

Originally, we used spark.sql.sources.schema to store the schema of a data source table.
After SPARK-6024, we removed this flag. Although we are not using spark.sql.sources.schema
any more, we need to still support it.

## How was this patch tested?

Unit test.

When using spark2.0 to load a table generated by spark 1.2.
Before change:
`DESCRIBE table` => Schema of this table is inferred at runtime,,

After change:
`DESCRIBE table` => correct output.

Author: Sean Zhong <seanzhong@databricks.com>

Closes #13073 from clockfly/spark-15253.

(cherry picked from commit 4a5ee1954a6fb77231abb492355fe70313f0b35b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4f2f96f5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4f2f96f5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4f2f96f5

Branch: refs/heads/branch-2.0
Commit: 4f2f96f5e31004045d2ac2cd9f76467325ee9aa1
Parents: d305f72
Author: Sean Zhong <seanzhong@databricks.com>
Authored: Mon May 16 10:41:20 2016 +0800
Committer: Wenchen Fan <wenchen@databricks.com>
Committed: Mon May 16 10:41:44 2016 +0800

----------------------------------------------------------------------
 .../spark/sql/execution/command/ddl.scala       | 29 ++++++++++++--------
 .../sql/hive/MetastoreDataSourcesSuite.scala    |  3 ++
 2 files changed, 21 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/4f2f96f5/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index 1c1716f..49d7fe9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -497,20 +497,27 @@ private[sql] object DDLUtils {
   // will be inferred at runtime when the table is referenced.
   def getSchemaFromTableProperties(metadata: CatalogTable): Option[StructType] = {
     require(isDatasourceTable(metadata))
+    val props = metadata.properties
+    if (props.isDefinedAt("spark.sql.sources.schema")) {
+      // Originally, we used spark.sql.sources.schema to store the schema of a data source
table.
+      // After SPARK-6024, we removed this flag.
+      // Although we are not using spark.sql.sources.schema any more, we need to still support.
+      props.get("spark.sql.sources.schema").map(DataType.fromJson(_).asInstanceOf[StructType])
+    } else {
+      metadata.properties.get("spark.sql.sources.schema.numParts").map { numParts =>
+        val parts = (0 until numParts.toInt).map { index =>
+          val part = metadata.properties.get(s"spark.sql.sources.schema.part.$index").orNull
+          if (part == null) {
+            throw new AnalysisException(
+              "Could not read schema from the metastore because it is corrupted " +
+                s"(missing part $index of the schema, $numParts parts are expected).")
+          }
 
-    metadata.properties.get("spark.sql.sources.schema.numParts").map { numParts =>
-      val parts = (0 until numParts.toInt).map { index =>
-        val part = metadata.properties.get(s"spark.sql.sources.schema.part.$index").orNull
-        if (part == null) {
-          throw new AnalysisException(
-            "Could not read schema from the metastore because it is corrupted " +
-              s"(missing part $index of the schema, $numParts parts are expected).")
+          part
         }
-
-        part
+        // Stick all parts back to a single schema string.
+        DataType.fromJson(parts.mkString).asInstanceOf[StructType]
       }
-      // Stick all parts back to a single schema string.
-      DataType.fromJson(parts.mkString).asInstanceOf[StructType]
     }
   }
 

http://git-wip-us.apache.org/repos/asf/spark/blob/4f2f96f5/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index 676fbd0..b507018 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -746,6 +746,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with
TestHiv
       sessionState.invalidateTable(tableName)
       val actualSchema = table(tableName).schema
       assert(schema === actualSchema)
+
+      // Checks the DESCRIBE output.
+      checkAnswer(sql("DESCRIBE spark6655"), Row("int", "int", "") :: Nil)
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message