spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wenc...@apache.org
Subject spark git commit: [SPARK-16926][SQL] Add unit test to compare table and partition column metadata.
Date Fri, 02 Sep 2016 03:12:31 GMT
Repository: spark
Updated Branches:
  refs/heads/master 06e33985c -> f2d6e2ef2


[SPARK-16926][SQL] Add unit test to compare table and partition column metadata.

## What changes were proposed in this pull request?

Add unit test for changes made in PR #14515. It makes sure that a newly created table has
the same number of columns in table and partition metadata. This test fails before the changes
introduced in #14515.

## How was this patch tested?

Run new unit test.

Author: Brian Cho <bcho@fb.com>

Closes #14930 from dafrista/partition-metadata-unit-test.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f2d6e2ef
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f2d6e2ef
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f2d6e2ef

Branch: refs/heads/master
Commit: f2d6e2ef23b3f862c336ce5f7b98c43c3fde1e36
Parents: 06e3398
Author: Brian Cho <bcho@fb.com>
Authored: Fri Sep 2 11:12:34 2016 +0800
Committer: Wenchen Fan <wenchen@databricks.com>
Committed: Fri Sep 2 11:12:34 2016 +0800

----------------------------------------------------------------------
 .../sql/hive/execution/HiveTableScanSuite.scala | 35 ++++++++++++++++++++
 1 file changed, 35 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/f2d6e2ef/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
index 5b46476..5c460d2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.hive.execution
 
 import org.apache.spark.sql.Row
+import org.apache.spark.sql.hive.MetastoreRelation
 import org.apache.spark.sql.hive.test.{TestHive, TestHiveSingleton}
 import org.apache.spark.sql.hive.test.TestHive._
 import org.apache.spark.sql.hive.test.TestHive.implicits._
@@ -143,4 +144,38 @@ class HiveTableScanSuite extends HiveComparisonTest with SQLTestUtils
with TestH
       }
     }
   }
+
+  test("SPARK-16926: number of table and partition columns match for new partitioned table")
{
+    val view = "src"
+    withTempView(view) {
+      spark.range(1, 5).createOrReplaceTempView(view)
+      val table = "table_with_partition"
+      withTable(table) {
+        sql(
+          s"""
+             |CREATE TABLE $table(id string)
+             |PARTITIONED BY (p1 string,p2 string,p3 string,p4 string,p5 string)
+           """.stripMargin)
+        sql(
+          s"""
+             |FROM $view v
+             |INSERT INTO TABLE $table
+             |PARTITION (p1='a',p2='b',p3='c',p4='d',p5='e')
+             |SELECT v.id
+             |INSERT INTO TABLE $table
+             |PARTITION (p1='a',p2='c',p3='c',p4='d',p5='e')
+             |SELECT v.id
+           """.stripMargin)
+        val plan = sql(
+          s"""
+             |SELECT * FROM $table
+           """.stripMargin).queryExecution.sparkPlan
+        val relation = plan.collectFirst {
+          case p: HiveTableScanExec => p.relation
+        }.get
+        val tableCols = relation.hiveQlTable.getCols
+        relation.getHiveQlPartitions().foreach(p => assert(p.getCols.size == tableCols.size))
+      }
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message