spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lix...@apache.org
Subject spark git commit: [SPARK-21089][SQL] Fix DESC EXTENDED/FORMATTED to Show Table Properties
Date Wed, 14 Jun 2017 18:20:17 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-2.2 626511953 -> 3dda682c4


[SPARK-21089][SQL] Fix DESC EXTENDED/FORMATTED to Show Table Properties

Since both table properties and storage properties share the same key values, table properties
are not shown in the output of DESC EXTENDED/FORMATTED when the storage properties are not
empty.

This PR is to fix the above issue by renaming them to different keys.

Added test cases.

Author: Xiao Li <gatorsmile@gmail.com>

Closes #18294 from gatorsmile/tableProperties.

(cherry picked from commit df766a471426625fe86c8845f6261e0fc087772d)
Signed-off-by: Xiao Li <gatorsmile@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3dda682c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3dda682c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3dda682c

Branch: refs/heads/branch-2.2
Commit: 3dda682c43daa5eb016d1c02a23cc40a443a6523
Parents: 6265119
Author: Xiao Li <gatorsmile@gmail.com>
Authored: Wed Jun 14 11:13:16 2017 -0700
Committer: Xiao Li <gatorsmile@gmail.com>
Committed: Wed Jun 14 11:19:28 2017 -0700

----------------------------------------------------------------------
 .../spark/sql/catalyst/catalog/interface.scala  |   4 +-
 .../resources/sql-tests/inputs/describe.sql     |   3 +
 .../sql-tests/results/describe.sql.out          | 183 ++++++++++---------
 3 files changed, 104 insertions(+), 86 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/3dda682c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index cc0cbba..976d787 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -75,7 +75,7 @@ case class CatalogStorageFormat(
     CatalogUtils.maskCredentials(properties) match {
       case props if props.isEmpty => // No-op
       case props =>
-        map.put("Properties", props.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]"))
+        map.put("Storage Properties", props.map(p => p._1 + "=" + p._2).mkString("[",
", ", "]"))
     }
     map
   }
@@ -313,7 +313,7 @@ case class CatalogTable(
       }
     }
 
-    if (properties.nonEmpty) map.put("Properties", tableProperties)
+    if (properties.nonEmpty) map.put("Table Properties", tableProperties)
     stats.foreach(s => map.put("Statistics", s.simpleString))
     map ++= storage.toLinkedHashMap
     if (tracksPartitionsInCatalog) map.put("Partition Provider", "Catalog")

http://git-wip-us.apache.org/repos/asf/spark/blob/3dda682c/sql/core/src/test/resources/sql-tests/inputs/describe.sql
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql
index 6de4cf0..91b9668 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql
@@ -1,4 +1,5 @@
 CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
+  OPTIONS (a '1', b '2')
   PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
   COMMENT 'table_comment';
 
@@ -13,6 +14,8 @@ CREATE TEMPORARY VIEW temp_Data_Source_View
 
 CREATE VIEW v AS SELECT * FROM t;
 
+ALTER TABLE t SET TBLPROPERTIES (e = '3');
+
 ALTER TABLE t ADD PARTITION (c='Us', d=1);
 
 DESCRIBE t;

http://git-wip-us.apache.org/repos/asf/spark/blob/3dda682c/sql/core/src/test/resources/sql-tests/results/describe.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index 46d32bb..329532c 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -1,9 +1,10 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 31
+-- Number of queries: 32
 
 
 -- !query 0
 CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
+  OPTIONS (a '1', b '2')
   PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
   COMMENT 'table_comment'
 -- !query 0 schema
@@ -42,7 +43,7 @@ struct<>
 
 
 -- !query 4
-ALTER TABLE t ADD PARTITION (c='Us', d=1)
+ALTER TABLE t SET TBLPROPERTIES (e = '3')
 -- !query 4 schema
 struct<>
 -- !query 4 output
@@ -50,10 +51,18 @@ struct<>
 
 
 -- !query 5
-DESCRIBE t
+ALTER TABLE t ADD PARTITION (c='Us', d=1)
 -- !query 5 schema
-struct<col_name:string,data_type:string,comment:string>
+struct<>
 -- !query 5 output
+
+
+
+-- !query 6
+DESCRIBE t
+-- !query 6 schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query 6 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -64,11 +73,11 @@ c                   	string
 d                   	string
 
 
--- !query 6
+-- !query 7
 DESC default.t
--- !query 6 schema
+-- !query 7 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 6 output
+-- !query 7 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -79,11 +88,11 @@ c                   	string
 d                   	string
 
 
--- !query 7
+-- !query 8
 DESC TABLE t
--- !query 7 schema
+-- !query 8 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 7 output
+-- !query 8 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -94,11 +103,11 @@ c                   	string
 d                   	string
 
 
--- !query 8
+-- !query 9
 DESC FORMATTED t
--- !query 8 schema
+-- !query 9 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 8 output
+-- !query 9 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -118,16 +127,17 @@ Provider            	parquet
 Num Buckets         	2                   	                    
 Bucket Columns      	[`a`]               	                    
 Sort Columns        	[`b`]               	                    
-Comment             	table_comment       	                    
+Table Properties    	[e=3]               	                    
 Location [not included in comparison]sql/core/spark-warehouse/t	                    
+Storage Properties  	[a=1, b=2]          	                    
 Partition Provider  	Catalog
 
 
--- !query 9
+-- !query 10
 DESC EXTENDED t
--- !query 9 schema
+-- !query 10 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 9 output
+-- !query 10 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -147,16 +157,17 @@ Provider            	parquet
 Num Buckets         	2                   	                    
 Bucket Columns      	[`a`]               	                    
 Sort Columns        	[`b`]               	                    
-Comment             	table_comment       	                    
+Table Properties    	[e=3]               	                    
 Location [not included in comparison]sql/core/spark-warehouse/t	                    
+Storage Properties  	[a=1, b=2]          	                    
 Partition Provider  	Catalog
 
 
--- !query 10
+-- !query 11
 DESC t PARTITION (c='Us', d=1)
--- !query 10 schema
+-- !query 11 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 10 output
+-- !query 11 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -167,11 +178,11 @@ c                   	string
 d                   	string
 
 
--- !query 11
+-- !query 12
 DESC EXTENDED t PARTITION (c='Us', d=1)
--- !query 11 schema
+-- !query 12 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 11 output
+-- !query 12 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -186,19 +197,21 @@ Database            	default
 Table               	t                   	                    
 Partition Values    	[c=Us, d=1]         	                    
 Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1	               
    
+Storage Properties  	[a=1, b=2]          	                    
                     	                    	                    
 # Storage Information	                    	                    
 Num Buckets         	2                   	                    
 Bucket Columns      	[`a`]               	                    
 Sort Columns        	[`b`]               	                    
-Location [not included in comparison]sql/core/spark-warehouse/t
+Location [not included in comparison]sql/core/spark-warehouse/t	                    
+Storage Properties  	[a=1, b=2]
 
 
--- !query 12
+-- !query 13
 DESC FORMATTED t PARTITION (c='Us', d=1)
--- !query 12 schema
+-- !query 13 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 12 output
+-- !query 13 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -213,39 +226,41 @@ Database            	default
 Table               	t                   	                    
 Partition Values    	[c=Us, d=1]         	                    
 Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1	               
    
+Storage Properties  	[a=1, b=2]          	                    
                     	                    	                    
 # Storage Information	                    	                    
 Num Buckets         	2                   	                    
 Bucket Columns      	[`a`]               	                    
 Sort Columns        	[`b`]               	                    
-Location [not included in comparison]sql/core/spark-warehouse/t
+Location [not included in comparison]sql/core/spark-warehouse/t	                    
+Storage Properties  	[a=1, b=2]
 
 
--- !query 13
+-- !query 14
 DESC t PARTITION (c='Us', d=2)
--- !query 13 schema
+-- !query 14 schema
 struct<>
--- !query 13 output
+-- !query 14 output
 org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException
 Partition not found in table 't' database 'default':
 c -> Us
 d -> 2;
 
 
--- !query 14
+-- !query 15
 DESC t PARTITION (c='Us')
--- !query 14 schema
+-- !query 15 schema
 struct<>
--- !query 14 output
+-- !query 15 output
 org.apache.spark.sql.AnalysisException
 Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table
'`default`.`t`';
 
 
--- !query 15
+-- !query 16
 DESC t PARTITION (c='Us', d)
--- !query 15 schema
+-- !query 16 schema
 struct<>
--- !query 15 output
+-- !query 16 output
 org.apache.spark.sql.catalyst.parser.ParseException
 
 PARTITION specification is incomplete: `d`(line 1, pos 0)
@@ -255,19 +270,8 @@ DESC t PARTITION (c='Us', d)
 ^^^
 
 
--- !query 16
-DESC temp_v
--- !query 16 schema
-struct<col_name:string,data_type:string,comment:string>
--- !query 16 output
-a                   	string              	                    
-b                   	int                 	                    
-c                   	string              	                    
-d                   	string
-
-
 -- !query 17
-DESC TABLE temp_v
+DESC temp_v
 -- !query 17 schema
 struct<col_name:string,data_type:string,comment:string>
 -- !query 17 output
@@ -278,7 +282,7 @@ d                   	string
 
 
 -- !query 18
-DESC FORMATTED temp_v
+DESC TABLE temp_v
 -- !query 18 schema
 struct<col_name:string,data_type:string,comment:string>
 -- !query 18 output
@@ -289,7 +293,7 @@ d                   	string
 
 
 -- !query 19
-DESC EXTENDED temp_v
+DESC FORMATTED temp_v
 -- !query 19 schema
 struct<col_name:string,data_type:string,comment:string>
 -- !query 19 output
@@ -300,10 +304,21 @@ d                   	string
 
 
 -- !query 20
-DESC temp_Data_Source_View
+DESC EXTENDED temp_v
 -- !query 20 schema
 struct<col_name:string,data_type:string,comment:string>
 -- !query 20 output
+a                   	string              	                    
+b                   	int                 	                    
+c                   	string              	                    
+d                   	string
+
+
+-- !query 21
+DESC temp_Data_Source_View
+-- !query 21 schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query 21 output
 intType             	int                 	test comment test1  
 stringType          	string              	                    
 dateType            	date                	                    
@@ -322,42 +337,42 @@ arrayType           	array<string>
 structType          	struct<f1:string,f2:int>
 
 
--- !query 21
+-- !query 22
 DESC temp_v PARTITION (c='Us', d=1)
--- !query 21 schema
+-- !query 22 schema
 struct<>
--- !query 21 output
+-- !query 22 output
 org.apache.spark.sql.AnalysisException
 DESC PARTITION is not allowed on a temporary view: temp_v;
 
 
--- !query 22
+-- !query 23
 DESC v
--- !query 22 schema
+-- !query 23 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 22 output
+-- !query 23 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
 d                   	string
 
 
--- !query 23
+-- !query 24
 DESC TABLE v
--- !query 23 schema
+-- !query 24 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 23 output
+-- !query 24 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
 d                   	string
 
 
--- !query 24
+-- !query 25
 DESC FORMATTED v
--- !query 24 schema
+-- !query 25 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 24 output
+-- !query 25 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -372,14 +387,14 @@ Type                	VIEW
 View Text           	SELECT * FROM t     	                    
 View Default Database	default             	                    
 View Query Output Columns	[a, b, c, d]        	                    
-Properties          	[view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4,
view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
+Table Properties    	[view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4,
view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
 
 
--- !query 25
+-- !query 26
 DESC EXTENDED v
--- !query 25 schema
+-- !query 26 schema
 struct<col_name:string,data_type:string,comment:string>
--- !query 25 output
+-- !query 26 output
 a                   	string              	                    
 b                   	int                 	                    
 c                   	string              	                    
@@ -394,28 +409,20 @@ Type                	VIEW
 View Text           	SELECT * FROM t     	                    
 View Default Database	default             	                    
 View Query Output Columns	[a, b, c, d]        	                    
-Properties          	[view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4,
view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
-
-
--- !query 26
-DESC v PARTITION (c='Us', d=1)
--- !query 26 schema
-struct<>
--- !query 26 output
-org.apache.spark.sql.AnalysisException
-DESC PARTITION is not allowed on a view: v;
+Table Properties    	[view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4,
view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
 
 
 -- !query 27
-DROP TABLE t
+DESC v PARTITION (c='Us', d=1)
 -- !query 27 schema
 struct<>
 -- !query 27 output
-
+org.apache.spark.sql.AnalysisException
+DESC PARTITION is not allowed on a view: v;
 
 
 -- !query 28
-DROP VIEW temp_v
+DROP TABLE t
 -- !query 28 schema
 struct<>
 -- !query 28 output
@@ -423,7 +430,7 @@ struct<>
 
 
 -- !query 29
-DROP VIEW temp_Data_Source_View
+DROP VIEW temp_v
 -- !query 29 schema
 struct<>
 -- !query 29 output
@@ -431,8 +438,16 @@ struct<>
 
 
 -- !query 30
-DROP VIEW v
+DROP VIEW temp_Data_Source_View
 -- !query 30 schema
 struct<>
 -- !query 30 output
 
+
+
+-- !query 31
+DROP VIEW v
+-- !query 31 schema
+struct<>
+-- !query 31 output
+


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message