carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From manishgupt...@apache.org
Subject carbondata git commit: [CARBONDATA-1826] Spark 2.2: Describe table & Describe Formatted shows the same result
Date Wed, 13 Dec 2017 04:23:10 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 34cb55194 -> b6f29dec4


[CARBONDATA-1826] Spark 2.2: Describe table & Describe Formatted shows the same result

Carbon 1.3.0 - Spark 2.2: Describe table & Describe Formatted shows the same result.

This closes #1630


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b6f29dec
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b6f29dec
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b6f29dec

Branch: refs/heads/master
Commit: b6f29dec4335d536fe371ba985fe2c9b98cb2772
Parents: 34cb551
Author: dhatchayani <dhatcha.official@gmail.com>
Authored: Thu Dec 7 12:50:05 2017 +0530
Committer: manishgupta88 <tomanishgupta18@gmail.com>
Committed: Wed Dec 13 09:55:49 2017 +0530

----------------------------------------------------------------------
 .../describeTable/TestDescribeTable.scala       |  8 +++--
 .../spark/util/CarbonReflectionUtils.scala      |  8 +++++
 .../sql/execution/strategy/DDLStrategy.scala    | 33 +++++++++++++-------
 3 files changed, 35 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6f29dec/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
index b2a1120..a87292f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
@@ -16,6 +16,7 @@
  */
 package org.apache.carbondata.spark.testsuite.describeTable
 
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.BeforeAndAfterAll
 
@@ -34,8 +35,11 @@ class TestDescribeTable extends QueryTest with BeforeAndAfterAll {
     sql("CREATE TABLE Desc2(Dec2Col1 BigInt, Dec2Col2 String, Dec2Col3 Bigint, Dec2Col4 Decimal)
stored by 'carbondata'")
   }
 
-  ignore("test describe table") {
-    checkAnswer(sql("DESC Desc1"), sql("DESC Desc2"))
+  test("test describe table") {
+    checkAnswer(sql("DESC Desc1"), Seq(Row("dec2col1","bigint",null),
+      Row("dec2col2","string",null),
+      Row("dec2col3","bigint",null),
+      Row("dec2col4","decimal(10,0)",null)))
   }
 
   test("test describe formatted table") {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6f29dec/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
index d88f190..6cc7ae1 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
@@ -175,6 +175,14 @@ object CarbonReflectionUtils {
     }
   }
 
+  def getDescribeTableFormattedField[T: TypeTag : reflect.ClassTag](obj: T): Boolean = {
+    val im = rm.reflect(obj)
+    val isFormatted = im.symbol.typeSignature.members
+      .find(_.name.toString.equalsIgnoreCase("isFormatted"))
+      .map(l => im.reflectField(l.asTerm).get).getOrElse("false").asInstanceOf[Boolean]
+    isFormatted
+  }
+
   def createObject(className: String, conArgs: Object*): (Any, Class[_]) = {
     val clazz = Utils.classForName(className)
     val ctor = clazz.getConstructors.head

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6f29dec/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 9ae1979..8745900 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.execution.command.table.{CarbonDescribeFormattedComm
 import org.apache.spark.sql.hive.execution.command.{CarbonDropDatabaseCommand, CarbonResetCommand,
CarbonSetCommand}
 import org.apache.spark.sql.CarbonExpressions.{CarbonDescribeTable => DescribeTableCommand}
 import org.apache.spark.sql.execution.datasources.RefreshTable
-import org.apache.spark.util.FileUtils
+import org.apache.spark.util.{CarbonReflectionUtils, FileUtils}
 
 import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
 import org.apache.carbondata.core.util.CarbonProperties
@@ -134,17 +134,26 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy
{
         } else {
           throw new MalformedCarbonCommandException("Unsupported alter operation on hive
table")
         }
-      case desc@DescribeTableCommand(identifier, partitionSpec, isExtended)
-        if CarbonEnv.getInstance(sparkSession).carbonMetastore
-          .tableExists(identifier)(sparkSession) =>
-        val resolvedTable =
-          sparkSession.sessionState.executePlan(UnresolvedRelation(identifier)).analyzed
-        val resultPlan = sparkSession.sessionState.executePlan(resolvedTable).executedPlan
-        ExecutedCommandExec(
-          CarbonDescribeFormattedCommand(
-            resultPlan,
-            plan.output,
-            identifier)) :: Nil
+      case desc@DescribeTableCommand(identifier, partitionSpec, isExtended) =>
+        val isFormatted: Boolean = if (sparkSession.version.startsWith("2.1")) {
+          CarbonReflectionUtils
+            .getDescribeTableFormattedField(desc.asInstanceOf[DescribeTableCommand])
+        } else {
+          false
+        }
+        if (CarbonEnv.getInstance(sparkSession).carbonMetastore
+              .tableExists(identifier)(sparkSession) && (isExtended || isFormatted))
{
+          val resolvedTable =
+            sparkSession.sessionState.executePlan(UnresolvedRelation(identifier)).analyzed
+          val resultPlan = sparkSession.sessionState.executePlan(resolvedTable).executedPlan
+          ExecutedCommandExec(
+            CarbonDescribeFormattedCommand(
+              resultPlan,
+              plan.output,
+              identifier)) :: Nil
+        } else {
+          Nil
+        }
       case ShowPartitionsCommand(t, cols) =>
         val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
           .tableExists(t)(sparkSession)


Mime
View raw message