carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [41/47] incubator-carbondata git commit: Problem: Exception thrown while executing drop table command in spark-sql cli (#888)
Date Mon, 01 Aug 2016 10:05:39 GMT
Problem: Exception thrown while executing drop table command in spark-sql cli (#888)

Analysis: Whenever drop table operation is performed, first the cube is deleted from carbon
metastore and then command is executed to delete the same table from hive metastore. While
deleting from hive metastore the call comes back to carbon metastore catalog where we again
check for table existence but table is not found and exception is thrown. Call comes again
to carbon because the command is executed as spark sql command and not as hive sql command
and we catch the logical plan when drop command is executed as spark sql command.
Impact: drop table operation

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b82a960b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b82a960b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b82a960b

Branch: refs/heads/master
Commit: b82a960bd58acd83d1f6ecf79340128170178414
Parents: 87f0dec
Author: manishgupta88 <tomanishgupta18@gmail.com>
Authored: Fri Jul 29 15:31:33 2016 +0530
Committer: sujith71955 <sujithchacko.2010@gmail.com>
Committed: Fri Jul 29 18:01:33 2016 +0800

----------------------------------------------------------------------
 .../spark/sql/hive/CarbonMetastoreCatalog.scala | 12 +---------
 .../spark/rdd/CarbonGlobalDictionaryRDD.scala   | 14 ++++++------
 .../spark/util/GlobalDictionaryUtil.scala       |  6 ++---
 .../resources/complexTypeDecimalNestedHive.csv  |  8 +++++++
 .../test/resources/emptyDimensionDataHive.csv   | 20 +++++++++++++++++
 .../dataload/TestLoadDataWithNullMeasures.scala |  1 +
 .../TestDataWithDicExcludeAndInclude.scala      | 22 ++++++++++++++-----
 .../dataload/TestLoadDataWithHiveSyntax.scala   | 23 +++++++++++++++-----
 .../NoDictionaryColumnTestCase.scala            |  6 +++--
 .../NullMeasureValueTestCaseFilter.scala        |  7 +++---
 10 files changed, 82 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
index 7ec1b87..e18e1d3 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
@@ -495,17 +495,7 @@ class CarbonMetastoreCatalog(hive: HiveContext, val storePath: String,
client: C
     org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
       .removeTable(schemaName + "_" + cubeName)
 
-    try {
-      sqlContext.sql(s"DROP TABLE $schemaName.$cubeName").collect()
-    } catch {
-      case e: Exception =>
-        LOGGER.audit(
-          s"Error While deleting the table $schemaName.$cubeName during drop Table" + e.getMessage)
-    }
-
-    logInfo(s"Table $cubeName of $schemaName Database dropped syccessfully.")
-    LOGGER.info("Table " + cubeName + " of " + schemaName + " Database dropped syccessfully.")
-
+    sqlContext.asInstanceOf[HiveContext].runSqlHive(s"DROP TABLE IF EXISTS $schemaName.$cubeName")
   }
 
   private def getTimestampFileAndType(schemaName: String, cubeName: String) = {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index a1870e9..70d0cb1 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -156,7 +156,7 @@ case class DictionaryLoadModel(table: CarbonTableIdentifier,
     rowCountPercentage: Double,
     columnIdentifier: Array[ColumnIdentifier],
     isFirstLoad: Boolean,
-    hdfstemplocation: String,
+    hdfsTempLocation: String,
     lockType: String,
     zooKeeperUrl: String) extends Serializable
 
@@ -301,11 +301,11 @@ class CarbonGlobalDictionaryGenerateRDD(
       val carbonTablePath = pathService.getCarbonTablePath(model.columnIdentifier(split.index),
           model.hdfsLocation, model.table)
       CarbonProperties.getInstance.addProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION,
-              model.hdfstemplocation)
+        model.hdfsTempLocation)
       CarbonProperties.getInstance.addProperty(CarbonCommonConstants.LOCK_TYPE,
-              model.lockType)
-       CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL,
-              model.zooKeeperUrl)
+        model.lockType)
+      CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL,
+        model.zooKeeperUrl)
       val dictLock = CarbonLockFactory
         .getCarbonLockObj(carbonTablePath.getRelativeDictionaryDirectory,
           model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
@@ -408,8 +408,8 @@ class CarbonGlobalDictionaryGenerateRDD(
             .clearDictionaryCache(dictionaryForDistinctValueLookUp);
         }
         org.carbondata.core.util.CarbonUtil.clearDictionaryCache(dictionaryForSortIndexWriting);
-        if (dictLock != null) {
-          if (isDictionaryLocked && dictLock.unlock()) {
+        if (dictLock != null && isDictionaryLocked) {
+          if (dictLock.unlock()) {
             logInfo(s"Dictionary ${
               model.primDimensions(split.index).getColName
             } Unlocked Successfully.")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
index 0e15803..1c45230 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -297,8 +297,8 @@ object GlobalDictionaryUtil extends Logging {
     val dictFilePaths = dictDetail.dictFilePaths
     val dictFileExists = dictDetail.dictFileExists
     val columnIdentifier = dictDetail.columnIdentifiers
-    val hdfstemplocation = CarbonProperties.getInstance.getProperty(
-        CarbonCommonConstants.HDFS_TEMP_LOCATION, System.getProperty("java.io.tmpdir"))
+    val hdfsTempLocation = CarbonProperties.getInstance.
+      getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION, System.getProperty("java.io.tmpdir"))
     val lockType = CarbonProperties.getInstance
       .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
     val zookeeperUrl = CarbonProperties.getInstance.getProperty(CarbonCommonConstants.ZOOKEEPER_URL)
@@ -331,7 +331,7 @@ object GlobalDictionaryUtil extends Logging {
       rowCountPercentage,
       columnIdentifier,
       carbonLoadModel.getLoadMetadataDetails.size() == 0,
-      hdfstemplocation,
+      hdfsTempLocation,
       lockType,
       zookeeperUrl)
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv b/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv
new file mode 100644
index 0000000..ae67723
--- /dev/null
+++ b/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv
@@ -0,0 +1,8 @@
+1.2,2015/7/23,china,aaa1,phone197,ASD69643,15000,3.113:imei$3.33:imsi
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001,3.123:imei$7.33:imsi
+4.3,2015/7/26,china,aaa4,phone2435,ASD66902,15003,3.123:imei$56.33:imsi
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,3.133:imei$5.33:imsi
+6.5,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.133:imei$54.33:imsi
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,32.13:imei$56.33:imsi
+9.1,2015/7/18,china,aaa9,phone706,ASD86717,15008,3.213:imei$44.33:imsi
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,32.13:imei$33.33:imsi
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/resources/emptyDimensionDataHive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/emptyDimensionDataHive.csv b/integration/spark/src/test/resources/emptyDimensionDataHive.csv
new file mode 100644
index 0000000..2e89887
--- /dev/null
+++ b/integration/spark/src/test/resources/emptyDimensionDataHive.csv
@@ -0,0 +1,20 @@
+1,2015/7/23,,aaa1,,,15000
+2,2015/7/24,,,,,15001
+3,2015/7/25,china,aaa3,phone1904,ASD37014,15002
+4,2015/7/26,china,aaa4,,ASD66902,15003
+5,2015/7/27,china,aaa5,phone2441,,15004
+6,2015/7/28,,aaa6,phone294,ASD59961,15005
+7,2015/7/29,china,aaa7,phone610,ASD14875,15006
+8,2015/7/30,china,aaa8,,ASD57308,15007
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009
+11,2015/7/18,china,aaa11,phone1554,ASD26101,15010
+12,2015/7/19,,aaa12,phone1781,ASD85711,15011
+13,2015/7/20,china,,phone943,,15012
+14,2015/7/21,china,aaa14,phone1954,ASD80468,15013
+15,2015/7/22,china,aaa15,phone451,ASD1954,15014
+16,2015/7/23,china,aaa16,phone390,ASD38513,15015
+17,2015/7/24,,aaa17,phone1929,ASD86213,15016
+18,2015/7/25,usa,aaa18,,ASD88812,15017
+19,2015/7/26,china,aaa19,phone2151,ASD9316,15018
+20,2015/7/27,china,aaa20,phone2625,ASD62597,15019
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
index 09f44ad..3a920b8 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
@@ -29,6 +29,7 @@ import org.scalatest.BeforeAndAfterAll
  */
 class TestLoadDataWithNullMeasures extends QueryTest with BeforeAndAfterAll {
   override def beforeAll {
+    sql("DROP TABLE IF EXISTS carbontable")
     sql(
       "CREATE TABLE carbontable (empno int, empname String, designation String, doj String,
" +
       "workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
" +

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
index 7ab1a33..0e333dd 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
@@ -40,14 +40,22 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with
BeforeA
   }
 
   def dropTable() = {
-    sql("DROP TABLE IF EXISTS t3")
+    sql("DROP TABLE IF EXISTS exclude_include_t3")
+    sql("DROP TABLE IF EXISTS exclude_include_hive_t3")
   }
 
   def buildTable() = {
     try {
       sql(
         """
-           CREATE TABLE IF NOT EXISTS t3
+           CREATE TABLE exclude_include_hive_t3
+           (ID Int, date Timestamp, country String,
+           name String, phonetype String, serialname String, salary Int)
+           row format delimited fields terminated by ','
+        """)
+      sql(
+        """
+           CREATE TABLE exclude_include_t3
            (ID Int, date Timestamp, country String,
            name String, phonetype String, serialname String, salary Int)
            STORED BY 'org.apache.carbondata.format'
@@ -65,7 +73,11 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with
BeforeA
         .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
       sql(
         s"""
-           LOAD DATA LOCAL INPATH '$filePath' into table t3
+           LOAD DATA LOCAL INPATH '$filePath' into table exclude_include_t3
+           """)
+      sql(
+        s"""
+           LOAD DATA LOCAL INPATH './src/test/resources/emptyDimensionDataHive.csv' into
table exclude_include_hive_t3
            """)
     } catch {
       case ex: Throwable => logError(ex.getMessage + "\r\n" + ex.getStackTraceString)
@@ -81,9 +93,7 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with
BeforeA
 
   test("test load data with dictionary exclude & include and with empty dimension") {
     checkAnswer(
-      sql("select ID from t3"), Seq(Row(1), Row(2), Row(3), Row(4), Row(5), Row(6), Row(7),
-        Row(8), Row(9), Row(10), Row(11), Row(12), Row(13), Row(14), Row(15), Row(16), Row
-        (17), Row(18), Row(19), Row(20))
+      sql("select ID from exclude_include_t3"), sql("select ID from exclude_include_hive_t3")
     )
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
index 173fe6b..731ed89 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
@@ -529,25 +529,38 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll
{
   }
 
   test("test data which contain column with decimal data type in array of struct."){
-    sql("DROP TABLE IF EXISTS t3")
+    sql("DROP TABLE IF EXISTS complex_t3")
+    sql("DROP TABLE IF EXISTS complex_hive_t3")
 
     sql(
       """
-           CREATE TABLE IF NOT EXISTS t3
+           CREATE TABLE complex_t3
            (ID decimal, date Timestamp, country String,
            name String, phonetype String, serialname String, salary Int, complex
            array<struct<a:decimal(4,2),str:string>>)
            STORED BY 'org.apache.carbondata.format'
       """
     )
+    sql(
+      """
+           CREATE TABLE complex_hive_t3
+           (ID decimal, date Timestamp, country String,
+           name String, phonetype String, serialname String, salary Int, complex
+           array<struct<a:decimal(4,2),str:string>>)
+           row format delimited fields terminated by ','
+      """
+    )
 
     CarbonProperties.getInstance()
       .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
     sql(s"""
-         LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNested.csv' into
table t3
+         LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNested.csv' into
table complex_t3
         """)
-    checkAnswer(sql("select count(*) from t3"),Seq(Row(8)))
-    checkAnswer(sql("select id from t3 where salary = 15000"),Seq(Row(1)))
+    sql(s"""
+         LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNestedHive.csv' into
table complex_hive_t3
+        """)
+    checkAnswer(sql("select count(*) from complex_t3"),sql("select count(*) from complex_hive_t3"))
+    checkAnswer(sql("select id from complex_t3 where salary = 15000"),sql("select id from
complex_hive_t3 where salary = 15000"))
   }
 
   test("test data loading when delimiter is '|' and data with header") {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
index 7ec5e9a..9e9fa90 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
@@ -31,6 +31,8 @@ import org.scalatest.BeforeAndAfterAll
 class NoDictionaryColumnTestCase extends QueryTest with BeforeAndAfterAll {
 
   override def beforeAll {
+    sql("DROP TABLE IF EXISTS carbonTable")
+    sql("DROP TABLE IF EXISTS hiveTable")
     sql("CREATE TABLE carbonTable (imei String, age Int, num BigInt) STORED BY 'org.apache.carbondata.format'
TBLPROPERTIES('DICTIONARY_EXCLUDE'='age,num')")
     sql("LOAD DATA LOCAL INPATH './src/test/resources/datawithNegtiveNumber.csv' INTO TABLE
carbonTable")
     sql("CREATE TABLE hiveTable (imei String, age Int, num BigInt) ROW FORMAT DELIMITED FIELDS
TERMINATED BY ','")
@@ -52,7 +54,7 @@ class NoDictionaryColumnTestCase extends QueryTest with BeforeAndAfterAll
{
   }
 
   override def afterAll {
-    sql("DROP TABLE carbonTable")
-    sql("DROP TABLE hiveTable")
+    sql("DROP TABLE IF EXISTS carbonTable")
+    sql("DROP TABLE IF EXISTS hiveTable")
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
index 2449f64..d81974d 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
@@ -10,8 +10,9 @@ import org.scalatest.BeforeAndAfterAll
 class NullMeasureValueTestCaseFilter extends QueryTest with BeforeAndAfterAll {
 
   override def beforeAll {
+    sql("drop table if exists t3")
     sql(
-      "CREATE TABLE IF NOT EXISTS t3 (ID Int, date Timestamp, country String, name String,
" +
+      "CREATE TABLE t3 (ID bigInt, date Timestamp, country String, name String, " +
         "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata.format'"
     )
     CarbonProperties.getInstance()
@@ -22,13 +23,13 @@ class NullMeasureValueTestCaseFilter extends QueryTest with BeforeAndAfterAll
{
   test("select ID from t3 where salary is not null") {
     checkAnswer(
       sql("select ID from t3 where salary is not null"),
-      Seq(Row(1.0),Row(4.0)))
+      Seq(Row(1),Row(4)))
   }
 
   test("select ID from t3 where salary is null") {
     checkAnswer(
       sql("select ID from t3 where salary is null"),
-      Seq(Row(2.0),Row(3.0)))
+      Seq(Row(2),Row(3)))
   }
 
   override def afterAll {


Mime
View raw message