carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/3] incubator-carbondata git commit: support datatype: date and char
Date Thu, 15 Dec 2016 09:07:48 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master bbb5919a6 -> ecf29472e


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index 4f8bf1a..f166025 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -50,16 +50,16 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
           TimeStampGranularityConstants.TIME_GRAN_SEC.toString
         )
       CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "true")
+      sql("drop table if exists directDictionaryTable")
+      sql("drop table if exists directDictionaryTable_hive")
       sql(
-        "CREATE TABLE if not exists directDictionaryTable (empno int,doj Timestamp, " +
-          "salary int) " +
+        "CREATE TABLE if not exists directDictionaryTable (empno int,doj Timestamp, salary
int) " +
           "STORED BY 'org.apache.carbondata.format'"
       )
 
       sql(
-        "CREATE TABLE if not exists directDictionaryTable_hive (empno int,doj Timestamp,
" +
-        "salary int) " +
-        "row format delimited fields terminated by ','"
+        "CREATE TABLE if not exists directDictionaryTable_hive (empno int,doj Timestamp,
salary int) " +
+          "row format delimited fields terminated by ','"
       )
 
       CarbonProperties.getInstance()
@@ -68,8 +68,8 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
         .getCanonicalPath
       val csvFilePath = currentDirectory + "/src/test/resources/datasample.csv"
       sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable
OPTIONS" +
-        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')");
-      sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive");
+        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')")
+      sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive")
     } catch {
       case x: Throwable => CarbonProperties.getInstance()
         .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
@@ -97,8 +97,8 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
 
   test("test direct dictionary for not equals condition") {
     checkAnswer(
-      sql("select doj from directDictionaryTable where doj != '2016-04-14 15:00:09.0'"),
-      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0"))
+      sql("select doj from directDictionaryTable where doj != '2016-04-14 15:00:09'"),
+      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09"))
       )
     )
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
index 22678af..991b1bf 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
@@ -88,8 +88,8 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest
with
     )
 
   }
-  
-    test("select doj from directDictionaryTable with greater than filter") {
+
+  test("select doj from directDictionaryTable with greater than filter") {
     checkAnswer(
       sql("select doj from directDictionaryTable where doj>'2016-03-14 15:00:09'"),
       Seq(Row(Timestamp.valueOf("2016-04-14 15:00:09")))
@@ -99,7 +99,7 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest
with
 
 
   override def afterAll {
-     sql("drop table directDictionaryTable")
+    sql("drop table directDictionaryTable")
     CarbonProperties.getInstance()
       .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
     CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
index f1a9ccb..9b7359f 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
@@ -50,7 +50,7 @@ class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll
{
         )
       sql(
         """CREATE TABLE IF NOT EXISTS timestampTyeNullData
-                     (ID Int, date Timestamp, country String,
+                     (ID Int, dateField Timestamp, country String,
                      name String, phonetype String, serialname String, salary Int)
                     STORED BY 'org.apache.carbondata.format'"""
       )
@@ -68,18 +68,18 @@ class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll
{
     }
   }
 
-  test("SELECT max(date) FROM timestampTyeNullData where date is not null") {
+  test("SELECT max(dateField) FROM timestampTyeNullData where dateField is not null") {
     checkAnswer(
-      sql("SELECT max(date) FROM timestampTyeNullData where date is not null"),
+      sql("SELECT max(dateField) FROM timestampTyeNullData where dateField is not null"),
       Seq(Row(Timestamp.valueOf("2015-07-23 00:00:00.0"))
       )
     )
   }
-    test("SELECT * FROM timestampTyeNullData where date is null") {
-      checkAnswer(
-        sql("SELECT date FROM timestampTyeNullData where date is null"),
-        Seq(Row(null)
-        ))
+  test("SELECT * FROM timestampTyeNullData where dateField is null") {
+    checkAnswer(
+      sql("SELECT dateField FROM timestampTyeNullData where dateField is null"),
+      Seq(Row(null)
+      ))
   }
 
   override def afterAll {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index 65d583c..1bfcdea 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -39,11 +39,13 @@ public class SparkRowReadSupportImpl extends AbstractDictionaryDecodedReadSuppor
 
   @Override public Row readRow(Object[] data) {
     for (int i = 0; i < dictionaries.length; i++) {
-      if (dictionaries[i] == null) {
+      if (dictionaries[i] == null && data[i] != null) {
         if (carbonColumns[i].hasEncoding(Encoding.DIRECT_DICTIONARY)) {
           //convert the long to timestamp in case of direct dictionary column
           if (DataType.TIMESTAMP == carbonColumns[i].getDataType()) {
-            data[i] = new Timestamp((long) data[i] / 1000);
+            data[i] = new Timestamp((long) data[i]);
+          } else if(DataType.DATE == carbonColumns[i].getDataType()) {
+            data[i] = new java.sql.Date((int) data[i]);
           }
         } else if(dataTypes[i].equals(DataType.INT)) {
           data[i] = ((Long)(data[i])).intValue();

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 2468962..5db5d14 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -134,6 +134,7 @@ class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame)
{
       case DoubleType => CarbonType.DOUBLE.getName
       case BooleanType => CarbonType.DOUBLE.getName
       case TimestampType => CarbonType.TIMESTAMP.getName
+      case DateType => CarbonType.DATE.getName
       case other => sys.error(s"unsupported type: $other")
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index 940c6d7..3b63021 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -104,6 +104,7 @@ case class CarbonDictionaryDecoder(
           DecimalType(precision, scale)
         }
       case DataType.TIMESTAMP => TimestampType
+      case DataType.DATE => DateType
       case DataType.STRUCT =>
         CarbonMetastoreTypes
           .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName)
}>")
@@ -261,6 +262,7 @@ class CarbonDecoderRDD(
           DecimalType(precision, scale)
         }
       case DataType.TIMESTAMP => TimestampType
+      case DataType.DATE => DateType
       case DataType.STRUCT =>
         CarbonMetastoreTypes
             .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName)
}>")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index b14a95c..2ba8a03 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -102,12 +102,10 @@ class CarbonSource extends CreatableRelationProvider
 
   private def createTableIfNotExists(sparkSession: SparkSession, parameters: Map[String,
String],
                                      dataSchema: StructType): String = {
-    val (dbName, tableName) = parameters.get("path") match {
-      case Some(path) =>
-        val p = path.split(File.separator)
-        ("default", p(p.length - 1))
-      case _ => throw new Exception("do not have dbname and tablename for carbon table")
-    }
+
+    val dbName: String = parameters.getOrElse("dbName", CarbonCommonConstants.DATABASE_DEFAULT_NAME)
+    val tableName: String = parameters.getOrElse("tableName", "default_table")
+
     try {
       CarbonEnv.get.carbonMetastore.lookupRelation(Option(dbName), tableName)(sparkSession)
       CarbonEnv.get.carbonMetastore.storePath + s"/$dbName/$tableName"
@@ -134,7 +132,8 @@ class CarbonSource extends CreatableRelationProvider
         val cm = TableCreator.prepareTableModel(false, Option(dbName), tableName, fields,
Nil, map)
         CreateTable(cm).run(sparkSession)
         CarbonEnv.get.carbonMetastore.storePath + s"/$dbName/$tableName"
-      case _ => throw new Exception("do not have dbname and tablename for carbon table")
+      case ex: Exception =>
+        throw new Exception("do not have dbname and tablename for carbon table", ex)
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index e375710..1faaafa 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -39,7 +39,7 @@ object TableCreator {
 
   // detects whether double or decimal column is part of dictionary_exclude
   def isStringAndTimestampColDictionaryExclude(columnDataType: String): Boolean = {
-    val dataTypes = Array("string", "timestamp")
+    val dataTypes = Array("string", "timestamp", "date", "stringtype", "timestamptype", "datetype")
     dataTypes.exists(x => x.equalsIgnoreCase(columnDataType))
   }
 
@@ -47,7 +47,7 @@ object TableCreator {
   def isDetectAsDimentionDatatype(dimensionDatatype: String): Boolean = {
     val dimensionType =
       Array("string", "stringtype", "array", "arraytype", "struct",
-        "structtype", "timestamp", "timestamptype")
+        "structtype", "timestamp", "timestamptype", "date", "datetype")
     dimensionType.exists(x => x.equalsIgnoreCase(dimensionDatatype))
   }
 
@@ -111,7 +111,8 @@ object TableCreator {
     fields.foreach(field => {
 
       if (dictExcludeCols.toSeq.exists(x => x.equalsIgnoreCase(field.column))) {
-        if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.TIMESTAMP)
{
+        if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.TIMESTAMP
&&
+            DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.DATE)
{
           noDictionaryDims :+= field.column
         }
         dimFields += field

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 24601f4..9467804 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -609,6 +609,7 @@ object CarbonMetastoreTypes extends RegexParsers {
     fixedDecimalType |
     "decimal" ^^^ "decimal" ^^^ DecimalType(18, 2) |
     "varchar\\((\\d+)\\)".r ^^^ StringType |
+    "date" ^^^ DateType |
     "timestamp" ^^^ TimestampType
 
   protected lazy val fixedDecimalType: Parser[DataType] =
@@ -668,6 +669,7 @@ object CarbonMetastoreTypes extends RegexParsers {
       case BooleanType => "boolean"
       case DecimalType() => "decimal"
       case TimestampType => "timestamp"
+      case DateType => "date"
     }
   }
 }


Mime
View raw message