carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [1/2] incubator-carbondata git commit: support smallint
Date Mon, 05 Dec 2016 09:16:17 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 8a6733041 -> e642ceb01


support smallint

fix DataTypeUtilTest issue


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e5ee02c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e5ee02c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e5ee02c8

Branch: refs/heads/master
Commit: e5ee02c863229317f58240acafa044a451ccba60
Parents: 8a67330
Author: cenyuhai <261810726@qq.com>
Authored: Sun Dec 4 18:17:44 2016 +0800
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Mon Dec 5 14:44:23 2016 +0530

----------------------------------------------------------------------
 .../core/carbon/metadata/datatype/DataType.java |  2 +-
 .../carbondata/core/util/DataTypeUtil.java      |  4 ++-
 .../impl/AbstractScannedResultCollector.java    |  1 +
 .../carbondata/core/util/DataTypeUtilTest.java  |  2 +-
 .../spark/util/DataTypeConverterUtil.scala      |  2 +-
 .../org/apache/spark/sql/CarbonSqlParser.scala  | 14 ++++++---
 .../dataload/TestLoadDataWithHiveSyntax.scala   | 33 ++++++++++++++++++++
 7 files changed, 50 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5ee02c8/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java
b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java
index c69dfa1..88127ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java
@@ -25,7 +25,7 @@ public enum DataType {
   DATE(1, "DATE"),
   TIMESTAMP(2, "TIMESTAMP"),
   BOOLEAN(1, "BOOLEAN"),
-  SHORT(2, "SHORT"),
+  SHORT(2, "SMALLINT"),
   INT(3, "INT"),
   FLOAT(4, "FLOAT"),
   LONG(5, "BIGINT"),

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5ee02c8/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index ae8775f..94a62b8 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -78,6 +78,7 @@ public final class DataTypeUtil {
         BigDecimal bigDecimal =
             new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
         return normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision());
+      case SHORT:
       case INT:
         return Double.valueOf(msrValue).longValue();
       case LONG:
@@ -123,6 +124,7 @@ public final class DataTypeUtil {
     switch (dataType) {
       case DECIMAL:
         return CarbonCommonConstants.BIG_DECIMAL_MEASURE;
+      case SHORT:
       case INT:
       case LONG:
         return CarbonCommonConstants.BIG_INT_MEASURE;
@@ -180,7 +182,7 @@ public final class DataTypeUtil {
       case "INT":
         dataType = DataType.INT;
         break;
-      case "SHORT":
+      case "SMALLINT":
         dataType = DataType.SHORT;
         break;
       case "LONG":

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5ee02c8/core/src/main/java/org/apache/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
b/core/src/main/java/org/apache/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
index 387b54f..90d4bd7 100644
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
@@ -102,6 +102,7 @@ public abstract class AbstractScannedResultCollector implements ScannedResultCol
     if (!dataChunk.getNullValueIndexHolder().getBitSet().get(index)) {
       Object msrVal;
       switch (dataType) {
+        case SHORT:
         case INT:
         case LONG:
           msrVal = dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5ee02c8/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
index 9aa0dfd..dd0662b 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
@@ -79,7 +79,7 @@ public class DataTypeUtilTest {
     assertEquals(DataType.TIMESTAMP, getDataType("TIMESTAMP"));
     assertEquals(DataType.STRING, getDataType("STRING"));
     assertEquals(DataType.INT, getDataType("INT"));
-    assertEquals(DataType.SHORT, getDataType("SHORT"));
+    assertEquals(DataType.SHORT, getDataType("SMALLINT"));
     assertEquals(DataType.LONG, getDataType("LONG"));
     assertEquals(DataType.DOUBLE, getDataType("DOUBLE"));
     assertEquals(DataType.DECIMAL, getDataType("DECIMAL"));

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5ee02c8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 5ec96df..5e656d9 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -26,7 +26,7 @@ object DataTypeConverterUtil {
       case "int" => DataType.INT
       case "integer" => DataType.INT
       case "tinyint" => DataType.SHORT
-      case "short" => DataType.SHORT
+      case "smallint" => DataType.SHORT
       case "long" => DataType.LONG
       case "bigint" => DataType.LONG
       case "numeric" => DataType.DOUBLE

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5ee02c8/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index aa9f95a..b7f8b90 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -147,6 +147,7 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
   protected val NUMERIC = carbonKeyWord("NUMERIC")
   protected val DECIMAL = carbonKeyWord("DECIMAL")
   protected val DOUBLE = carbonKeyWord("DOUBLE")
+  protected val SHORT = carbonKeyWord("SMALLINT")
   protected val INT = carbonKeyWord("INT")
   protected val BIGINT = carbonKeyWord("BIGINT")
   protected val ARRAY = carbonKeyWord("ARRAY")
@@ -1086,9 +1087,9 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
   protected lazy val dimCol: Parser[Field] = anyFieldDef
 
   protected lazy val primitiveTypes =
-    STRING ^^^ "string" | INTEGER ^^^ "integer" | TIMESTAMP ^^^
-                                                  "timestamp" | NUMERIC ^^^ "numeric" |
-    BIGINT ^^^ "bigint" |
+    STRING ^^^ "string" | INTEGER ^^^ "integer" |
+    TIMESTAMP ^^^ "timestamp" | NUMERIC ^^^ "numeric" |
+    BIGINT ^^^ "bigint" | SHORT ^^^ "smallint" |
     INT ^^^ "int" | DOUBLE ^^^ "double" | decimalType
 
   /**
@@ -1130,7 +1131,7 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
     }
 
   protected lazy val measureCol: Parser[Field] =
-    (ident | stringLit) ~ (INTEGER ^^^ "integer" | NUMERIC ^^^ "numeric" |
+    (ident | stringLit) ~ (INTEGER ^^^ "integer" | NUMERIC ^^^ "numeric" | SHORT ^^^ "smallint"
|
                            BIGINT ^^^ "bigint" | DECIMAL ^^^ "decimal").? ~
     (AS ~> (ident | stringLit)).? ~ (IN ~> (ident | stringLit)).? ^^ {
       case e1 ~ e2 ~ e3 ~ e4 => Field(e1, e2, e3, Some(null))
@@ -1159,6 +1160,9 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
       case "string" => Field(field.column, Some("String"), field.name, Some(null), field.parent,
         field.storeType, field.schemaOrdinal
       )
+      case "smallint"  => Field(field.column, Some("SmallInt"), field.name, Some(null),
+        field.parent, field.storeType, field.schemaOrdinal
+      )
       case "integer" | "int" => Field(field.column, Some("Integer"), field.name, Some(null),
         field.parent, field.storeType, field.schemaOrdinal
       )
@@ -1222,6 +1226,8 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
     field.dataType.getOrElse("NIL") match {
       case "String" => Field(parentName + "." + field.column, Some("String"),
         Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName)
+      case "SmallInt" => Field(parentName + "." + field.column, Some("SmallInt"),
+        Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName)
       case "Integer" => Field(parentName + "." + field.column, Some("Integer"),
         Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName)
       case "Long" => Field(parentName + "." + field.column, Some("Long"),

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5ee02c8/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
index d149b9d..ad695f4 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
@@ -61,6 +61,8 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll
{
     sql("drop table if exists carbontable1")
     sql("drop table if exists hivetable1")
     sql("drop table if exists comment_test")
+    sql("drop table if exists smallinttable")
+    sql("drop table if exists smallinthivetable")
     sql(
       "CREATE table carbontable (empno int, empname String, designation String, doj String,
" +
         "workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
" +
@@ -76,6 +78,37 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll
{
 
   }
 
+  test("create table with smallint type and query smallint table") {
+    sql(
+      "create table smallinttable(empno smallint, empname String, designation string, " +
+        "doj String, workgroupcategory int, workgroupcategoryname String,deptno int, " +
+        "deptname String, projectcode int, projectjoindate String,projectenddate String,
" +
+        "attendance String, utilization String,salary String)" +
+        "STORED BY 'org.apache.carbondata.format'"
+    )
+
+    sql(
+      "create table smallinthivetable(empno smallint, empname String, designation string,
" +
+        "doj String, workgroupcategory int, workgroupcategoryname String,deptno int, " +
+        "deptname String, projectcode int, projectjoindate String,projectenddate String,
" +
+        "attendance String, utilization String,salary String)" +
+        "row format delimited fields terminated by ','"
+    )
+
+    sql("LOAD DATA local inpath './src/test/resources/data.csv' INTO table smallinttable
" +
+      "OPTIONS('USE_KETTLE'='false')")
+    sql("LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' overwrite "
+
+      "INTO table smallinthivetable")
+
+    checkAnswer(
+      sql("select empno from smallinttable"),
+      sql("select empno from smallinthivetable")
+    )
+
+    sql("drop table if exists smallinttable")
+    sql("drop table if exists smallinthivetable")
+  }
+
   test("test data loading and validate query output") {
     //Create test cube and hive table
     sql(


Mime
View raw message