Mailing-List: contact commits-help@carbondata.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@carbondata.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: chenliang613@apache.org
To: commits@carbondata.incubator.apache.org
Date: Wed, 20 Jul 2016 10:13:48 -0000
Message-Id: <c40b034c8adb4babb97c4683584aea4a@git.apache.org>
In-Reply-To: <5e0daf87cb5643eba1c1b8e77c61f141@git.apache.org>
References: <5e0daf87cb5643eba1c1b8e77c61f141@git.apache.org>
Subject: [20/50] [abbrv] incubator-carbondata git commit: [Bug] Dataload is
 failing by flipping datatype of string column to array<string> column (#812)
archived-at: Wed, 20 Jul 2016 10:13:36 -0000

[Bug] Dataload is failing by flipping datatype of string column to array<string> column (#812)

* Data load is failing by flipping datatype of string column to array<string> column

* Fixed merge conflicts


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a0446801
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a0446801
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a0446801

Branch: refs/heads/master
Commit: a044680187a90140f009d7b52bcc61f24aabe9e7
Parents: 58d672b
Author: nareshpr <prnaresh.naresh@gmail.com>
Authored: Sat Jul 16 12:54:25 2016 +0530
Committer: Ravindra Pesala <ravi.pesala@gmail.com>
Committed: Sat Jul 16 12:54:25 2016 +0530

----------------------------------------------------------------------
 .../src/test/resources/complexdatareordered.csv    | 10 ++++++++++
 .../dataload/TestLoadDataWithHiveSyntax.scala      | 17 +++++++++++++++++
 .../csvbased/CarbonCSVBasedSeqGenStep.java         |  6 +++---
 3 files changed, 30 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0446801/integration/spark/src/test/resources/complexdatareordered.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/complexdatareordered.csv b/integration/spark/src/test/resources/complexdatareordered.csv
new file mode 100644
index 0000000..a0d6089
--- /dev/null
+++ b/integration/spark/src/test/resources/complexdatareordered.csv
@@ -0,0 +1,10 @@
+1,MAC1$MAC2$MAC3,109,4ROM size,29-11-2015,109,2738.562
+10,MAC4$MAC5$MAC6,93,1ROM size,29-11-2015,93,1714.635
+100,MAC7$MAC8$MAC9,2591,2ROM size,29-11-2015,2591,1271
+1000,MAC10$$MAC12,2531,2ROM size,29-11-2015,2531,692
+10000,MAC13$$MAC15,2408,0ROM size,29-11-2015,2408,2175
+100000,MAC16$$MAC18,1815,0ROM size,29-11-2015,1815,136
+1000000,MAC19$$MAC21,2479,4ROM size,29-11-2015,2479,1600
+100001,MAC22$$MAC24,1845,7ROM size,29-11-2015,1845,505
+100002,MAC25$$MAC27,2008,1ROM size,29-11-2015,2008,1341
+100003,MAC28$$MAC30,1121,5ROM size,29-11-2015,1121,2239

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0446801/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
index 65a09fc..73d3a32 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
@@ -295,6 +295,23 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
     sql("drop table if exists complexcarbontable")
   }
 
+  test("array<string> and string datatype for same column is not working properly") {
+    sql("create table complexcarbontable(deviceInformationId int, MAC array<string>, channelsId string, "+ 
+        "ROMSize string, purchasedate string, gamePointId double,contractNumber double) STORED BY 'org.apache.carbondata.format' "+
+        "TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId')")
+    sql("LOAD DATA local inpath './src/test/resources/complexdatareordered.csv' INTO table complexcarbontable "+
+        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,MAC,channelsId,ROMSize,purchasedate,gamePointId,contractNumber',"+
+        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
+    sql("drop table if exists complexcarbontable")
+    sql("create table primitivecarbontable(deviceInformationId int, MAC string, channelsId string, "+ 
+        "ROMSize string, purchasedate string, gamePointId double,contractNumber double) STORED BY 'org.apache.carbondata.format' "+
+        "TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId')")
+    sql("LOAD DATA local inpath './src/test/resources/complexdatareordered.csv' INTO table primitivecarbontable "+
+        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,MAC,channelsId,ROMSize,purchasedate,gamePointId,contractNumber',"+
+        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
+    sql("drop table if exists primitivecarbontable")
+  }
+  
   test(
     "test carbon table data loading when table name is in different case with create table, for " +
       "UpperCase"

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0446801/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index 57dc8a8..c7b2024 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -962,7 +962,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
           out[memberMapping[dimLen + index]] = surrogate.doubleValue();
         } else {
           try {
-            out[memberMapping[dimLen - meta.complexTypes.size() + index]] =
+            out[memberMapping[dimLen + index] - meta.complexTypes.size()] =
                 (isNull || msr == null || msr.length() == 0) ?
                     null :
                     DataTypeUtil
@@ -971,13 +971,13 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
           } catch (NumberFormatException e) {
             try {
               msr = msr.replaceAll(",", "");
-              out[memberMapping[dimLen + index]] = DataTypeUtil
+              out[memberMapping[dimLen + index] - meta.complexTypes.size()] = DataTypeUtil
                   .getMeasureValueBasedOnDataType(msr, msrDataType[meta.msrMapping[msrCount]],
                       meta.carbonMeasures[meta.msrMapping[msrCount]]);
             } catch (NumberFormatException ex) {
               LOGGER.warn("Cant not convert : " + msr
                   + " to Numeric type value. Value considered as null.");
-              out[memberMapping[dimLen - meta.complexTypes.size() + index]] = null;
+              out[memberMapping[dimLen + index] - meta.complexTypes.size()] = null;
             }
           }
         }