carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [29/50] [abbrv] incubator-carbondata git commit: [Bug] While defining Column groups, if group is not defined in schema order than data loading is failing (#829)
Date Wed, 20 Jul 2016 10:13:57 GMT
[Bug] While defining Column groups, if group is not defined in schema order than data loading
is failing (#829)

This fix is to arrange the group in schema order

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8b3c1350
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8b3c1350
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8b3c1350

Branch: refs/heads/master
Commit: 8b3c13506e3344f01b2893704baf411ec76501f9
Parents: 8c83954
Author: ashokblend <ashok.blend@gmail.com>
Authored: Mon Jul 18 16:51:30 2016 +0530
Committer: Venkata Ramana G <g.ramana.v1@gmail.com>
Committed: Mon Jul 18 16:51:30 2016 +0530

----------------------------------------------------------------------
 .../CarbonDictionaryMetadataReaderImpl.java     |  8 +++--
 .../core/reader/CarbonDictionaryReaderImpl.java |  8 +++--
 .../CarbonDictionarySortIndexReaderImpl.java    |  8 +++--
 .../CarbonDictionarySortIndexWriterImpl.java    |  8 +++--
 .../org/apache/spark/sql/CarbonSqlParser.scala  |  2 +-
 .../org/carbondata/spark/util/CommonUtil.scala  | 32 ++++++++++++++++++++
 .../apache/spark/sql/TestCarbonSqlParser.scala  | 11 +++++++
 .../ColumnGroupDataTypesTestCase.scala          | 13 ++++++++
 8 files changed, 77 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
b/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
index a43fc22..227dc8d 100644
--- a/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
+++ b/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
@@ -23,11 +23,12 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.carbondata.common.factory.CarbonCommonFactory;
 import org.carbondata.core.carbon.CarbonTableIdentifier;
 import org.carbondata.core.carbon.ColumnIdentifier;
-import org.carbondata.core.carbon.path.CarbonStorePath;
 import org.carbondata.core.carbon.path.CarbonTablePath;
 import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.service.PathService;
 import org.carbondata.format.ColumnDictionaryChunkMeta;
 
 import org.apache.thrift.TBase;
@@ -154,8 +155,9 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad
    * This method will form the path for dictionary metadata file for a given column
    */
   protected void initFileLocation() {
-    CarbonTablePath carbonTablePath =
-        CarbonStorePath.getCarbonTablePath(this.hdfsStorePath, carbonTableIdentifier);
+    PathService pathService = CarbonCommonFactory.getPathService();
+    CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(columnIdentifier,
+                this.hdfsStorePath, carbonTableIdentifier);
     this.columnDictionaryMetadataFilePath =
         carbonTablePath.getDictionaryMetaFilePath(columnIdentifier.getColumnId());
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java
b/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java
index 5e5207a..a93d47a 100644
--- a/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java
+++ b/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java
@@ -24,11 +24,12 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.carbondata.common.factory.CarbonCommonFactory;
 import org.carbondata.core.carbon.CarbonTableIdentifier;
 import org.carbondata.core.carbon.ColumnIdentifier;
-import org.carbondata.core.carbon.path.CarbonStorePath;
 import org.carbondata.core.carbon.path.CarbonTablePath;
 import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.service.PathService;
 import org.carbondata.format.ColumnDictionaryChunk;
 
 import org.apache.thrift.TBase;
@@ -199,8 +200,9 @@ public class CarbonDictionaryReaderImpl implements CarbonDictionaryReader
{
    * This method will form the path for dictionary file for a given column
    */
   protected void initFileLocation() {
-    CarbonTablePath carbonTablePath =
-        CarbonStorePath.getCarbonTablePath(this.hdfsStorePath, carbonTableIdentifier);
+    PathService pathService = CarbonCommonFactory.getPathService();
+    CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(columnIdentifier,
+                this.hdfsStorePath, carbonTableIdentifier);
     this.columnDictionaryFilePath = carbonTablePath
         .getDictionaryFilePath(columnIdentifier.getColumnId());
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
b/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
index 337a857..54e8950 100644
--- a/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
+++ b/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
@@ -21,14 +21,15 @@ package org.carbondata.core.reader.sortindex;
 import java.io.IOException;
 import java.util.List;
 
+import org.carbondata.common.factory.CarbonCommonFactory;
 import org.carbondata.common.logging.LogService;
 import org.carbondata.common.logging.LogServiceFactory;
 import org.carbondata.core.carbon.CarbonTableIdentifier;
 import org.carbondata.core.carbon.ColumnIdentifier;
-import org.carbondata.core.carbon.path.CarbonStorePath;
 import org.carbondata.core.carbon.path.CarbonTablePath;
 import org.carbondata.core.datastorage.store.impl.FileFactory;
 import org.carbondata.core.reader.ThriftReader;
+import org.carbondata.core.service.PathService;
 import org.carbondata.core.util.CarbonUtil;
 import org.carbondata.format.ColumnSortInfo;
 
@@ -150,8 +151,9 @@ public class CarbonDictionarySortIndexReaderImpl implements CarbonDictionarySort
   }
 
   protected void initPath() {
-    CarbonTablePath carbonTablePath =
-        CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
+    PathService pathService = CarbonCommonFactory.getPathService();
+    CarbonTablePath carbonTablePath = pathService
+            .getCarbonTablePath(columnIdentifier, carbonStorePath, carbonTableIdentifier);
     String dictionaryPath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
     long dictOffset = CarbonUtil.getFileSize(dictionaryPath);
     this.sortIndexFilePath =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
index 89fba44..f0b935c 100644
--- a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
+++ b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
@@ -23,7 +23,7 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
 
-import org.carbondata.common.ext.PathFactory;
+import org.carbondata.common.factory.CarbonCommonFactory;
 import org.carbondata.common.logging.LogService;
 import org.carbondata.common.logging.LogServiceFactory;
 import org.carbondata.core.carbon.CarbonTableIdentifier;
@@ -32,6 +32,7 @@ import org.carbondata.core.carbon.path.CarbonTablePath;
 import org.carbondata.core.constants.CarbonCommonConstants;
 import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
 import org.carbondata.core.datastorage.store.impl.FileFactory;
+import org.carbondata.core.service.PathService;
 import org.carbondata.core.util.CarbonProperties;
 import org.carbondata.core.util.CarbonUtil;
 import org.carbondata.core.writer.ThriftWriter;
@@ -150,7 +151,8 @@ public class CarbonDictionarySortIndexWriterImpl implements CarbonDictionarySort
   }
 
   protected void initPath() {
-    CarbonTablePath carbonTablePath = PathFactory.getInstance()
+    PathService pathService = CarbonCommonFactory.getPathService();
+    CarbonTablePath carbonTablePath = pathService
         .getCarbonTablePath(columnIdentifier, carbonStorePath, carbonTableIdentifier);
     String dictionaryPath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
     long dictOffset = CarbonUtil.getFileSize(dictionaryPath);
@@ -164,7 +166,7 @@ public class CarbonDictionarySortIndexWriterImpl implements CarbonDictionarySort
    *
    * @param carbonTablePath
    */
-  private void cleanUpOldSortIndex(CarbonTablePath carbonTablePath) {
+  protected void cleanUpOldSortIndex(CarbonTablePath carbonTablePath) {
     CarbonFile sortIndexFile =
         FileFactory.getCarbonFile(sortIndexFilePath, FileFactory.getFileType(sortIndexFilePath));
     CarbonFile[] files =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 7de452a..40f9e70 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -527,7 +527,7 @@ class CarbonSqlParser()
         splittedColGrps :+= arrangedColGrp
       }
       // This will  be furthur handled.
-      splittedColGrps
+      CommonUtil.arrangeColGrpsInSchemaOrder(splittedColGrps, dims)
     }
     else {
       null

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala
index 5b8c0b2..708a00b 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala
@@ -167,4 +167,36 @@ object CommonUtil {
       false
     }
   }
+
+  /**
+   * @param colGrps
+   * @param dims
+   * @return columns of column groups in schema order
+   */
+  def arrangeColGrpsInSchemaOrder(colGrps: Seq[String], dims: Seq[Field]): Seq[String] =
{
+    def sortByIndex(colGrp1: String, colGrp2: String) = {
+      val firstCol1 = colGrp1.split(",")(0)
+      val firstCol2 = colGrp2.split(",")(0)
+      val dimIndex1: Int = getDimIndex(firstCol1, dims)
+      val dimIndex2: Int = getDimIndex(firstCol2, dims)
+      dimIndex1 < dimIndex2
+    }
+    val sortedColGroups: Seq[String] = colGrps.sortWith(sortByIndex)
+    sortedColGroups
+  }
+
+  /**
+   * @param colName
+   * @param dims
+   * @return return index for given column in dims
+   */
+  def getDimIndex(colName: String, dims: Seq[Field]): Int = {
+    var index: Int = -1
+    dims.zipWithIndex.foreach { h =>
+      if (h._1.column.equalsIgnoreCase(colName)) {
+        index = h._2.toInt
+      }
+    }
+    index
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
index f825ce2..983d488 100644
--- a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
+++ b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
@@ -79,6 +79,17 @@ class TestCarbonSqlParser extends QueryTest {
     assert(colgrps.lift(2).get.equalsIgnoreCase("col7,col8"))
 
   }
+  test("Test-updateColumnGroupsInField_disordered") {
+    val colGroupStr = "(col5,col6),(col2,col3),(col7,col8)"
+    val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
+    var fields: Seq[Field] = loadAllFields
+    val stub = new TestCarbonSqlParserStub()
+    val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
+    assert(colgrps.lift(0).get.equalsIgnoreCase("col2,col3"))
+    assert(colgrps.lift(1).get.equalsIgnoreCase("col5,col6"))
+    assert(colgrps.lift(2).get.equalsIgnoreCase("col7,col8"))
+
+  }
   test("Test-ColumnGroupsInvalidField_Shouldnotallow") {
     val colGroupStr = "(col1,col2),(col10,col6),(col7,col8)"
     val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
index 122e0f1..759b84c 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
@@ -40,6 +40,7 @@ class ColumnGroupDataTypesTestCase extends QueryTest with BeforeAndAfterAll
{
     //column group with dictionary exclude after column group
     sql("create table colgrp_dictexclude_after (column1 string,column2 string,column3 string,column4
string,column5 string,column6 string,column7 string,column8 string,column9 string,column10
string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format'
TBLPROPERTIES ('DICTIONARY_EXCLUDE'='column10',\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")")
     sql("LOAD DATA LOCAL INPATH './src/test/resources/10dim_4msr.csv' INTO table colgrp_dictexclude_after
options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
+    
   }
 
   test("select all dimension query") {
@@ -127,10 +128,22 @@ class ColumnGroupDataTypesTestCase extends QueryTest with BeforeAndAfterAll
{
       sql("select * from normal where column3 != column4"))
   }
 
+  test("Column Group not in order with schema") {
+      //Add column group in order different then schema
+    try {
+      sql("create table colgrp_disorder (column1 string,column2 string,column3 string,column4
string,column5 string,column6 string,column7 string,column8 string,column9 string,column10
string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format'
TBLPROPERTIES (\"COLUMN_GROUPS\"=\"(column7,column8),(column2,column3,column4)\")")
+      sql("LOAD DATA LOCAL INPATH './src/test/resources/10dim_4msr.csv' INTO table colgrp_disorder
options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
+      assert(true)  
+    } catch {
+      case ex: Exception => assert(false)
+    }
+    
+  }
   override def afterAll {
     sql("drop table colgrp")
     sql("drop table normal")
     sql("drop table colgrp_dictexclude_before")
     sql("drop table colgrp_dictexclude_after")
+    sql("drop table if exists colgrp_disorder")
   }
 }
\ No newline at end of file


Mime
View raw message