carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: Save useInvertedIndex info into thrift store
Date Mon, 27 Feb 2017 15:49:26 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 76c059a29 -> 1474bbf9a


Save useInvertedIndex info into thrift store

Save useInvertedIndex info into thrift store

Fix the judge of no_dic_col

add commont

style

Add comment


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/96a75b30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/96a75b30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/96a75b30

Branch: refs/heads/master
Commit: 96a75b30ae327475d22db152eaed7853dca9799c
Parents: 76c059a
Author: Zhangshunyu <zhangshunyu@huawei.com>
Authored: Thu Sep 8 15:48:03 2016 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Mon Feb 27 23:48:29 2017 +0800

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java      |  2 ++
 .../metadata/schema/table/column/ColumnSchema.java | 17 ++++++++++-------
 .../carbondata/core/util/CarbonMetadataUtil.java   |  4 +++-
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala    |  4 ++--
 .../csvbased/CarbonCSVBasedSeqGenStep.java         |  6 +++++-
 5 files changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 146b78e..9e890ef 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -804,6 +804,8 @@ public final class CarbonCommonConstants {
   public static final String COLUMN_PROPERTIES = "columnproperties";
   // table block size in MB
   public static final String TABLE_BLOCKSIZE = "table_blocksize";
+  // set in column level to disable inverted index
+  public static final String NO_INVERTED_INDEX = "no_inverted_index";
 
   /**
    * this variable is to enable/disable identify high cardinality during first data loading

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 54114d3..7aa0900 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -73,11 +73,6 @@ public class ColumnSchema implements Serializable {
   private boolean isDimensionColumn;
 
   /**
-   * Whether the column should use inverted index
-   */
-  private boolean useInvertedIndex = true;
-
-  /**
    * The group ID for column used for row format columns,
    * where in columns in each group are chunked together.
    */
@@ -174,14 +169,22 @@ public class ColumnSchema implements Serializable {
    * the isUseInvertedIndex
    */
   public boolean isUseInvertedIndex() {
-    return useInvertedIndex;
+    return this.hasEncoding(Encoding.INVERTED_INDEX);
   }
 
   /**
    * @param useInvertedIndex the useInvertedIndex to set
    */
   public void setUseInvertedIndex(boolean useInvertedIndex) {
-    this.useInvertedIndex = useInvertedIndex;
+    if (useInvertedIndex) {
+      if (!hasEncoding(Encoding.INVERTED_INDEX)) {
+        this.getEncodingList().add(Encoding.INVERTED_INDEX);
+      }
+    } else {
+      if (hasEncoding(Encoding.INVERTED_INDEX)) {
+        this.getEncodingList().remove(Encoding.INVERTED_INDEX);
+      }
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 55c0302..371855b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -345,7 +345,9 @@ public class CarbonMetadataUtil {
       if (!isSortedKeyColumn[i]) {
         dataChunk.setRowid_page_offset(blockletInfoColumnar.getKeyBlockIndexOffSets()[j]);
         dataChunk.setRowid_page_length(blockletInfoColumnar.getKeyBlockIndexLength()[j]);
-        encodings.add(Encoding.INVERTED_INDEX);
+        if (!encodings.contains(Encoding.INVERTED_INDEX)) {
+          encodings.add(Encoding.INVERTED_INDEX);
+        }
         j++;
       }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 742a145..9bfa8a9 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -448,9 +448,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser
{
     var noInvertedIdxColsProps: Array[String] = Array[String]()
     var noInvertedIdxCols: Seq[String] = Seq[String]()
 
-    if (tableProperties.get("NO_INVERTED_INDEX").isDefined) {
+    if (tableProperties.get(CarbonCommonConstants.NO_INVERTED_INDEX).isDefined) {
       noInvertedIdxColsProps =
-        tableProperties.get("NO_INVERTED_INDEX").get.split(',').map(_.trim)
+        tableProperties.get(CarbonCommonConstants.NO_INVERTED_INDEX).get.split(',').map(_.trim)
       noInvertedIdxColsProps.foreach { noInvertedIdxColProp =>
           if (!fields.exists(x => x.column.equalsIgnoreCase(noInvertedIdxColProp))) {
             val errormsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp +

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index 6e0a4a4..3edd43c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -51,6 +51,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
 import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
 import org.apache.carbondata.core.metadata.CarbonMetadata;
 import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
@@ -1933,7 +1934,10 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
       dimListExcludingNoDictionaryColumn =
           new ArrayList<>(dimensionsList.size() - meta.noDictionaryCols.length);
       for (CarbonDimension dimension : dimensionsList) {
-        if (!dimension.getEncoder().isEmpty()) {
+        // Here if dimension.getEncoder() lnly contains Encoding.INVERTED_INDEX, it
+        // means that NoDicColumn using InvertedIndex, so not put it into dic dims list.
+        if (!dimension.getEncoder().isEmpty() && !((1 == dimension.getEncoder().size())
&&
+            dimension.getEncoder().contains(Encoding.INVERTED_INDEX))) {
           dimListExcludingNoDictionaryColumn.add(dimension);
         }
       }


Mime
View raw message