carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-2823] Support streaming property with datamap
Date Tue, 07 Aug 2018 10:21:48 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master abcd4f6e2 -> b9e510640


[CARBONDATA-2823] Support streaming property with datamap

Since during query, carbondata get splits from streaming segment and
columnar segments repectively, we can support streaming with index
datamap.

For preaggregate datamap, it already supported streaming table, so here
we will remove the outdated comments.

This closes #2609


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b9e51064
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b9e51064
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b9e51064

Branch: refs/heads/master
Commit: b9e51064017e428962d86ff8fccb3c1493e9864b
Parents: abcd4f6
Author: xuchuanyin <xuchuanyin@hust.edu.cn>
Authored: Mon Aug 6 15:34:51 2018 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Tue Aug 7 18:21:30 2018 +0800

----------------------------------------------------------------------
 .../datamap/bloom/BloomCoarseGrainDataMapFactory.java        | 2 +-
 .../datamap/lucene/LuceneFineGrainDataMapFactory.java        | 2 +-
 .../apache/spark/sql/execution/strategy/DDLStrategy.scala    | 8 --------
 3 files changed, 2 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b9e51064/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index 80a86cc..0d240c4 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -391,7 +391,7 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
       case ALTER_CHANGE_DATATYPE:
         return true;
       case STREAMING:
-        return true;
+        return false;
       case DELETE:
         return true;
       case UPDATE:

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b9e51064/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
index 8c7539f..ff2ffdd 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
@@ -98,7 +98,7 @@ public class LuceneFineGrainDataMapFactory extends LuceneDataMapFactoryBase<Fine
       case ALTER_CHANGE_DATATYPE:
         return true;
       case STREAMING:
-        return true;
+        return false;
       case DELETE:
         return true;
       case UPDATE:

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b9e51064/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 416dbec..94b988c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -35,7 +35,6 @@ import org.apache.spark.util.{CarbonReflectionUtils, FileUtils}
 
 import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
 import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
-import org.apache.carbondata.core.features.TableOperation
 import org.apache.carbondata.core.util.CarbonProperties
 
 /**
@@ -257,8 +256,6 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
         if CarbonEnv.getInstance(sparkSession).carbonMetastore
           .tableExists(tableName)(sparkSession) => {
 
-        // TODO remove this limiation after streaming table support 'preaggregate' DataMap
-        // if the table has 'preaggregate' DataMap, it doesn't support streaming now
         val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
           .lookupRelation(tableName)(sparkSession).asInstanceOf[CarbonRelation].carbonTable
         if (carbonTable != null && !carbonTable.getTableInfo.isTransactionalTable)
{
@@ -266,11 +263,6 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
             "Unsupported operation on non transactional table")
         }
 
-        if (carbonTable != null && !carbonTable.canAllow(carbonTable, TableOperation.STREAMING))
{
-          throw new MalformedCarbonCommandException(
-            "streaming is not supported for index datamap")
-        }
-
         // TODO remove this limitation later
         val property = properties.find(_._1.equalsIgnoreCase("streaming"))
         if (property.isDefined) {


Mime
View raw message