carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xuchuan...@apache.org
Subject carbondata git commit: [CARBONDATA-2724][DataMap]Unsupported create datamap on table with V1 or V2 format data
Date Tue, 17 Jul 2018 15:37:31 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 81038f55e -> a16289786


[CARBONDATA-2724][DataMap]Unsupported create datamap on table with V1 or V2 format data

block creating datamap on carbon table with V1 or V2 format
Currently the version info is read from carbon data file

This closes #2488


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a1628978
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a1628978
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a1628978

Branch: refs/heads/master
Commit: a162897862c92947ea8fd63713b7dbe6098f3b13
Parents: 81038f5
Author: ndwangsen <luffy.wang@huawei.com>
Authored: Wed Jul 11 17:41:25 2018 +0800
Committer: xuchuanyin <xuchuanyin@hust.edu.cn>
Committed: Tue Jul 17 23:35:50 2018 +0800

----------------------------------------------------------------------
 .../apache/carbondata/core/util/CarbonUtil.java | 51 ++++++++++++++++++++
 .../datamap/CarbonCreateDataMapCommand.scala    |  8 ++-
 2 files changed, 58 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/a1628978/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 9796696..642fe8e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -88,6 +88,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
 import org.apache.carbondata.format.BlockletHeader;
 import org.apache.carbondata.format.DataChunk2;
 import org.apache.carbondata.format.DataChunk3;
+import org.apache.carbondata.format.FileHeader;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -3184,4 +3185,54 @@ public final class CarbonUtil {
     }
     return columnLocalDictGenMap;
   }
+
+  /**
+   * This method get the carbon file format version
+   *
+   * @param carbonTable
+   * carbon Table
+   */
+  public static ColumnarFormatVersion getFormatVersion(CarbonTable carbonTable)
+      throws IOException {
+    String storePath = null;
+    // if the carbontable is support flat folder
+    boolean supportFlatFolder = carbonTable.isSupportFlatFolder();
+    if (supportFlatFolder) {
+      storePath = carbonTable.getTablePath();
+    } else {
+      // get the valid segments
+      SegmentStatusManager segmentStatusManager =
+          new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier());
+      SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo =
+          segmentStatusManager.getValidAndInvalidSegments();
+      List<Segment> validSegments = validAndInvalidSegmentsInfo.getValidSegments();
+      CarbonProperties carbonProperties = CarbonProperties.getInstance();
+      if (validSegments.isEmpty()) {
+        return carbonProperties.getFormatVersion();
+      }
+      storePath = carbonTable.getSegmentPath(validSegments.get(0).getSegmentNo());
+    }
+
+    CarbonFile[] carbonFiles = FileFactory
+        .getCarbonFile(storePath)
+        .listFiles(new CarbonFileFilter() {
+          @Override
+          public boolean accept(CarbonFile file) {
+            if (file == null) {
+              return false;
+            }
+            return file.getName().endsWith("carbondata");
+          }
+        });
+    if (carbonFiles == null || carbonFiles.length < 1) {
+      return CarbonProperties.getInstance().getFormatVersion();
+    }
+
+    CarbonFile carbonFile = carbonFiles[0];
+    // get the carbon file header
+    CarbonHeaderReader headerReader = new CarbonHeaderReader(carbonFile.getCanonicalPath());
+    FileHeader fileHeader = headerReader.readHeader();
+    int version = fileHeader.getVersion();
+    return ColumnarFormatVersion.valueOf((short)version);
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a1628978/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 7600160..336793e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -26,9 +26,10 @@ import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandExcept
 import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.core.datamap.{DataMapProvider, DataMapStoreManager}
 import org.apache.carbondata.core.datamap.status.DataMapStatusManager
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion
 import org.apache.carbondata.core.metadata.schema.datamap.{DataMapClassProvider, DataMapProperty}
 import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
-import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
 import org.apache.carbondata.datamap.{DataMapManager, IndexDataMapProvider}
 import org.apache.carbondata.events._
 
@@ -79,6 +80,11 @@ case class CarbonCreateDataMapCommand(
                                                 s"$dmProviderName datamap")
     }
 
+    if (mainTable !=null && CarbonUtil.getFormatVersion(mainTable) != ColumnarFormatVersion.V3)
{
+      throw new MalformedCarbonCommandException(s"Unsupported operation on table with " +
+                                                s"V1 or V2 format data")
+    }
+
     dataMapSchema = new DataMapSchema(dataMapName, dmProviderName)
 
     val property = dmProperties.map(x => (x._1.trim, x._2.trim)).asJava


Mime
View raw message