From commits-return-12107-archive-asf-public=cust-asf.ponee.io@carbondata.apache.org Tue Jul 17 17:37:32 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 21056180600 for ; Tue, 17 Jul 2018 17:37:31 +0200 (CEST) Received: (qmail 65181 invoked by uid 500); 17 Jul 2018 15:37:31 -0000 Mailing-List: contact commits-help@carbondata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.apache.org Delivered-To: mailing list commits@carbondata.apache.org Received: (qmail 65172 invoked by uid 99); 17 Jul 2018 15:37:31 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 17 Jul 2018 15:37:31 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 251F8DFAD2; Tue, 17 Jul 2018 15:37:31 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xuchuanyin@apache.org To: commits@carbondata.apache.org Message-Id: <71955a8decf8468d83e92f7f4e0aeabf@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: carbondata git commit: [CARBONDATA-2724][DataMap]Unsupported create datamap on table with V1 or V2 format data Date: Tue, 17 Jul 2018 15:37:31 +0000 (UTC) Repository: carbondata Updated Branches: refs/heads/master 81038f55e -> a16289786 [CARBONDATA-2724][DataMap]Unsupported create datamap on table with V1 or V2 format data block creating datamap on carbon table with V1 or V2 format Currently the version info is read from carbon data file This closes #2488 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a1628978 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a1628978 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a1628978 Branch: refs/heads/master Commit: a162897862c92947ea8fd63713b7dbe6098f3b13 Parents: 81038f5 Author: ndwangsen Authored: Wed Jul 11 17:41:25 2018 +0800 Committer: xuchuanyin Committed: Tue Jul 17 23:35:50 2018 +0800 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonUtil.java | 51 ++++++++++++++++++++ .../datamap/CarbonCreateDataMapCommand.scala | 8 ++- 2 files changed, 58 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/a1628978/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 9796696..642fe8e 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -88,6 +88,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.format.BlockletHeader; import org.apache.carbondata.format.DataChunk2; import org.apache.carbondata.format.DataChunk3; +import org.apache.carbondata.format.FileHeader; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -3184,4 +3185,54 @@ public final class CarbonUtil { } return columnLocalDictGenMap; } + + /** + * This method get the carbon file format version + * + * @param carbonTable + * carbon Table + */ + public static ColumnarFormatVersion getFormatVersion(CarbonTable carbonTable) + throws IOException { + String storePath = null; + // if the carbontable is support flat folder + boolean supportFlatFolder = carbonTable.isSupportFlatFolder(); + if (supportFlatFolder) { + storePath = carbonTable.getTablePath(); + } else { + // get the valid segments + SegmentStatusManager segmentStatusManager = + new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier()); + SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = + segmentStatusManager.getValidAndInvalidSegments(); + List validSegments = validAndInvalidSegmentsInfo.getValidSegments(); + CarbonProperties carbonProperties = CarbonProperties.getInstance(); + if (validSegments.isEmpty()) { + return carbonProperties.getFormatVersion(); + } + storePath = carbonTable.getSegmentPath(validSegments.get(0).getSegmentNo()); + } + + CarbonFile[] carbonFiles = FileFactory + .getCarbonFile(storePath) + .listFiles(new CarbonFileFilter() { + @Override + public boolean accept(CarbonFile file) { + if (file == null) { + return false; + } + return file.getName().endsWith("carbondata"); + } + }); + if (carbonFiles == null || carbonFiles.length < 1) { + return CarbonProperties.getInstance().getFormatVersion(); + } + + CarbonFile carbonFile = carbonFiles[0]; + // get the carbon file header + CarbonHeaderReader headerReader = new CarbonHeaderReader(carbonFile.getCanonicalPath()); + FileHeader fileHeader = headerReader.readHeader(); + int version = fileHeader.getVersion(); + return ColumnarFormatVersion.valueOf((short)version); + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/a1628978/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala index 7600160..336793e 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala @@ -26,9 +26,10 @@ import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandExcept import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.datamap.{DataMapProvider, DataMapStoreManager} import org.apache.carbondata.core.datamap.status.DataMapStatusManager +import org.apache.carbondata.core.metadata.ColumnarFormatVersion import org.apache.carbondata.core.metadata.schema.datamap.{DataMapClassProvider, DataMapProperty} import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema} -import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil} import org.apache.carbondata.datamap.{DataMapManager, IndexDataMapProvider} import org.apache.carbondata.events._ @@ -79,6 +80,11 @@ case class CarbonCreateDataMapCommand( s"$dmProviderName datamap") } + if (mainTable !=null && CarbonUtil.getFormatVersion(mainTable) != ColumnarFormatVersion.V3) { + throw new MalformedCarbonCommandException(s"Unsupported operation on table with " + + s"V1 or V2 format data") + } + dataMapSchema = new DataMapSchema(dataMapName, dmProviderName) val property = dmProperties.map(x => (x._1.trim, x._2.trim)).asJava