From commits-return-12036-archive-asf-public=cust-asf.ponee.io@carbondata.apache.org Fri Jul 13 10:28:58 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id ED141180626 for ; Fri, 13 Jul 2018 10:28:57 +0200 (CEST) Received: (qmail 93935 invoked by uid 500); 13 Jul 2018 08:28:57 -0000 Mailing-List: contact commits-help@carbondata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.apache.org Delivered-To: mailing list commits@carbondata.apache.org Received: (qmail 93918 invoked by uid 99); 13 Jul 2018 08:28:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 13 Jul 2018 08:28:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8ED14DFA59; Fri, 13 Jul 2018 08:28:56 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ravipesala@apache.org To: commits@carbondata.apache.org Message-Id: <309d79c09a3f44f6b1e28077b717de64@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: carbondata git commit: [CARBONDATA-2648] Fixed NPE issue with legacy store when CACHE_LEVEL is Blocklet Date: Fri, 13 Jul 2018 08:28:56 +0000 (UTC) Repository: carbondata Updated Branches: refs/heads/master 1fd370399 -> bd02656ab [CARBONDATA-2648] Fixed NPE issue with legacy store when CACHE_LEVEL is Blocklet Things done as part of this PR: Fixed Null pointer exception when store is of <= 1.1 version and DataMap is of type BlockletDataMap. Added clearing of SegmentProperties cache holder from executor Problem 1: Null pointer exception thrown when store is of <= 1.1 version and DataMap is of type BlockletDataMap. Analysis: In BlcokletDataMap schema is created to consider blockletInfo while adding to unsafe but in case of legacy store we dont weite the blocklet Info. This lead to Null pointer exception while calculating the size of row during adding to unsafe. Solution For legacy store always call super class methods which is BlockDataMap. This closes #2499 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bd02656a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bd02656a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bd02656a Branch: refs/heads/master Commit: bd02656abc559040111c636c909541c8441b8132 Parents: 1fd3703 Author: manishgupta88 Authored: Thu Jul 12 21:35:45 2018 +0530 Committer: ravipesala Committed: Fri Jul 13 13:58:46 2018 +0530 ---------------------------------------------------------------------- .../core/datamap/DistributableDataMapFormat.java | 4 ++++ .../indexstore/BlockletDataMapIndexStore.java | 9 +++++---- .../TableBlockIndexUniqueIdentifierWrapper.java | 18 ++++++++++++++++++ .../indexstore/blockletindex/BlockletDataMap.java | 14 +++++++++++++- .../core/indexstore/row/DataMapRow.java | 5 +++++ 5 files changed, 45 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/bd02656a/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java b/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java index 762d89c..007541d 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java @@ -26,6 +26,7 @@ import java.util.List; import org.apache.carbondata.core.datamap.dev.DataMap; import org.apache.carbondata.core.datamap.dev.expr.DataMapDistributableWrapper; import org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapper; +import org.apache.carbondata.core.datastore.block.SegmentPropertiesAndSchemaHolder; import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.PartitionSpec; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; @@ -97,6 +98,9 @@ public class DistributableDataMapFormat extends FileInputFormat blockMetaInfoMap, - CarbonTable carbonTable) + CarbonTable carbonTable, boolean addTableBlockToUnsafe) throws IOException, MemoryException { String uniqueTableSegmentIdentifier = identifier.getUniqueTableSegmentIdentifier(); @@ -265,7 +266,7 @@ public class BlockletDataMapIndexStore dataMap.init(new BlockletDataMapModel(carbonTable, identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier .getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()), - blockMetaInfoMap, identifier.getSegmentId())); + blockMetaInfoMap, identifier.getSegmentId(), addTableBlockToUnsafe)); } return dataMap; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/bd02656a/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifierWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifierWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifierWrapper.java index 3411397..0924f1f 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifierWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifierWrapper.java @@ -35,6 +35,10 @@ public class TableBlockIndexUniqueIdentifierWrapper implements Serializable { // holds the reference to CarbonTable private CarbonTable carbonTable; + /** + * flag to specify whether to load table block metadata in unsafe or safe. Default value is true + */ + private boolean addTableBlockToUnsafe = true; public TableBlockIndexUniqueIdentifierWrapper( TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier, CarbonTable carbonTable) { @@ -42,6 +46,16 @@ public class TableBlockIndexUniqueIdentifierWrapper implements Serializable { this.carbonTable = carbonTable; } + // Note: The constructor is getting used in extensions with other functionalities. + // Kindly do not remove + public TableBlockIndexUniqueIdentifierWrapper( + TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier, CarbonTable carbonTable, + boolean addTableBlockToUnsafe) { + this(tableBlockIndexUniqueIdentifier, carbonTable); + this.addTableBlockToUnsafe = addTableBlockToUnsafe; + } + + public TableBlockIndexUniqueIdentifier getTableBlockIndexUniqueIdentifier() { return tableBlockIndexUniqueIdentifier; } @@ -49,4 +63,8 @@ public class TableBlockIndexUniqueIdentifierWrapper implements Serializable { public CarbonTable getCarbonTable() { return carbonTable; } + + public boolean isAddTableBlockToUnsafe() { + return addTableBlockToUnsafe; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/bd02656a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java index d53e936..bbe37c0 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java @@ -73,6 +73,9 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { } protected CarbonRowSchema[] getTaskSummarySchema() { + if (isLegacyStore) { + return super.getTaskSummarySchema(); + } SegmentPropertiesAndSchemaHolder.SegmentPropertiesWrapper segmentPropertiesWrapper = SegmentPropertiesAndSchemaHolder.getInstance() .getSegmentPropertiesWrapper(segmentPropertiesIndex); @@ -84,6 +87,9 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { } protected CarbonRowSchema[] getFileFooterEntrySchema() { + if (isLegacyStore) { + return super.getFileFooterEntrySchema(); + } return SegmentPropertiesAndSchemaHolder.getInstance() .getSegmentPropertiesWrapper(segmentPropertiesIndex).getBlockletFileFooterEntrySchema(); } @@ -199,7 +205,7 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { public ExtendedBlocklet getDetailedBlocklet(String blockletId) { if (isLegacyStore) { - super.getDetailedBlocklet(blockletId); + return super.getDetailedBlocklet(blockletId); } int absoluteBlockletId = Integer.parseInt(blockletId); DataMapRow safeRow = memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), absoluteBlockletId) @@ -210,10 +216,16 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { } protected short getBlockletId(DataMapRow dataMapRow) { + if (isLegacyStore) { + return super.getBlockletId(dataMapRow); + } return dataMapRow.getShort(BLOCKLET_ID_INDEX); } protected ExtendedBlocklet createBlocklet(DataMapRow row, String fileName, short blockletId) { + if (isLegacyStore) { + return super.createBlocklet(row, fileName, blockletId); + } ExtendedBlocklet blocklet = new ExtendedBlocklet(fileName, blockletId + ""); BlockletDetailInfo detailInfo = getBlockletDetailInfo(row, blockletId, blocklet); detailInfo.setColumnSchemas(getColumnSchema()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/bd02656a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java index e9ce170..d9467c6 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java @@ -86,6 +86,11 @@ public abstract class DataMapRow implements Serializable { case VARIABLE_INT: return getLengthInBytes(ordinal) + 4; case STRUCT: + DataMapRow row = getRow(ordinal); + CarbonRowSchema[] childSchemas = + ((CarbonRowSchema.StructCarbonRowSchema) schemas[ordinal]).getChildSchemas(); + // set the child schema. Because schema is transient it can be null + row.setSchemas(childSchemas); return getRow(ordinal).getTotalSizeInBytes(); default: throw new UnsupportedOperationException("wrong type");