From commits-return-13535-archive-asf-public=cust-asf.ponee.io@carbondata.apache.org Mon Sep 24 08:54:59 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 506D6180675 for ; Mon, 24 Sep 2018 08:54:59 +0200 (CEST) Received: (qmail 78579 invoked by uid 500); 24 Sep 2018 06:54:58 -0000 Mailing-List: contact commits-help@carbondata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.apache.org Delivered-To: mailing list commits@carbondata.apache.org Received: (qmail 78485 invoked by uid 99); 24 Sep 2018 06:54:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 24 Sep 2018 06:54:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 194F3E09EC; Mon, 24 Sep 2018 06:54:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kumarvishal09@apache.org To: commits@carbondata.apache.org Message-Id: <81f1e3c7a94743f4975c1ab57c623ae6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: carbondata git commit: [CARBONDATA-2958] Compaction with CarbonProperty 'carbon.enable.page.level.reader.in.compaction' enabled fails as Compressor is null Date: Mon, 24 Sep 2018 06:54:58 +0000 (UTC) Repository: carbondata Updated Branches: refs/heads/master 8320918e5 -> ed8564421 [CARBONDATA-2958] Compaction with CarbonProperty 'carbon.enable.page.level.reader.in.compaction' enabled fails as Compressor is null Problem: When CarbonProperty 'carbon.enable.page.level.reader.in.compaction' is enabled, compaction fails throwing Null Pointer Exception as compressor is Null Solution: Set compressor from pageMetaData This closes #2745 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ed856442 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ed856442 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ed856442 Branch: refs/heads/master Commit: ed856442166a96d1b414336945fb1dbc1d514c4a Parents: 8320918 Author: Indhumathi27 Authored: Fri Sep 21 15:24:39 2018 +0530 Committer: kumarvishal09 Committed: Mon Sep 24 12:24:28 2018 +0530 ---------------------------------------------------------------------- ...essedDimChunkFileBasedPageLevelReaderV3.java | 7 +++++++ ...andardPartitionTableCompactionTestCase.scala | 22 ++++++++++++++++++++ 2 files changed, 29 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/ed856442/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java index e69984b..6efaf8a 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java @@ -23,8 +23,10 @@ import java.nio.ByteBuffer; import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; +import org.apache.carbondata.core.datastore.compression.CompressorFactory; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; +import org.apache.carbondata.core.util.CarbonMetadataUtil; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.format.DataChunk2; import org.apache.carbondata.format.DataChunk3; @@ -146,6 +148,11 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3 DataChunk3 dataChunk3 = dimensionRawColumnChunk.getDataChunkV3(); pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber); + + if (compressor == null) { + this.compressor = CompressorFactory.getInstance().getCompressor( + CarbonMetadataUtil.getCompressorNameFromChunkMeta(pageMetadata.getChunk_meta())); + } // calculating the start point of data // as buffer can contain multiple column data, start point will be datachunkoffset + // data chunk length + page offset http://git-wip-us.apache.org/repos/asf/carbondata/blob/ed856442/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala index 33e761f..23c2aa0 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala @@ -16,6 +16,7 @@ */ package org.apache.carbondata.spark.testsuite.standardpartition +import org.apache.spark.sql.Row import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll @@ -183,6 +184,27 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA sql(s"""alter table compactionupdatepartition compact 'major'""").collect } + test("test compaction when 'carbon.enable.page.level.reader.in.compaction' is set to true") { + sql("DROP TABLE IF EXISTS originTable") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION, "true") + sql("create table originTable(a int, b string) stored by 'carbondata'") + sql("insert into originTable values(1,'abc')") + sql("insert into originTable values(1,'abc')") + sql("insert into originTable values(1,'abc')") + sql("insert into originTable values(1,'abc')") + sql("alter table originTable compact 'minor'") + checkAnswer(sql("select count(*) from originTable"), Seq(Row(4))) + sql("insert into originTable values(1,'abc')") + sql("insert into originTable values(1,'abc')") + sql("insert into originTable values(1,'abc')") + sql("insert into originTable values(1,'abc')") + sql("alter table originTable compact 'major'") + checkAnswer(sql("select count(*) from originTable"), Seq(Row(8))) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION, "false") + } + override def afterAll = { dropTable }