From commits-return-11658-archive-asf-public=cust-asf.ponee.io@carbondata.apache.org Fri Jun 22 03:34:07 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id B0A891807C4 for ; Fri, 22 Jun 2018 03:34:04 +0200 (CEST) Received: (qmail 89744 invoked by uid 500); 22 Jun 2018 01:34:03 -0000 Mailing-List: contact commits-help@carbondata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.apache.org Delivered-To: mailing list commits@carbondata.apache.org Received: (qmail 89211 invoked by uid 99); 22 Jun 2018 01:34:03 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 22 Jun 2018 01:34:03 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B885DE1147; Fri, 22 Jun 2018 01:34:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jackylk@apache.org To: commits@carbondata.apache.org Date: Fri, 22 Jun 2018 01:34:21 -0000 Message-Id: <76daf950f42449fcbd99f5cb965f900c@git.apache.org> In-Reply-To: <3146ba9062d748839b5da035a0a6df0e@git.apache.org> References: <3146ba9062d748839b5da035a0a6df0e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [20/50] [abbrv] carbondata git commit: [CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure case for creation of multi reader for non-transactional table [CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure case for creation of multi reader for non-transactional table Issue : CarbonIterator inside CarbonRecordReader was keeping reference of RowBatch and it is not being closed inside CarbonRecordReader. sort_column with measure was considering all the dimension column along with given column. if creation of one CarbonReader for non transactional table is failed then we are not able to create another CarbonReader. Solution : close() called inside hasNext() to clear the previous iterator before iterating over the next CarbonReader.. if sortcolumn is not empty and sortcolumnsList contains the fields check is added to finally in sortcolumn props. Clear the datamap in catch block, if creation of CarbonReader is failed. This closes #2362 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/19312ab5 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/19312ab5 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/19312ab5 Branch: refs/heads/carbonstore Commit: 19312ab5b35654e89f04ec79881d928c859f8a07 Parents: 290ef5a Author: rahul Authored: Tue Jun 5 13:09:36 2018 +0530 Committer: kumarvishal09 Committed: Tue Jun 12 17:01:53 2018 +0530 ---------------------------------------------------------------------- .../createTable/TestNonTransactionalCarbonTable.scala | 7 ++++--- .../java/org/apache/carbondata/sdk/file/CarbonReader.java | 2 ++ .../org/apache/carbondata/sdk/file/CarbonWriterBuilder.java | 5 ----- 3 files changed, 6 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala index b275bb8..805fc71 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala @@ -368,8 +368,9 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { |'carbondata' LOCATION |'$writerPath' """.stripMargin) - checkExistence(sql("describe formatted sdkOutputTable"), true, "age") - + checkExistence(sql("describe formatted sdkOutputTable"), true, "SORT_COLUMNS age") + checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS name,age") + checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS age,name") buildTestDataSingleFile() assert(new File(writerPath).exists()) sql("DROP TABLE IF EXISTS sdkOutputTable") @@ -402,7 +403,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { intercept[RuntimeException] { buildTestDataWithSortColumns(List("")) } - + assert(!(new File(writerPath).exists())) cleanTestData() } http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java index 9af710f..be809e6 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java @@ -74,6 +74,8 @@ public class CarbonReader { return false; } else { index++; + // current reader is closed + currentReader.close(); currentReader = readers.get(index); return currentReader.nextKeyValue(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java index bd64568..0f670fe 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java @@ -484,11 +484,6 @@ public class CarbonWriterBuilder { if (isSortColumn > -1) { columnSchema.setSortColumn(true); sortColumnsSchemaList[isSortColumn] = columnSchema; - } else if (!sortColumnsList.isEmpty() && columnSchema.isDimensionColumn() - && columnSchema.getNumberOfChild() < 1) { - columnSchema.setSortColumn(true); - sortColumnsSchemaList[i] = columnSchema; - i++; } } }