carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [20/50] [abbrv] carbondata git commit: [CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure case for creation of multi reader for non-transactional table
Date Fri, 22 Jun 2018 01:34:21 GMT
[CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure case for creation
of
 multi reader for non-transactional table

Issue :
CarbonIterator inside CarbonRecordReader was keeping reference of RowBatch and it is not being
closed inside CarbonRecordReader. sort_column with measure was considering all the dimension
column along with given column. if creation of one CarbonReader for non transactional table
is failed then we are not able to create another CarbonReader.

Solution :
close() called inside hasNext() to clear the previous iterator before iterating over the next
CarbonReader..
if sortcolumn is not empty and sortcolumnsList contains the fields check is added to finally
in sortcolumn props.
Clear the datamap in catch block, if creation of CarbonReader is failed.

This closes #2362


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/19312ab5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/19312ab5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/19312ab5

Branch: refs/heads/carbonstore
Commit: 19312ab5b35654e89f04ec79881d928c859f8a07
Parents: 290ef5a
Author: rahul <rahul.kumar@knoldus.in>
Authored: Tue Jun 5 13:09:36 2018 +0530
Committer: kumarvishal09 <kumarvishal1802@gmail.com>
Committed: Tue Jun 12 17:01:53 2018 +0530

----------------------------------------------------------------------
 .../createTable/TestNonTransactionalCarbonTable.scala         | 7 ++++---
 .../java/org/apache/carbondata/sdk/file/CarbonReader.java     | 2 ++
 .../org/apache/carbondata/sdk/file/CarbonWriterBuilder.java   | 5 -----
 3 files changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index b275bb8..805fc71 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -368,8 +368,9 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
-    checkExistence(sql("describe formatted sdkOutputTable"), true, "age")
-
+    checkExistence(sql("describe formatted sdkOutputTable"), true, "SORT_COLUMNS        
               age")
+    checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS       
                name,age")
+    checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS       
                age,name")
     buildTestDataSingleFile()
     assert(new File(writerPath).exists())
     sql("DROP TABLE IF EXISTS sdkOutputTable")
@@ -402,7 +403,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
     intercept[RuntimeException] {
       buildTestDataWithSortColumns(List(""))
     }
-    
+
     assert(!(new File(writerPath).exists()))
     cleanTestData()
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index 9af710f..be809e6 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -74,6 +74,8 @@ public class CarbonReader<T> {
         return false;
       } else {
         index++;
+        // current reader is closed
+        currentReader.close();
         currentReader = readers.get(index);
         return currentReader.nextKeyValue();
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index bd64568..0f670fe 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -484,11 +484,6 @@ public class CarbonWriterBuilder {
           if (isSortColumn > -1) {
             columnSchema.setSortColumn(true);
             sortColumnsSchemaList[isSortColumn] = columnSchema;
-          } else if (!sortColumnsList.isEmpty() && columnSchema.isDimensionColumn()
-              && columnSchema.getNumberOfChild() < 1) {
-            columnSchema.setSortColumn(true);
-            sortColumnsSchemaList[i] = columnSchema;
-            i++;
           }
         }
       }


Mime
View raw message