carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [21/23] carbondata git commit: [CARBONDATA-1488] JVM crashes when unsafe columnpage is enabled
Date Thu, 21 Sep 2017 00:27:31 GMT
[CARBONDATA-1488] JVM crashes when unsafe columnpage is enabled

Fixed capacity size calculation when column value size is larger than initial capacity in
this PR.

This closes #1370


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e9c24c50
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e9c24c50
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e9c24c50

Branch: refs/heads/streaming_ingest
Commit: e9c24c50536e7700fe4ce468114820cb21942ee3
Parents: 36ceb59
Author: Ravindra Pesala <ravi.pesala@gmail.com>
Authored: Tue Sep 19 16:24:35 2017 +0530
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Tue Sep 19 20:13:32 2017 +0800

----------------------------------------------------------------------
 .../core/datastore/page/UnsafeVarLengthColumnPage.java  | 12 ------------
 .../core/datastore/page/VarLengthColumnPageBase.java    |  2 +-
 .../vectorreader/VectorizedCarbonRecordReader.java      |  4 ++--
 3 files changed, 3 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9c24c50/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
index c9737a4..9d6e161 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
@@ -42,18 +42,6 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase
{
     baseOffset = memoryBlock.getBaseOffset();
   }
 
-  /**
-   * create a page with initial capacity
-   */
-  UnsafeVarLengthColumnPage(TableSpec.ColumnSpec columnSpec, DataType dataType, int pageSize,
-      int capacity) throws MemoryException {
-    super(columnSpec, dataType, pageSize);
-    this.capacity = capacity;
-    memoryBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, (long)(capacity));
-    baseAddress = memoryBlock.getBaseObject();
-    baseOffset = memoryBlock.getBaseOffset();
-  }
-
   @Override
   public void freeMemory() {
     if (memoryBlock != null) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9c24c50/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
index 83e6ef0..276b7ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
@@ -363,7 +363,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
    */
   protected void ensureMemory(int requestSize) throws MemoryException {
     if (totalLength + requestSize > capacity) {
-      int newSize = 2 * capacity;
+      int newSize = Math.max(2 * capacity, totalLength + requestSize);
       MemoryBlock newBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, newSize);
       CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset,
           newBlock.getBaseObject(), newBlock.getBaseOffset(), capacity);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9c24c50/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 5356ad5..e1946a9 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -72,9 +72,9 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object>
{
   private boolean returnColumnarBatch;
 
   /**
-   * The default config on whether columnarBatch should be offheap.
+   * The default config on whether columnarBatch should be onheap.
    */
-  private static final MemoryMode DEFAULT_MEMORY_MODE = MemoryMode.OFF_HEAP;
+  private static final MemoryMode DEFAULT_MEMORY_MODE = MemoryMode.ON_HEAP;
 
   private QueryModel queryModel;
 


Mime
View raw message