carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gvram...@apache.org
Subject carbondata git commit: [CARBONDATA-2735] Fixed Performance issue for complex array data type when number of elements in array is more
Date Thu, 19 Jul 2018 06:10:31 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 63b930c89 -> 892594743


[CARBONDATA-2735] Fixed Performance issue for complex array data type when number of elements
in array is more

GC issue due to repeated arraylist resize, is fixed

This closes #2487


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/89259474
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/89259474
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/89259474

Branch: refs/heads/master
Commit: 89259474374eda6ba8754b213eb401b70c093f3e
Parents: 63b930c
Author: kumarvishal09 <kumarvishal1802@gmail.com>
Authored: Wed Jul 11 15:24:08 2018 +0530
Committer: Venkata Ramana G <ramana.gollamudi@huawei.com>
Committed: Thu Jul 19 11:37:00 2018 +0530

----------------------------------------------------------------------
 .../core/datastore/page/ColumnPage.java         |   6 +-
 .../datastore/page/SafeDecimalColumnPage.java   |   1 +
 .../datastore/page/SafeFixLengthColumnPage.java |   6 +-
 .../datastore/page/SafeVarLengthColumnPage.java |   1 +
 .../datastore/page/UnsafeDecimalColumnPage.java |  13 +--
 .../page/UnsafeFixLengthColumnPage.java         |  27 ++++-
 .../page/UnsafeVarLengthColumnPage.java         |  17 +--
 .../datastore/page/VarLengthColumnPageBase.java | 106 +++++++++++--------
 8 files changed, 113 insertions(+), 64 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 8745545..ea250cf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -49,7 +49,7 @@ import static org.apache.carbondata.core.metadata.datatype.DataTypes.SHORT_INT;
 public abstract class ColumnPage {
 
   // number of row in this page
-  protected final int pageSize;
+  protected int pageSize;
 
   // data type of the page storage
   protected final DataType dataType;
@@ -832,4 +832,8 @@ public abstract class ColumnPage {
   public PageLevelDictionary getColumnPageDictionary() {
     throw new UnsupportedOperationException("Operation not supported");
   }
+
+  public int getActualRowCount() {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
index 0e0ba85..c220fa4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
@@ -218,5 +218,6 @@ public class SafeDecimalColumnPage extends DecimalColumnPage {
   @Override
   public void freeMemory() {
     byteArrayData = null;
+    super.freeMemory();
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
index 8f0d934..74bb3fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
@@ -388,7 +388,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
     }
   }
 
-  protected void ensureArraySize(int requestSize, DataType dataType) {
+  private void ensureArraySize(int requestSize, DataType dataType) {
     if (dataType == DataTypes.BYTE) {
       if (requestSize >= byteData.length) {
         byte[] newArray = new byte[arrayElementCount + 16];
@@ -436,4 +436,8 @@ public class SafeFixLengthColumnPage extends ColumnPage {
           "not support value conversion on " + dataType + " page");
     }
   }
+  public int getActualRowCount() {
+    return arrayElementCount;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
index c2eb40c..274b8a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
@@ -40,6 +40,7 @@ public class SafeVarLengthColumnPage extends VarLengthColumnPageBase {
   @Override
   public void freeMemory() {
     byteArrayData = null;
+    super.freeMemory();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java
index 7449da6..e4a949a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java
@@ -119,6 +119,7 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
       memoryBlock = null;
       baseAddress = null;
       baseOffset = 0;
+      super.freeMemory();
     }
   }
 
@@ -168,7 +169,7 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
       throw new RuntimeException(e);
     }
     CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, baseAddress,
-        baseOffset + rowOffset.get(rowId), length);
+        baseOffset + rowOffset.getInt(rowId), length);
   }
 
   @Override
@@ -193,9 +194,9 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
 
   @Override
   public byte[] getBytes(int rowId) {
-    int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+    int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
     byte[] bytes = new byte[length];
-    CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId),
+    CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
         bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
     return bytes;
   }
@@ -242,9 +243,9 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
     } else if (dataType == DataTypes.LONG) {
       value = getLong(rowId);
     } else {
-      int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+      int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
       byte[] bytes = new byte[length];
-      CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId),
bytes,
+      CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
bytes,
           CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
       return decimalConverter.getDecimal(bytes);
     }
@@ -253,7 +254,7 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
 
   @Override
   void copyBytes(int rowId, byte[] dest, int destOffset, int length) {
-    CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId), dest,
+    CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
dest,
         CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length);
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index a4cea5d..c5805d5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -110,6 +110,12 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
     }
   }
 
+  private void updatePageSize(int rowId) {
+    if (pageSize < rowId) {
+      pageSize = rowId;
+    }
+  }
+
   @Override
   public void putByte(int rowId, byte value) {
     try {
@@ -120,6 +126,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
     long offset = ((long)rowId) << byteBits;
     CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset, value);
     totalLength += ByteUtil.SIZEOF_BYTE;
+    updatePageSize(rowId);
   }
 
 
@@ -134,6 +141,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
     long offset = ((long)rowId) << shortBits;
     CarbonUnsafe.getUnsafe().putShort(baseAddress, baseOffset + offset, value);
     totalLength += ByteUtil.SIZEOF_SHORT;
+    updatePageSize(rowId);
   }
 
   @Override
@@ -149,6 +157,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
     CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 1, data[1]);
     CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 2, data[2]);
     totalLength += ByteUtil.SIZEOF_SHORT_INT;
+    updatePageSize(rowId);
   }
 
   @Override
@@ -161,6 +170,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
     long offset = ((long)rowId) << intBits;
     CarbonUnsafe.getUnsafe().putInt(baseAddress, baseOffset + offset, value);
     totalLength += ByteUtil.SIZEOF_INT;
+    updatePageSize(rowId);
   }
 
   @Override
@@ -173,6 +183,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
     long offset = ((long)rowId) << longBits;
     CarbonUnsafe.getUnsafe().putLong(baseAddress, baseOffset + offset, value);
     totalLength += ByteUtil.SIZEOF_LONG;
+    updatePageSize(rowId);
   }
 
   @Override
@@ -185,6 +196,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
     long offset = ((long)rowId) << doubleBits;
     CarbonUnsafe.getUnsafe().putDouble(baseAddress, baseOffset + offset, value);
     totalLength += ByteUtil.SIZEOF_DOUBLE;
+    updatePageSize(rowId);
   }
 
   @Override
@@ -194,6 +206,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
     CarbonUnsafe.getUnsafe()
         .copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
             baseOffset + offset, bytes.length);
+    updatePageSize(rowId);
   }
 
   @Override
@@ -274,7 +287,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
 
   @Override
   public byte[] getBytePage() {
-    byte[] data = new byte[getPageSize()];
+    byte[] data = new byte[getEndLoop()];
     for (long i = 0; i < data.length; i++) {
       long offset = i << byteBits;
       data[(int)i] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset);
@@ -284,7 +297,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
 
   @Override
   public short[] getShortPage() {
-    short[] data = new short[getPageSize()];
+    short[] data = new short[getEndLoop()];
     for (long i = 0; i < data.length; i++) {
       long offset = i << shortBits;
       data[(int)i] = CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset);
@@ -294,7 +307,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
 
   @Override
   public byte[] getShortIntPage() {
-    byte[] data = new byte[pageSize * 3];
+    byte[] data = new byte[getEndLoop() * 3];
     CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset,
         data, CarbonUnsafe.BYTE_ARRAY_OFFSET, data.length);
     return data;
@@ -302,7 +315,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
 
   @Override
   public int[] getIntPage() {
-    int[] data = new int[getPageSize()];
+    int[] data = new int[getEndLoop()];
     for (long i = 0; i < data.length; i++) {
       long offset = i << intBits;
       data[(int)i] = CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset);
@@ -312,7 +325,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
 
   @Override
   public long[] getLongPage() {
-    long[] data = new long[getPageSize()];
+    long[] data = new long[getEndLoop()];
     for (long i = 0; i < data.length; i++) {
       long offset = i << longBits;
       data[(int)i] = CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset);
@@ -514,4 +527,8 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
       capacity = newSize;
     }
   }
+
+  public int getActualRowCount() {
+    return getEndLoop();
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
index f60e505..ae57dcd 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
@@ -49,6 +49,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
       memoryBlock = null;
       baseAddress = null;
       baseOffset = 0;
+      super.freeMemory();
     }
   }
 
@@ -65,7 +66,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
       throw new RuntimeException(e);
     }
     CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset,
-        baseAddress, baseOffset + rowOffset.get(rowId), length);
+        baseAddress, baseOffset + rowOffset.getInt(rowId), length);
   }
 
   @Override
@@ -89,20 +90,20 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase
{
 
   @Override
   public byte[] getBytes(int rowId) {
-    int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+    int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
     byte[] bytes = new byte[length];
-    CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId),
+    CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
         bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
     return bytes;
   }
 
   @Override
   public byte[][] getByteArrayPage() {
-    byte[][] bytes = new byte[rowOffset.size() - 1][];
-    for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) {
-      int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+    byte[][] bytes = new byte[rowOffset.getActualRowCount() - 1][];
+    for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) {
+      int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
       byte[] rowData = new byte[length];
-      CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId),
+      CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
           rowData, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
       bytes[rowId] = rowData;
     }
@@ -111,7 +112,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase
{
 
   @Override
   void copyBytes(int rowId, byte[] dest, int destOffset, int length) {
-    CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId),
+    CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
         dest, CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length);
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
index fa163bc..55877c3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
@@ -18,10 +18,10 @@
 package org.apache.carbondata.core.datastore.page;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.datastore.ColumnType;
 import org.apache.carbondata.core.datastore.TableSpec;
 import org.apache.carbondata.core.memory.CarbonUnsafe;
 import org.apache.carbondata.core.memory.MemoryBlock;
@@ -53,7 +53,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
   Object baseAddress;
 
   // the offset of row in the unsafe memory, its size is pageSize + 1
-  List<Integer> rowOffset;
+  ColumnPage rowOffset;
 
   // the length of bytes added in the page
   int totalLength;
@@ -66,7 +66,14 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
 
   VarLengthColumnPageBase(TableSpec.ColumnSpec columnSpec, DataType dataType, int pageSize)
{
     super(columnSpec, dataType, pageSize);
-    rowOffset = new ArrayList<>();
+    TableSpec.ColumnSpec spec = TableSpec.ColumnSpec
+        .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE);
+    try {
+      rowOffset =
+          ColumnPage.newPage(spec, DataTypes.INT, pageSize);
+    } catch (MemoryException e) {
+      throw new RuntimeException(e);
+    }
     totalLength = 0;
   }
 
@@ -142,14 +149,19 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
 
   private static ColumnPage getDecimalColumnPage(TableSpec.ColumnSpec columnSpec,
       byte[] lvEncodedBytes, int size) throws MemoryException {
-    List<Integer> rowOffset = new ArrayList<>();
+    TableSpec.ColumnSpec spec = TableSpec.ColumnSpec
+        .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE);
+    ColumnPage rowOffset = ColumnPage.newPage(spec, DataTypes.INT,
+        CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
     int offset;
     int rowId = 0;
+    int counter = 0;
     for (offset = 0; offset < lvEncodedBytes.length; offset += size) {
-      rowOffset.add(offset);
+      rowOffset.putInt(counter, offset);
       rowId++;
+      counter++;
     }
-    rowOffset.add(offset);
+    rowOffset.putInt(counter, offset);
 
     VarLengthColumnPageBase page;
     if (unsafe) {
@@ -160,10 +172,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
 
     // set total length and rowOffset in page
     page.totalLength = offset;
-    page.rowOffset = new ArrayList<>();
-    for (int i = 0; i < rowOffset.size(); i++) {
-      page.rowOffset.add(rowOffset.get(i));
-    }
+    page.rowOffset = rowOffset;
     for (int i = 0; i < rowId; i++) {
       page.putBytes(i, lvEncodedBytes, i * size, size);
     }
@@ -175,24 +184,26 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
       throws MemoryException {
     // extract length and data, set them to rowOffset and unsafe memory correspondingly
     int rowId = 0;
-    List<Integer> rowOffset = new ArrayList<>();
-    List<Integer> rowLength = new ArrayList<>();
+    TableSpec.ColumnSpec spec = TableSpec.ColumnSpec
+        .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE);
+    ColumnPage rowOffset = ColumnPage.newPage(spec, DataTypes.INT,
+        CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
     int length;
     int offset;
     int lvEncodedOffset = 0;
-
+    int counter = 0;
     // extract Length field in input and calculate total length
     for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) {
       length = ByteUtil.toInt(lvEncodedBytes, lvEncodedOffset);
-      rowOffset.add(offset);
-      rowLength.add(length);
+      rowOffset.putInt(counter, offset);
       lvEncodedOffset += lvLength + length;
       rowId++;
+      counter++;
     }
-    rowOffset.add(offset);
+    rowOffset.putInt(counter, offset);
     VarLengthColumnPageBase page =
         getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset,
-            rowLength, offset);
+            offset);
     return page;
   }
 
@@ -201,31 +212,33 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
       throws MemoryException {
     // extract length and data, set them to rowOffset and unsafe memory correspondingly
     int rowId = 0;
-    List<Integer> rowOffset = new ArrayList<>();
-    List<Integer> rowLength = new ArrayList<>();
+    TableSpec.ColumnSpec spec = TableSpec.ColumnSpec
+        .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE);
+    ColumnPage rowOffset = ColumnPage.newPage(spec, DataTypes.INT,
+        CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
     int length;
     int offset;
     int lvEncodedOffset = 0;
-
+    int counter = 0;
     // extract Length field in input and calculate total length
     for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) {
       length = ByteUtil.toShort(lvEncodedBytes, lvEncodedOffset);
-      rowOffset.add(offset);
-      rowLength.add(length);
+      rowOffset.putInt(counter, offset);
       lvEncodedOffset += lvLength + length;
       rowId++;
+      counter++;
     }
-    rowOffset.add(offset);
+    rowOffset.putInt(counter, offset);
 
     VarLengthColumnPageBase page =
         getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset,
-            rowLength, offset);
+             offset);
     return page;
   }
 
   private static VarLengthColumnPageBase getVarLengthColumnPage(TableSpec.ColumnSpec columnSpec,
-      byte[] lvEncodedBytes, DataType dataType, int lvLength, int rowId, List<Integer>
rowOffset,
-      List<Integer> rowLength, int offset) throws MemoryException {
+      byte[] lvEncodedBytes, DataType dataType, int lvLength, int rowId, ColumnPage rowOffset,
+      int offset) throws MemoryException {
     int lvEncodedOffset;
     int length;
     int numRows = rowId;
@@ -240,15 +253,12 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
 
     // set total length and rowOffset in page
     page.totalLength = offset;
-    page.rowOffset = new ArrayList<>();
-    for (int i = 0; i < rowOffset.size(); i++) {
-      page.rowOffset.add(rowOffset.get(i));
-    }
+    page.rowOffset = rowOffset;
 
     // set data in page
     lvEncodedOffset = 0;
     for (int i = 0; i < numRows; i++) {
-      length = rowLength.get(i);
+      length = rowOffset.getInt(i + 1) - rowOffset.getInt(i);
       page.putBytes(i, lvEncodedBytes, lvEncodedOffset + lvLength, length);
       lvEncodedOffset += lvLength + length;
     }
@@ -296,9 +306,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
           + " exceed this limit at rowId " + rowId);
     }
     if (rowId == 0) {
-      rowOffset.add(0);
+      rowOffset.putInt(0, 0);
     }
-    rowOffset.add(rowOffset.get(rowId) + bytes.length);
+    rowOffset.putInt(rowId + 1, rowOffset.getInt(rowId) + bytes.length);
     putBytesAtRow(rowId, bytes);
     totalLength += bytes.length;
   }
@@ -379,7 +389,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
     int offset = 0;
     byte[] data = new byte[totalLength];
     for (int rowId = 0; rowId < pageSize; rowId++) {
-      int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+      int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
       copyBytes(rowId, data, offset, length);
       offset += length;
     }
@@ -395,9 +405,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
   public byte[] getLVFlattenedBytePage() throws IOException {
     // output LV encoded byte array
     int offset = 0;
-    byte[] data = new byte[totalLength + ((rowOffset.size() - 1) * 4)];
-    for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) {
-      int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+    byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 4)];
+    for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) {
+      int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
       ByteUtil.setInt(data, offset, length);
       copyBytes(rowId, data, offset + 4, length);
       offset += 4 + length;
@@ -408,9 +418,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
   @Override public byte[] getComplexChildrenLVFlattenedBytePage() throws IOException {
     // output LV encoded byte array
     int offset = 0;
-    byte[] data = new byte[totalLength + ((rowOffset.size() - 1) * 2)];
-    for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) {
-      short length = (short) (rowOffset.get(rowId + 1) - rowOffset.get(rowId));
+    byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 2)];
+    for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) {
+      short length = (short) (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId));
       ByteUtil.setShort(data, offset, length);
       copyBytes(rowId, data, offset + 2, length);
       offset += 2 + length;
@@ -423,8 +433,8 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
     // output LV encoded byte array
     int offset = 0;
     byte[] data = new byte[totalLength];
-    for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) {
-      int length =  (rowOffset.get(rowId + 1) - rowOffset.get(rowId));
+    for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) {
+      int length =  (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId));
       copyBytes(rowId, data, offset, length);
       offset += length;
     }
@@ -452,4 +462,14 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
       capacity = newSize;
     }
   }
+
+  /**
+   * free memory as needed
+   */
+  public void freeMemory() {
+    if (null != rowOffset) {
+      rowOffset.freeMemory();
+      rowOffset = null;
+    }
+  }
 }


Mime
View raw message