carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: fixed offheap crash issue
Date Sat, 14 Jan 2017 04:24:07 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 6d29fa2f0 -> b58146531


fixed offheap crash issue


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a45ace20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a45ace20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a45ace20

Branch: refs/heads/master
Commit: a45ace20368f939980d64ffedd1b08774ce61a81
Parents: 6d29fa2
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Fri Jan 13 12:50:26 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Sat Jan 14 12:23:38 2017 +0800

----------------------------------------------------------------------
 .../chunk/store/MeasureChunkStoreFactory.java      |  2 +-
 .../scan/executor/impl/AbstractQueryExecutor.java  | 10 ++++++++++
 .../scan/executor/impl/DetailQueryExecutor.java    |  3 ++-
 .../executor/impl/VectorDetailQueryExecutor.java   |  3 ++-
 .../scan/processor/AbstractDataBlockIterator.java  | 10 ++++------
 .../scan/processor/impl/DataBlockIteratorImpl.java |  6 ++++--
 .../scan/result/AbstractScannedResult.java         |  1 -
 .../AbstractDetailQueryResultIterator.java         | 17 ++++++++++++++++-
 .../scan/scanner/impl/FilterScanner.java           |  2 ++
 9 files changed, 41 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a45ace20/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
index e7068b0..87ef7b5 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
@@ -62,7 +62,7 @@ public class MeasureChunkStoreFactory {
    * @return measure chunk store
    */
   public MeasureDataChunkStore getMeasureDataChunkStore(DataType dataType, int numberOfRows)
{
-    if (isUnsafe) {
+    if (!isUnsafe) {
       switch (dataType) {
         case DATA_BYTE:
           return new SafeByteMeasureChunkStore(numberOfRows);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a45ace20/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
index 9976db3..8c9daaf 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
@@ -28,6 +28,7 @@ import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.carbondata.common.CarbonIterator;
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.common.logging.impl.StandardLogService;
@@ -80,6 +81,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E>
{
    */
   protected QueryExecutorProperties queryProperties;
 
+  /**
+   * query result iterator which will execute the query
+   * and give the result
+   */
+  protected CarbonIterator queryIterator;
+
   public AbstractQueryExecutor() {
     queryProperties = new QueryExecutorProperties();
   }
@@ -470,6 +477,9 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E>
{
    */
   @Override public void finish() throws QueryExecutionException {
     CarbonUtil.clearBlockCache(queryProperties.dataBlocks);
+    if(null != queryIterator) {
+      queryIterator.close();
+    }
     if (null != queryProperties.executorService) {
       queryProperties.executorService.shutdown();
       try {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a45ace20/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
index 92e7fc8..6a34e68 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
@@ -39,11 +39,12 @@ public class DetailQueryExecutor extends AbstractQueryExecutor<BatchResult>
{
   public CarbonIterator<BatchResult> execute(QueryModel queryModel)
       throws QueryExecutionException, IOException {
     List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
-    return new DetailQueryResultIterator(
+    this.queryIterator = new DetailQueryResultIterator(
         blockExecutionInfoList,
         queryModel,
         queryProperties.executorService
     );
+    return queryIterator;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a45ace20/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
index 1f1dd7d..04e60fb 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
@@ -36,11 +36,12 @@ public class VectorDetailQueryExecutor extends AbstractQueryExecutor<Object>
{
   public CarbonIterator<Object> execute(QueryModel queryModel)
       throws QueryExecutionException, IOException {
     List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
-    return new VectorDetailQueryResultIterator(
+    this.queryIterator = new VectorDetailQueryResultIterator(
         blockExecutionInfoList,
         queryModel,
         queryProperties.executorService
     );
+    return this.queryIterator;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a45ace20/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
b/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
index f996c16..9a4fae2 100644
--- a/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
@@ -76,14 +76,12 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
 
   protected AbstractScannedResult scannedResult;
 
-  public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo,
-      FileHolder fileReader, int batchSize, QueryStatisticsModel queryStatisticsModel) {
+  public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
+      int batchSize, QueryStatisticsModel queryStatisticsModel,
+      BlocksChunkHolder blockChunkHolder) {
     dataBlockIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
         blockExecutionInfo.getNumberOfBlockToScan());
-    blocksChunkHolder = new BlocksChunkHolder(blockExecutionInfo.getTotalNumberDimensionBlock(),
-        blockExecutionInfo.getTotalNumberOfMeasureBlock());
-    blocksChunkHolder.setFileReader(fileReader);
-
+    blocksChunkHolder = blockChunkHolder;
     if (blockExecutionInfo.getFilterExecuterTree() != null) {
       blockletScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel);
     } else {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a45ace20/core/src/main/java/org/apache/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
b/core/src/main/java/org/apache/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
index 2a8bdc4..f32f011 100644
--- a/core/src/main/java/org/apache/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsModel;
 import org.apache.carbondata.core.datastorage.store.FileHolder;
 import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.scan.processor.AbstractDataBlockIterator;
+import org.apache.carbondata.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
 
 /**
@@ -37,8 +38,9 @@ public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
    * @param blockExecutionInfo execution information
    */
   public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
-      int batchSize, QueryStatisticsModel queryStatisticsModel) {
-    super(blockExecutionInfo, fileReader, batchSize, queryStatisticsModel);
+      int batchSize, QueryStatisticsModel queryStatisticsModel,
+      BlocksChunkHolder blockChunkHolder) {
+    super(blockExecutionInfo, fileReader, batchSize, queryStatisticsModel, blockChunkHolder);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a45ace20/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
b/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
index 0b0fd88..9ea42b3 100644
--- a/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
@@ -383,7 +383,6 @@ public abstract class AbstractScannedResult {
     if (rowCounter < this.totalNumberOfRows) {
       return true;
     }
-    CarbonUtil.freeMemory(dataChunks, measureDataChunks);
     return false;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a45ace20/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
b/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 5a0d3c1..1b2c766 100644
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -35,9 +35,11 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastorage.store.FileHolder;
 import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
 import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.scan.model.QueryModel;
 import org.apache.carbondata.scan.processor.AbstractDataBlockIterator;
+import org.apache.carbondata.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.scan.processor.impl.DataBlockIteratorImpl;
 import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
 
@@ -87,6 +89,8 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
    */
   QueryStatisticsModel queryStatisticsModel;
 
+  private BlocksChunkHolder blocksChunkHolder;
+
   public AbstractDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel
queryModel,
       ExecutorService execService) {
     String batchSizeString =
@@ -101,10 +105,13 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
     } else {
       batchSize = CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT;
     }
+    this.blocksChunkHolder = new BlocksChunkHolder(infos.get(0).getTotalNumberDimensionBlock(),
+        infos.get(0).getTotalNumberOfMeasureBlock());
     this.recorder = queryModel.getStatisticsRecorder();
     this.blockExecutionInfos = infos;
     this.fileReader = FileFactory.getFileHolder(
         FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getStorePath()));
+    this.blocksChunkHolder.setFileReader(fileReader);
     this.execService = execService;
     intialiseInfos();
     initQueryStatiticsModel();
@@ -163,7 +170,10 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
       BlockExecutionInfo executionInfo = blockExecutionInfos.get(0);
       blockExecutionInfos.remove(executionInfo);
       queryStatisticsModel.setRecorder(recorder);
-      return new DataBlockIteratorImpl(executionInfo, fileReader, batchSize, queryStatisticsModel);
+      CarbonUtil.freeMemory(blocksChunkHolder.getDimensionDataChunk(),
+          blocksChunkHolder.getMeasureDataChunk());
+      return new DataBlockIteratorImpl(executionInfo, fileReader, batchSize, queryStatisticsModel,
+          blocksChunkHolder);
     }
     return null;
   }
@@ -182,4 +192,9 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
     throw new UnsupportedOperationException("Please use VectorDetailQueryResultIterator");
   }
 
+  @Override public void close() {
+    CarbonUtil.freeMemory(blocksChunkHolder.getDimensionDataChunk(),
+        blocksChunkHolder.getMeasureDataChunk());
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a45ace20/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
index c0364ee..2b2b8bf 100644
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
@@ -134,6 +134,8 @@ public class FilterScanner extends AbstractBlockletScanner {
     if (bitSet.isEmpty()) {
       scannedResult.setNumberOfRows(0);
       scannedResult.setIndexes(new int[0]);
+      CarbonUtil.freeMemory(blocksChunkHolder.getDimensionDataChunk(),
+          blocksChunkHolder.getMeasureDataChunk());
       return;
     }
     // valid scanned blocklet


Mime
View raw message