carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [1/2] incubator-carbondata git commit: Fixed memory leak in offhep during query
Date Fri, 17 Mar 2017 05:01:12 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master f75766119 -> f963ea419


Fixed memory leak in offhep during query


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/1f3efe33
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/1f3efe33
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/1f3efe33

Branch: refs/heads/master
Commit: 1f3efe339221f59ad3513eae6fcdb73115c1e6bc
Parents: f757661
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Thu Mar 16 22:35:06 2017 +0530
Committer: kumarvishal <kumarvishal.1802@gmail.com>
Committed: Thu Mar 16 22:44:18 2017 +0530

----------------------------------------------------------------------
 .../processor/AbstractDataBlockIterator.java    | 29 +++++++++++--
 .../processor/impl/DataBlockIteratorImpl.java   |  4 +-
 .../core/scan/result/AbstractScannedResult.java | 43 ++++++++++++++++++--
 .../AbstractDetailQueryResultIterator.java      | 23 +++++------
 .../scan/scanner/AbstractBlockletScanner.java   | 13 ++++++
 .../core/scan/scanner/impl/FilterScanner.java   | 19 +++++++--
 .../core/stats/QueryStatisticsConstants.java    |  7 +++-
 .../core/stats/QueryStatisticsRecorderImpl.java | 18 ++++++--
 .../carbondata/core/util/BitSetGroup.java       | 10 +++++
 .../writer/v3/CarbonFactDataWriterImplV3.java   |  1 +
 10 files changed, 135 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
index e4314da..4d01a05 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.scan.processor;
 import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -84,9 +85,8 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
 
   private AtomicBoolean nextRead;
 
-  public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo,
-      FileHolder fileReader, int batchSize, QueryStatisticsModel queryStatisticsModel,
-      ExecutorService executorService) {
+  public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
+      int batchSize, QueryStatisticsModel queryStatisticsModel, ExecutorService executorService)
{
     this.blockExecutionInfo = blockExecutionInfo;
     this.fileReader = fileReader;
     dataBlockIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
@@ -113,6 +113,10 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
   }
 
   protected boolean updateScanner() {
+    // clear the current result if all the records are processed
+    if (scannedResult != null && !scannedResult.hasNext()) {
+      scannedResult.freeMemory();
+    }
     try {
       if (scannedResult != null && scannedResult.hasNext()) {
         return true;
@@ -205,4 +209,23 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
   }
 
   public abstract void processNextBatch(CarbonColumnarBatch columnarBatch);
+
+  /**
+   * Close the resources
+   */
+  public void close() {
+    // free the current scanned result
+    if (null != scannedResult && !scannedResult.hasNext()) {
+      scannedResult.freeMemory();
+    }
+    // free any pre-fetched memory if present
+    if (null != future) {
+      try {
+        AbstractScannedResult abstractScannedResult = future.get();
+        abstractScannedResult.freeMemory();
+      } catch (InterruptedException | ExecutionException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
index 5eab7b6..1c97725 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
@@ -23,7 +23,6 @@ import java.util.concurrent.ExecutorService;
 import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
 import org.apache.carbondata.core.stats.QueryStatisticsModel;
 
@@ -37,8 +36,7 @@ public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
    * @param blockExecutionInfo execution information
    */
   public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
-      int batchSize, QueryStatisticsModel queryStatisticsModel,
-      BlocksChunkHolder blockChunkHolder, ExecutorService executorService) {
+      int batchSize, QueryStatisticsModel queryStatisticsModel, ExecutorService executorService)
{
     super(blockExecutionInfo, fileReader, batchSize, queryStatisticsModel, executorService);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
index fbd7044..eec9077 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
@@ -289,6 +289,7 @@ public abstract class AbstractScannedResult {
 
   /**
    * Get total rows in the current page
+   *
    * @return
    */
   public int getCurrentPageRowCount() {
@@ -380,7 +381,6 @@ public abstract class AbstractScannedResult {
     this.rowId = rowId;
   }
 
-
   /**
    * Below method will be used to get the complex type keys array based
    * on row id for all the complex type dimension selected in query
@@ -435,6 +435,44 @@ public abstract class AbstractScannedResult {
   }
 
   /**
+   * Below method will be used to free the occupied memory
+   */
+  public void freeMemory() {
+    // first free the dimension chunks
+    if (null != dataChunks) {
+      for (int i = 0; i < dataChunks.length; i++) {
+        if (null != dataChunks[i]) {
+          for (int j = 0; j < dataChunks[i].length; j++) {
+            if (null != dataChunks[i][j]) {
+              dataChunks[i][j].freeMemory();
+            }
+          }
+        }
+      }
+    }
+    // free the measure data chunks
+    if (null != measureDataChunks) {
+      for (int i = 0; i < measureDataChunks.length; i++) {
+        if (null != measureDataChunks[i]) {
+          for (int j = 0; j < measureDataChunks[i].length; j++) {
+            if (null != measureDataChunks[i][j]) {
+              measureDataChunks[i][j].freeMemory();
+            }
+          }
+        }
+      }
+    }
+    // free the raw chunks
+    if (null != rawColumnChunks) {
+      for (int i = 0; i < rawColumnChunks.length; i++) {
+        if (null != rawColumnChunks[i]) {
+          rawColumnChunks[i].freeMemory();
+        }
+      }
+    }
+  }
+
+  /**
    * As this class will be a flyweight object so
    * for one block all the blocklet scanning will use same result object
    * in that case we need to reset the counter to zero so
@@ -452,7 +490,7 @@ public abstract class AbstractScannedResult {
   public void setNumberOfRows(int[] numberOfRows) {
     this.numberOfRows = numberOfRows;
 
-    for (int count: numberOfRows) {
+    for (int count : numberOfRows) {
       totalNumberOfRows += count;
     }
   }
@@ -563,7 +601,6 @@ public abstract class AbstractScannedResult {
   public abstract String[] getNoDictionaryKeyStringArray();
 
   /**
-   *
    * @return BlockletLevelDeleteDeltaDataCache.
    */
   public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 1176e5a..1e308e2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -32,7 +32,6 @@ import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
 import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.core.scan.model.QueryModel;
 import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.core.scan.processor.impl.DataBlockIteratorImpl;
 import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
 import org.apache.carbondata.core.stats.QueryStatistic;
@@ -40,7 +39,6 @@ import org.apache.carbondata.core.stats.QueryStatisticsConstants;
 import org.apache.carbondata.core.stats.QueryStatisticsModel;
 import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
 import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
 
 /**
  * In case of detail query we cannot keep all the records in memory so for
@@ -88,8 +86,6 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
    */
   QueryStatisticsModel queryStatisticsModel;
 
-  private BlocksChunkHolder blocksChunkHolder;
-
   public AbstractDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel
queryModel,
       ExecutorService execService) {
     String batchSizeString =
@@ -104,13 +100,10 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
     } else {
       batchSize = CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT;
     }
-    this.blocksChunkHolder = new BlocksChunkHolder(infos.get(0).getTotalNumberDimensionBlock(),
-        infos.get(0).getTotalNumberOfMeasureBlock());
     this.recorder = queryModel.getStatisticsRecorder();
     this.blockExecutionInfos = infos;
     this.fileReader = FileFactory.getFileHolder(
         FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getStorePath()));
-    this.blocksChunkHolder.setFileReader(fileReader);
     this.execService = execService;
     intialiseInfos();
     initQueryStatiticsModel();
@@ -169,10 +162,8 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
       BlockExecutionInfo executionInfo = blockExecutionInfos.get(0);
       blockExecutionInfos.remove(executionInfo);
       queryStatisticsModel.setRecorder(recorder);
-      CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(),
-          blocksChunkHolder.getMeasureRawDataChunk());
       return new DataBlockIteratorImpl(executionInfo, fileReader, batchSize, queryStatisticsModel,
-          blocksChunkHolder, execService);
+          execService);
     }
     return null;
   }
@@ -185,6 +176,12 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
     QueryStatistic queryStatisticValidScanBlocklet = new QueryStatistic();
     queryStatisticsModel.getStatisticsTypeAndObjMap()
         .put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatisticValidScanBlocklet);
+    QueryStatistic totalNumberOfPages = new QueryStatistic();
+    queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .put(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalNumberOfPages);
+    QueryStatistic validPages = new QueryStatistic();
+    queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .put(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages);
   }
 
   public void processNextBatch(CarbonColumnarBatch columnarBatch) {
@@ -196,9 +193,9 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
       fileReader.finish();
     } catch (IOException e) {
       LOGGER.error(e);
-    } finally {
-      CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(),
-          blocksChunkHolder.getMeasureRawDataChunk());
+    }
+    if (null != dataBlockIterator) {
+      dataBlockIterator.close();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
index 341fb21..8980fa7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
@@ -66,6 +66,19 @@ public abstract class AbstractBlockletScanner implements BlockletScanner
{
         .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
             validScannedBlockletStatistic.getCount() + 1);
     queryStatisticsModel.getRecorder().recordStatistics(validScannedBlockletStatistic);
+    // adding statistics for valid number of pages
+    QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
+    validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
+        validPages.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
+    queryStatisticsModel.getRecorder().recordStatistics(validPages);
+    // adding statistics for number of pages
+    QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
+    totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
+        totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
+    queryStatisticsModel.getRecorder().recordStatistics(totalPagesScanned);
+
     scannedResult.setBlockletId(
         blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
             .getDataBlock().nodeNumber());

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index c3d86aa..915ed37 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -93,6 +93,12 @@ public class FilterScanner extends AbstractBlockletScanner {
   }
 
   @Override public boolean isScanRequired(BlocksChunkHolder blocksChunkHolder) throws IOException
{
+    // adding statistics for number of pages
+    QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
+    totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
+        totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
+    queryStatisticsModel.getRecorder().recordStatistics(totalPagesScanned);
     // apply min max
     if (isMinMaxEnabled) {
       BitSet bitSet = this.filterExecuter
@@ -149,6 +155,13 @@ public class FilterScanner extends AbstractBlockletScanner {
         .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
             validScannedBlockletStatistic.getCount() + 1);
     queryStatisticsModel.getRecorder().recordStatistics(validScannedBlockletStatistic);
+    // adding statistics for valid number of pages
+    QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
+    validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
+        validPages.getCount() + bitSetGroup.getValidPages());
+    queryStatisticsModel.getRecorder().recordStatistics(validPages);
+
     int[] rowCount = new int[bitSetGroup.getNumberOfPages()];
     // get the row indexes from bot set
     int[][] indexesGroup = new int[bitSetGroup.getNumberOfPages()][];
@@ -241,16 +254,14 @@ public class FilterScanner extends AbstractBlockletScanner {
     for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
       for (int j = 0; j < indexesGroup.length; j++) {
         if (dimensionRawColumnChunks[i] != null) {
-          dimensionColumnDataChunks[i][j] =
-              dimensionRawColumnChunks[i].convertToDimColDataChunk(j);
+          dimensionColumnDataChunks[i][j] = dimensionRawColumnChunks[i].convertToDimColDataChunk(j);
         }
       }
     }
     for (int i = 0; i < measureRawColumnChunks.length; i++) {
       for (int j = 0; j < indexesGroup.length; j++) {
         if (measureRawColumnChunks[i] != null) {
-          measureColumnDataChunks[i][j] =
-              measureRawColumnChunks[i].convertToMeasureColDataChunk(j);
+          measureColumnDataChunks[i][j] = measureRawColumnChunks[i].convertToMeasureColDataChunk(j);
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
index 1229e13..bf6b548 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
@@ -31,8 +31,7 @@ public interface QueryStatisticsConstants {
   String BLOCK_IDENTIFICATION = "Time taken to identify Block(s) to scan";
 
   // executor side
-  String EXECUTOR_PART =
-      "Total Time taken to execute the query in executor Side";
+  String EXECUTOR_PART = "Total Time taken to execute the query in executor Side";
 
   String LOAD_BLOCKS_EXECUTOR = "Time taken to load the Block(s) In Executor";
 
@@ -50,6 +49,10 @@ public interface QueryStatisticsConstants {
 
   String VALID_SCAN_BLOCKLET_NUM = "The num of valid scanned blocklet";
 
+  String VALID_PAGE_SCANNED = "The number of valid page scanned";
+
+  String TOTAL_PAGE_SCANNED = "The number of total page scanned";
+
   // clear no-use statistics timeout
   long CLEAR_STATISTICS_TIMEOUT = 60 * 1000 * 1000000L;
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
index aa1eb3b..177a8a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
@@ -28,7 +28,7 @@ import static org.apache.carbondata.core.util.CarbonUtil.printLine;
 /**
  * Class will be used to record and log the query statistics
  */
-public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder,Serializable
{
+public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder, Serializable
{
 
   private static final LogService LOGGER =
       LogServiceFactory.getLogService(QueryStatisticsRecorderImpl.class.getName());
@@ -96,6 +96,8 @@ public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder,Seri
     String splitChar = " ";
     String total_blocklet = "";
     String valid_scan_blocklet = "";
+    String valid_pages_blocklet = "";
+    String total_pages = "";
     try {
       for (QueryStatistic statistic : queryStatistics) {
         switch (statistic.getMessage()) {
@@ -123,13 +125,19 @@ public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder,Seri
           case QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM:
             valid_scan_blocklet = statistic.getCount() + splitChar;
             break;
+          case QueryStatisticsConstants.VALID_PAGE_SCANNED:
+            valid_pages_blocklet = statistic.getCount() + splitChar;
+            break;
+          case QueryStatisticsConstants.TOTAL_PAGE_SCANNED:
+            total_pages = statistic.getCount() + splitChar;
+            break;
           default:
             break;
         }
       }
-      String headers = "task_id,load_blocks_time,load_dictionary_time,scan_blocks_time,"
+
-          "total_executor_time,scan_blocks_num,total_blocklet," +
-          "valid_scan_blocklet,result_size";
+      String headers = "task_id,load_blocks_time,load_dictionary_time,scan_blocks_time,"
+          + "total_executor_time,scan_blocks_num,total_blocklet,"
+          + "valid_scan_blocklet,total_pages,valid_pages,result_size";
       List<String> values = new ArrayList<String>();
       values.add(queryIWthTask);
       values.add(load_blocks_time);
@@ -139,6 +147,8 @@ public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder,Seri
       values.add(scan_blocks_num);
       values.add(total_blocklet);
       values.add(valid_scan_blocklet);
+      values.add(total_pages);
+      values.add(valid_pages_blocklet);
       values.add(result_size);
       StringBuilder tableInfo = new StringBuilder();
       String[] columns = headers.split(",");

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/core/src/main/java/org/apache/carbondata/core/util/BitSetGroup.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/BitSetGroup.java b/core/src/main/java/org/apache/carbondata/core/util/BitSetGroup.java
index 323042a..87cbe77 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/BitSetGroup.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/BitSetGroup.java
@@ -81,4 +81,14 @@ public class BitSetGroup {
     return bitSets.length;
   }
 
+  /**
+   * @return return the valid pages
+   */
+  public int getValidPages() {
+    int numberOfPages = 0;
+    for (int i = 0; i < bitSets.length; i++) {
+      numberOfPages += (bitSets[i] == null || bitSets[i].isEmpty()) ? 0 : 1;
+    }
+    return numberOfPages;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f3efe33/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index 1bc6c9d..108fef4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -315,6 +315,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
     // check the number of pages present in data holder, if pages is exceeding threshold
     // it will write the pages to file
     if (dataWriterHolder.getSize() + holder.getHolderSize() >= blockletSize) {
+      LOGGER.info("Number of Pages for blocklet is: " + dataWriterHolder.getSize());
       writeDataToFile(fileChannel);
     }
     dataWriterHolder.addNodeHolder(holder);


Mime
View raw message