carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gvram...@apache.org
Subject [1/2] incubator-carbondata git commit: Added statistics for scan and read Time
Date Sat, 18 Mar 2017 07:39:47 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master acd5f1b48 -> 41ccba115


Added statistics for scan and read Time


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3eedfa01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3eedfa01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3eedfa01

Branch: refs/heads/master
Commit: 3eedfa015933f984d2d7c41fd7844949df612d7a
Parents: acd5f1b
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Fri Mar 17 19:31:59 2017 +0530
Committer: Venkata Ramana G <ramana.gollamudi@huawei.com>
Committed: Sat Mar 18 12:57:55 2017 +0530

----------------------------------------------------------------------
 .../AbstractDetailQueryResultIterator.java      | 23 ++++++--------------
 .../iterator/DetailQueryResultIterator.java     |  2 --
 .../scan/scanner/AbstractBlockletScanner.java   | 14 ++++++++++++
 .../core/scan/scanner/impl/FilterScanner.java   | 21 +++++++++++++++++-
 .../core/stats/QueryStatisticsConstants.java    |  4 +++-
 .../core/stats/QueryStatisticsRecorderImpl.java | 16 +++++++++-----
 .../stats/QueryStasticsRecorderImplTest.java    |  6 ++---
 7 files changed, 58 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 1e308e2..a3d2fd2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -65,14 +65,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
   protected FileHolder fileReader;
 
   protected AbstractDataBlockIterator dataBlockIterator;
-  /**
-   * total time scan the blocks
-   */
-  protected long totalScanTime;
-  /**
-   * is the statistic recorded
-   */
-  protected boolean isStatisticsRecorded;
+
   /**
    * QueryStatisticsRecorder
    */
@@ -110,7 +103,6 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
   }
 
   private void intialiseInfos() {
-    totalScanTime = System.currentTimeMillis();
     for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
       DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
       DataRefNode startDataBlock = finder
@@ -137,13 +129,6 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
     } else if (blockExecutionInfos.size() > 0) {
       return true;
     } else {
-      if (!isStatisticsRecorded) {
-        QueryStatistic statistic = new QueryStatistic();
-        statistic.addFixedTimeStatistic(QueryStatisticsConstants.SCAN_BLOCKS_TIME,
-            System.currentTimeMillis() - totalScanTime);
-        recorder.recordStatistics(statistic);
-        isStatisticsRecorded = true;
-      }
       return false;
     }
   }
@@ -182,6 +167,12 @@ public abstract class AbstractDetailQueryResultIterator<E> extends
CarbonIterato
     QueryStatistic validPages = new QueryStatistic();
     queryStatisticsModel.getStatisticsTypeAndObjMap()
         .put(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages);
+    QueryStatistic scanTime = new QueryStatistic();
+    queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .put(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, scanTime);
+    QueryStatistic readTime = new QueryStatistic();
+    queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .put(QueryStatisticsConstants.READ_BLOCKlET_TIME, readTime);
   }
 
   public void processNextBatch(CarbonColumnarBatch columnarBatch) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
index 1fa2f4c..e39b93f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
@@ -38,9 +38,7 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
   }
 
   @Override public BatchResult next() {
-    long startTime = System.currentTimeMillis();
     BatchResult batchResult = getBatchResult();
-    totalScanTime += System.currentTimeMillis() - startTime;
     return batchResult;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
index 8980fa7..e8bfc74 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
@@ -54,6 +54,7 @@ public abstract class AbstractBlockletScanner implements BlockletScanner
{
 
   @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
       throws IOException, FilterUnsupportedException {
+    long startTime = System.currentTimeMillis();
     AbstractScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
     QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
         .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
@@ -126,10 +127,17 @@ public abstract class AbstractBlockletScanner implements BlockletScanner
{
     scannedResult
         .setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
     scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
+    // adding statistics for carbon scan time
+    QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+    scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+        scanTime.getCount() + (System.currentTimeMillis() - startTime));
+    queryStatisticsModel.getRecorder().recordStatistics(scanTime);
     return scannedResult;
   }
 
   @Override public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException
{
+    long startTime = System.currentTimeMillis();
     DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDataBlock()
         .getDimensionChunks(blocksChunkHolder.getFileReader(),
             blockExecutionInfo.getAllSelectedDimensionBlocksIndexes());
@@ -138,6 +146,12 @@ public abstract class AbstractBlockletScanner implements BlockletScanner
{
         .getMeasureChunks(blocksChunkHolder.getFileReader(),
             blockExecutionInfo.getAllSelectedMeasureBlocksIndexes());
     blocksChunkHolder.setMeasureRawDataChunk(measureRawColumnChunks);
+    // adding statistics for carbon read time
+    QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+    readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+        readTime.getCount() + (System.currentTimeMillis() - startTime));
+    queryStatisticsModel.getRecorder().recordStatistics(readTime);
   }
 
   @Override public AbstractScannedResult createEmptyResult() {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index 915ed37..32b7bff 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -114,7 +114,14 @@ public class FilterScanner extends AbstractBlockletScanner {
   }
 
   @Override public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException
{
+    long startTime = System.currentTimeMillis();
     this.filterExecuter.readBlocks(blocksChunkHolder);
+    // adding statistics for carbon read time
+    QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+    readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+        readTime.getCount() + (System.currentTimeMillis() - startTime));
+    queryStatisticsModel.getRecorder().recordStatistics(readTime);
   }
 
   /**
@@ -135,6 +142,7 @@ public class FilterScanner extends AbstractBlockletScanner {
    */
   private AbstractScannedResult fillScannedResult(BlocksChunkHolder blocksChunkHolder)
       throws FilterUnsupportedException, IOException {
+    long startTime = System.currentTimeMillis();
     // apply filter on actual data
     BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder);
     // if indexes is empty then return with empty result
@@ -161,7 +169,11 @@ public class FilterScanner extends AbstractBlockletScanner {
     validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
         validPages.getCount() + bitSetGroup.getValidPages());
     queryStatisticsModel.getRecorder().recordStatistics(validPages);
-
+    QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
+    totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
+        totalBlockletStatistic.getCount() + 1);
+    queryStatisticsModel.getRecorder().recordStatistics(totalBlockletStatistic);
     int[] rowCount = new int[bitSetGroup.getNumberOfPages()];
     // get the row indexes from bot set
     int[][] indexesGroup = new int[bitSetGroup.getNumberOfPages()][];
@@ -270,6 +282,13 @@ public class FilterScanner extends AbstractBlockletScanner {
     scannedResult.setMeasureChunks(measureColumnDataChunks);
     scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
     scannedResult.setNumberOfRows(rowCount);
+    // adding statistics for carbon scan time
+    QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+        .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+    scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+        scanTime.getCount() + (System.currentTimeMillis() - startTime));
+    queryStatisticsModel.getRecorder().recordStatistics(scanTime);
+
     return scannedResult;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
index bf6b548..c8fa4a1 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
@@ -37,7 +37,9 @@ public interface QueryStatisticsConstants {
 
   String SCAN_BLOCKS_NUM = "The num of blocks scanned";
 
-  String SCAN_BLOCKS_TIME = "Time taken to scan blocks";
+  String SCAN_BLOCKlET_TIME = "Time taken to scan blocks";
+
+  String READ_BLOCKlET_TIME = "Time taken to read blocks";
 
   String LOAD_DICTIONARY = "Time taken to load the Dictionary In Executor";
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
index 177a8a7..3f048c6 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
@@ -98,14 +98,15 @@ public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder,
Ser
     String valid_scan_blocklet = "";
     String valid_pages_blocklet = "";
     String total_pages = "";
+    String readTime = "";
     try {
       for (QueryStatistic statistic : queryStatistics) {
         switch (statistic.getMessage()) {
           case QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR:
             load_blocks_time += statistic.getTimeTaken() + splitChar;
             break;
-          case QueryStatisticsConstants.SCAN_BLOCKS_TIME:
-            scan_blocks_time += statistic.getTimeTaken() + splitChar;
+          case QueryStatisticsConstants.SCAN_BLOCKlET_TIME:
+            scan_blocks_time += statistic.getCount() + splitChar;
             break;
           case QueryStatisticsConstants.SCAN_BLOCKS_NUM:
             scan_blocks_num += statistic.getCount() + splitChar;
@@ -131,18 +132,23 @@ public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder,
Ser
           case QueryStatisticsConstants.TOTAL_PAGE_SCANNED:
             total_pages = statistic.getCount() + splitChar;
             break;
+          case QueryStatisticsConstants.READ_BLOCKlET_TIME:
+            readTime = statistic.getCount() + splitChar;
+            break;
           default:
             break;
         }
       }
-      String headers = "task_id,load_blocks_time,load_dictionary_time,scan_blocks_time,"
-          + "total_executor_time,scan_blocks_num,total_blocklet,"
-          + "valid_scan_blocklet,total_pages,valid_pages,result_size";
+      String headers =
+          "task_id,load_blocks_time,load_dictionary_time,carbon_scan_time,carbon_IO_time,
"
+              + "total_executor_time,scan_blocks_num,total_blocklets,"
+              + "valid_blocklets,total_pages,valid_pages,result_size";
       List<String> values = new ArrayList<String>();
       values.add(queryIWthTask);
       values.add(load_blocks_time);
       values.add(load_dictionary_time);
       values.add(scan_blocks_time);
+      values.add(readTime);
       values.add(total_executor_time);
       values.add(scan_blocks_num);
       values.add(total_blocklet);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
b/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
index fd2e0fd..39195b3 100644
--- a/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
@@ -48,10 +48,10 @@ public class QueryStasticsRecorderImplTest {
     queryStasticsRecorderImpl.logStatistics();
     queryStasticsRecorderImpl.recordStatistics(queryStatisticWithLOAD_BLOCKS_EXECUTOR);
     queryStatisticWithSCAN_BLOCKS_TIME = new QueryStatistic();
-    queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_TIME,
5L);
+    queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
5L);
     queryStatisticWithSCAN_BLOCKS_TIME
-        .addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
-    queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_TIME,
5L);
+        .addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, 5L);
+    queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
5L);
     queryStasticsRecorderImpl.logStatistics();
     queryStasticsRecorderImpl.recordStatistics(queryStatisticWithSCAN_BLOCKS_TIME);
     queryStatisticWithSCAN_BLOCKS_NUM = new QueryStatistic();


Mime
View raw message