carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [09/47] incubator-carbondata git commit: [CARBONDATA-68] Added query statistic for detail query case measuring carbon executor time (#852)
Date Mon, 01 Aug 2016 10:05:07 GMT
[CARBONDATA-68] Added query statistic for detail query case measuring carbon executor time
(#852)



Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a81ed981
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a81ed981
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a81ed981

Branch: refs/heads/master
Commit: a81ed981230e3fddd8e1c29d774266c171a54643
Parents: 584402e
Author: Kumar Vishal <kumarvishal.1802@gmail.com>
Authored: Sat Jul 23 12:21:02 2016 +0530
Committer: Venkata Ramana G <g.ramana.v1@gmail.com>
Committed: Sat Jul 23 12:21:02 2016 +0530

----------------------------------------------------------------------
 .../carbon/querystatistics/QueryStatistic.java  | 12 ++++++++++
 .../AbstractDetailQueryResultIterator.java      | 25 +++++++++++++-------
 .../iterator/DetailQueryResultIterator.java     |  8 ++++++-
 .../carbondata/spark/rdd/CarbonQueryRDD.scala   | 18 +++++++++++---
 4 files changed, 51 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
b/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
index 41f3d22..80398c1 100644
--- a/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
+++ b/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
@@ -61,6 +61,18 @@ public class QueryStatistic implements Serializable {
   }
 
   /**
+   * Below method will be used to add fixed time statistic.
+   * For example total time taken for scan or result preparation
+   *
+   * @param message   statistic message
+   * @param timetaken
+   */
+  public void addFixedTimeStatistic(String message, long timetaken) {
+    this.timeTaken = timetaken;
+    this.message = message;
+  }
+
+  /**
    * Below method will be used to get the statistic message, which will
    * be used to log
    *

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
b/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
index 01e94c8..dc94704 100644
--- a/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
@@ -66,14 +66,18 @@ public abstract class AbstractDetailQueryResultIterator extends CarbonIterator
{
    */
   protected int[] blockIndexToBeExecuted;
   /**
-   * to store the statistics
-   */
-  protected QueryStatistic statistic;
-  /**
    * total number of records processed
    */
   protected long totalNumberOfOutputRecords;
   /**
+   * total scan time of the query
+   */
+  protected long totalScanTime;
+  /**
+   * total time taken to prepare the result
+   */
+  protected long totalResultPreparationTime;
+  /**
    * number of cores which can be used
    */
   private long numberOfCores;
@@ -113,7 +117,6 @@ public abstract class AbstractDetailQueryResultIterator extends CarbonIterator
{
     executor = queryExecutor;
     this.blockExecutionInfos = infos;
     this.blockIndexToBeExecuted = new int[(int) numberOfCores];
-    statistic = new QueryStatistic();
     intialiseInfos();
   }
 
@@ -142,9 +145,15 @@ public abstract class AbstractDetailQueryResultIterator extends CarbonIterator
{
     if (currentCounter < totalNumberOfNode) {
       return true;
     }
-    statistic.addStatistics("Time taken to processed " + blockExecutionInfos.size()
-            + " block(s) of output record size: " + totalNumberOfOutputRecords,
-        System.currentTimeMillis());
+    QueryStatistic statistic = new QueryStatistic();
+    statistic
+        .addFixedTimeStatistic("Time taken to scan " + blockExecutionInfos.size() + " block(s)
",
+            totalScanTime);
+    blockExecutionInfos.get(0).getStatisticsRecorder().recordStatistics(statistic);
+    statistic = new QueryStatistic();
+    statistic.addFixedTimeStatistic(
+        "Time take to prepare query result of size " + totalNumberOfOutputRecords,
+        totalResultPreparationTime);
     blockExecutionInfos.get(0).getStatisticsRecorder().recordStatistics(statistic);
     return false;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
b/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
index db6e9aa..e2f54ee 100644
--- a/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
@@ -55,8 +55,11 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
   @Override public BatchResult next() {
     currentCounter += updateSliceIndexToBeExecuted();
     CarbonIterator<Result> result = null;
+    long startTime = System.currentTimeMillis();
+    BatchResult batchResult = null;
     try {
       result = executor.executeQuery(blockExecutionInfos, blockIndexToBeExecuted);
+      totalScanTime+=System.currentTimeMillis()-startTime;
     } catch (QueryExecutionException e) {
       throw new RuntimeException(e.getCause().getMessage());
     }
@@ -71,7 +74,10 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
       Result next = result.next();
       if (next.size() > 0) {
         totalNumberOfOutputRecords += next.size();
-        return queryResultPreparator.prepareQueryResult(next);
+        startTime=System.currentTimeMillis();
+        batchResult = queryResultPreparator.prepareQueryResult(next);
+        totalResultPreparationTime+=System.currentTimeMillis()-startTime;
+        return batchResult;
       } else {
         return new BatchResult();
       }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
index e0ec5fe..97f1993 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
@@ -18,7 +18,6 @@
 
 package org.carbondata.spark.rdd
 
-
 import java.util
 
 import scala.collection.JavaConverters._
@@ -44,6 +43,7 @@ import org.carbondata.spark.Value
 import org.carbondata.spark.load.CarbonLoaderUtil
 import org.carbondata.spark.util.QueryPlanUtil
 
+
 class CarbonSparkPartition(rddId: Int, val idx: Int,
   val locations: Array[String],
   val tableBlockInfos: util.List[TableBlockInfo])
@@ -216,7 +216,13 @@ class CarbonQueryRDD[V: ClassTag](
         }
         if (finished) {
           clearDictionaryCache(queryModel.getColumnToDictionaryMapping)
-          if(null!=queryModel.getStatisticsRecorder) {
+          if (null != queryModel.getStatisticsRecorder) {
+            val queryStatistic = new QueryStatistic
+            queryStatistic
+              .addStatistics("Total Time taken to execute the query in executor Side",
+                System.currentTimeMillis - queryStartTime
+              )
+            queryModel.getStatisticsRecorder.recordStatistics(queryStatistic);
             queryModel.getStatisticsRecorder.logStatistics();
           }
         }
@@ -231,7 +237,13 @@ class CarbonQueryRDD[V: ClassTag](
         recordCount += 1
         if (queryModel.getLimit != -1 && recordCount >= queryModel.getLimit) {
           clearDictionaryCache(queryModel.getColumnToDictionaryMapping)
-           if(null!=queryModel.getStatisticsRecorder) {
+          if (null != queryModel.getStatisticsRecorder) {
+            val queryStatistic = new QueryStatistic
+            queryStatistic
+              .addStatistics("Total Time taken to execute the query in executor Side",
+                System.currentTimeMillis - queryStartTime
+              )
+            queryModel.getStatisticsRecorder.recordStatistics(queryStatistic);
             queryModel.getStatisticsRecorder.logStatistics();
           }
         }


Mime
View raw message