carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: use recoder for all statistic log
Date Sun, 09 Oct 2016 14:05:53 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 958dfde64 -> 4cab5bf38


use recoder for all statistic log


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4e1154d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4e1154d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4e1154d5

Branch: refs/heads/master
Commit: 4e1154d538a96d8862a2b66815cb5db1dc7e3ed5
Parents: 958dfde
Author: foryou2030 <foryou2030@126.com>
Authored: Sun Oct 9 16:41:20 2016 +0800
Committer: foryou2030 <foryou2030@126.com>
Committed: Sun Oct 9 16:41:20 2016 +0800

----------------------------------------------------------------------
 .../core/carbon/querystatistics/QueryStatistic.java    |  5 +++++
 .../apache/spark/sql/optimizer/CarbonOptimizer.scala   | 13 ++++++++-----
 2 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4e1154d5/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
index 939b819..74fd651 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
@@ -20,6 +20,8 @@ package org.apache.carbondata.core.carbon.querystatistics;
 
 import java.io.Serializable;
 
+import org.apache.commons.lang3.StringUtils;
+
 /**
  * Wrapper class to maintain the query statistics for each phase of the query
  */
@@ -91,6 +93,9 @@ public class QueryStatistic implements Serializable {
    * @return statistic message
    */
   public String getStatistics(String queryWithTaskId) {
+    if (StringUtils.isEmpty(queryWithTaskId)) {
+      return message + timeTaken;
+    }
     return message + " for the taskid : " + queryWithTaskId + " Is : " + timeTaken;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4e1154d5/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
index b8af85e..e428bba 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
@@ -24,16 +24,16 @@ import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.optimizer.Optimizer
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.TreeNode
 import org.apache.spark.sql.execution.RunnableCommand
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.types.{IntegerType, StringType}
 
 import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory
 import org.apache.carbondata.spark.CarbonFilters
 
 /**
@@ -74,10 +74,13 @@ class ResolveCarbonFunctions(relations: Seq[CarbonDecoderRelation])
   def apply(plan: LogicalPlan): LogicalPlan = {
     if (relations.nonEmpty && !isOptimized(plan)) {
       LOGGER.info("Starting to optimize plan")
-      val startTime = System.currentTimeMillis
+      val recorder = CarbonTimeStatisticsFactory.createExecutorRecorder("");
+      val queryStatistic = new QueryStatistic()
       val result = transformCarbonPlan(plan, relations)
-      LOGGER.statistic("Time taken for Carbon Optimizer to optimize: " +
-        ( System.currentTimeMillis - startTime))
+      queryStatistic.addStatistics("Time taken for Carbon Optimizer to optimize: ",
+        System.currentTimeMillis)
+      recorder.recordStatistics(queryStatistic)
+      recorder.logStatistics()
       result
     } else {
       LOGGER.info("Skip CarbonOptimizer")


Mime
View raw message