carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [02/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-57] BLOCK distribution un wanted wait for the executor node even though the sufficient nodes are available (#810)
Date Wed, 20 Jul 2016 10:13:30 GMT
[CARBONDATA-57] BLOCK distribution un wanted wait for the executor node even though the sufficient
nodes are available (#810)

Added the log to the know the cost of ensuring the required executor

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/18ef176f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/18ef176f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/18ef176f

Branch: refs/heads/master
Commit: 18ef176f1e96688246aa938afc390c0e748790ba
Parents: 17c8128
Author: Mohammad Shahid Khan <mohdshahidkhan1987@gmail.com>
Authored: Fri Jul 15 02:25:03 2016 +0530
Committer: Venkata Ramana G <g.ramana.v1@gmail.com>
Committed: Fri Jul 15 02:25:03 2016 +0530

----------------------------------------------------------------------
 .../main/scala/org/apache/spark/sql/hive/DistributionUtil.scala | 5 ++++-
 .../main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala   | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/18ef176f/integration/spark/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
index f63a690..baff2c7 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
@@ -122,14 +122,17 @@ object DistributionUtil {
       confExecutors
     } else {nodeMapping.size()}
 
+    var startTime = System.currentTimeMillis();
     CarbonContext.ensureExecutors(sparkContext, requiredExecutors)
     var nodes = DistributionUtil.getNodeList(sparkContext)
     var maxTimes = 30;
-    while (nodes.length != requiredExecutors && maxTimes > 0) {
+    while (nodes.length < requiredExecutors && maxTimes > 0) {
       Thread.sleep(500);
       nodes = DistributionUtil.getNodeList(sparkContext)
       maxTimes = maxTimes - 1;
     }
+    var timDiff = System.currentTimeMillis() - startTime;
+    LOGGER.info("Total Time taken to ensure the required executors : " + timDiff)
     LOGGER.info("Time elapsed to allocate the required executors : " + (30 - maxTimes) *
500)
     nodes
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/18ef176f/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
index 521ba82..e2dc900 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -247,7 +247,7 @@ class CarbonMergerRDD[K, V](
       + ", no.of.nodes where data present=" + nodeMapping.size())
     var nodes = DistributionUtil.getNodeList(sparkContext)
     var maxTimes = 30
-    while (nodes.length != requiredExecutors && maxTimes > 0) {
+    while (nodes.length < requiredExecutors && maxTimes > 0) {
       Thread.sleep(500)
       nodes = DistributionUtil.getNodeList(sparkContext)
       maxTimes = maxTimes - 1


Mime
View raw message