hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject hadoop git commit: MAPREDUCE-6473. Job submission can take a long time during Cluster initialization. Contributed by Kuhu Shukla (cherry picked from commit f657b54281d517c69df0a23b97b47befa77d1dc2)
Date Tue, 12 Jan 2016 23:58:45 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 2605e78e4 -> 0af01092b


MAPREDUCE-6473. Job submission can take a long time during Cluster initialization. Contributed
by Kuhu Shukla
(cherry picked from commit f657b54281d517c69df0a23b97b47befa77d1dc2)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0af01092
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0af01092
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0af01092

Branch: refs/heads/branch-2
Commit: 0af01092b63fc804e56052a8fcaac1045e402960
Parents: 2605e78
Author: Jason Lowe <jlowe@apache.org>
Authored: Tue Jan 12 23:56:23 2016 +0000
Committer: Jason Lowe <jlowe@apache.org>
Committed: Tue Jan 12 23:58:01 2016 +0000

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  3 +
 .../org/apache/hadoop/mapreduce/Cluster.java    | 67 ++++++++++++--------
 2 files changed, 43 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af01092/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index caa0d77..ef67240 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -171,6 +171,9 @@ Release 2.8.0 - UNRELEASED
     MAPREDUCE-6394. Speed up Task processing loop in HsTasksBlock#render()
     (Ray Chiang via jlowe)
 
+    MAPREDUCE-6473. Job submission can take a long time during Cluster
+    initialization (Kuhu Shukla via jlowe)
+
   BUG FIXES
 
     MAPREDUCE-6314. TestPipeApplication fails on trunk.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af01092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
index fc228e6..9563c0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
@@ -66,7 +66,23 @@ public class Cluster {
 
   private static ServiceLoader<ClientProtocolProvider> frameworkLoader =
       ServiceLoader.load(ClientProtocolProvider.class);
-  
+  private volatile List<ClientProtocolProvider> providerList = null;
+
+  private void initProviderList() {
+    if (providerList == null) {
+      synchronized (frameworkLoader) {
+        if (providerList == null) {
+          List<ClientProtocolProvider> localProviderList =
+              new ArrayList<ClientProtocolProvider>();
+          for (ClientProtocolProvider provider : frameworkLoader) {
+            localProviderList.add(provider);
+          }
+          providerList = localProviderList;
+        }
+      }
+    }
+  }
+
   static {
     ConfigUtil.loadResources();
   }
@@ -85,34 +101,31 @@ public class Cluster {
   private void initialize(InetSocketAddress jobTrackAddr, Configuration conf)
       throws IOException {
 
-    synchronized (frameworkLoader) {
-      for (ClientProtocolProvider provider : frameworkLoader) {
-        LOG.debug("Trying ClientProtocolProvider : "
-            + provider.getClass().getName());
-        ClientProtocol clientProtocol = null; 
-        try {
-          if (jobTrackAddr == null) {
-            clientProtocol = provider.create(conf);
-          } else {
-            clientProtocol = provider.create(jobTrackAddr, conf);
-          }
+    initProviderList();
+    for (ClientProtocolProvider provider : providerList) {
+      LOG.debug("Trying ClientProtocolProvider : "
+          + provider.getClass().getName());
+      ClientProtocol clientProtocol = null;
+      try {
+        if (jobTrackAddr == null) {
+          clientProtocol = provider.create(conf);
+        } else {
+          clientProtocol = provider.create(jobTrackAddr, conf);
+        }
 
-          if (clientProtocol != null) {
-            clientProtocolProvider = provider;
-            client = clientProtocol;
-            LOG.debug("Picked " + provider.getClass().getName()
-                + " as the ClientProtocolProvider");
-            break;
-          }
-          else {
-            LOG.debug("Cannot pick " + provider.getClass().getName()
-                + " as the ClientProtocolProvider - returned null protocol");
-          }
-        } 
-        catch (Exception e) {
-          LOG.info("Failed to use " + provider.getClass().getName()
-              + " due to error: ", e);
+        if (clientProtocol != null) {
+          clientProtocolProvider = provider;
+          client = clientProtocol;
+          LOG.debug("Picked " + provider.getClass().getName()
+              + " as the ClientProtocolProvider");
+          break;
+        } else {
+          LOG.debug("Cannot pick " + provider.getClass().getName()
+              + " as the ClientProtocolProvider - returned null protocol");
         }
+      } catch (Exception e) {
+        LOG.info("Failed to use " + provider.getClass().getName()
+            + " due to error: ", e);
       }
     }
 


Mime
View raw message