hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject hadoop git commit: MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. Contributed by Xuan Gong
Date Mon, 01 Feb 2016 16:05:37 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk af2dccbca -> 59a212b6e


MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. Contributed by Xuan
Gong


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59a212b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59a212b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59a212b6

Branch: refs/heads/trunk
Commit: 59a212b6e1265adfa9b55c71b65a22157dfccf77
Parents: af2dccb
Author: Jason Lowe <jlowe@apache.org>
Authored: Mon Feb 1 16:05:06 2016 +0000
Committer: Jason Lowe <jlowe@apache.org>
Committed: Mon Feb 1 16:05:06 2016 +0000

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  6 +++++
 .../org/apache/hadoop/mapred/ClientCache.java   | 24 +++++++++++++++++++-
 .../hadoop/mapred/ClientServiceDelegate.java    | 16 +++++++++++++
 .../org/apache/hadoop/mapred/YARNRunner.java    | 11 +++++++++
 .../mapred/YarnClientProtocolProvider.java      |  5 ++--
 .../TestYarnClientProtocolProvider.java         |  6 +++--
 6 files changed, 63 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 8261b34..55284da 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -736,6 +736,9 @@ Release 2.7.3 - UNRELEASED
     MAPREDUCE-6554. MRAppMaster servicestart failing with NPE in
     MRAppMaster#parsePreviousJobHistory (Bibin A Chundatt via jlowe)
 
+    MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread.
+    (Xuan Gong via jlowe)
+
 Release 2.7.2 - 2016-01-25
 
   INCOMPATIBLE CHANGES
@@ -1041,6 +1044,9 @@ Release 2.6.4 - UNRELEASED
     MAPREDUCE-6554. MRAppMaster servicestart failing with NPE in
     MRAppMaster#parsePreviousJobHistory (Bibin A Chundatt via jlowe)
 
+    MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread.
+    (Xuan Gong via jlowe)
+
 Release 2.6.3 - 2015-12-17
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
index 4335c82..93ea5c4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
@@ -22,11 +22,11 @@ import java.io.IOException;
 import java.security.PrivilegedAction;
 import java.util.HashMap;
 import java.util.Map;
-
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.mapreduce.JobID;
 import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol;
 import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
@@ -97,4 +97,26 @@ public class ClientCache {
       }
     });
   }
+
+  public void close() throws IOException {
+    if (rm != null) {
+      rm.close();
+    }
+
+    if (hsProxy != null) {
+      RPC.stopProxy(hsProxy);
+      hsProxy = null;
+    }
+
+    if (cache != null && !cache.isEmpty()) {
+      for (ClientServiceDelegate delegate : cache.values()) {
+        if (delegate != null) {
+          delegate.close();
+          delegate = null;
+        }
+      }
+      cache.clear();
+      cache = null;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
index 8517c19..eac8dbc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.mapreduce.JobID;
 import org.apache.hadoop.mapreduce.JobStatus;
 import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -532,4 +533,19 @@ public class ClientServiceDelegate {
       throw new IOException("Cannot get log path for a in-progress job");
     }
   }
+
+  public void close() throws IOException {
+    if (rm != null) {
+      rm.close();
+    }
+
+    if (historyServerProxy != null) {
+      RPC.stopProxy(historyServerProxy);
+    }
+
+    if (realProxy != null) {
+      RPC.stopProxy(realProxy);
+      realProxy = null;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index fe631de..1342282 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -756,4 +756,15 @@ public class YARNRunner implements ClientProtocol {
                envConf + " config settings.");
     }
   }
+
+  public void close() throws IOException {
+    if (resMgrDelegate != null) {
+      resMgrDelegate.close();
+      resMgrDelegate = null;
+    }
+    if (clientCache != null) {
+      clientCache.close();
+      clientCache = null;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
index 29d3f66..6583fa2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
@@ -44,7 +44,8 @@ public class YarnClientProtocolProvider extends ClientProtocolProvider {
 
   @Override
   public void close(ClientProtocol clientProtocol) throws IOException {
-    // nothing to do
+    if (clientProtocol instanceof YARNRunner) {
+      ((YARNRunner)clientProtocol).close();
+    }
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
index aeb20cd..4d84fa9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
@@ -21,10 +21,11 @@ package org.apache.hadoop.mapreduce;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.doNothing;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-
 import junit.framework.TestCase;
 
 import org.apache.hadoop.conf.Configuration;
@@ -113,6 +114,8 @@ public class TestYarnClientProtocolProvider extends TestCase {
         @Override
         protected void serviceStart() throws Exception {
           assertTrue(this.client instanceof YarnClientImpl);
+          this.client = spy(this.client);
+          doNothing().when(this.client).close();
           ((YarnClientImpl) this.client).setRMClient(cRMProtocol);
         }
       };
@@ -126,5 +129,4 @@ public class TestYarnClientProtocolProvider extends TestCase {
       }
     }
   }
-
 }


Mime
View raw message