hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rkan...@apache.org
Subject hadoop git commit: YARN-5257. Fix unreleased resources and null dereferences (yufeigu via rkanter)
Date Tue, 27 Dec 2016 22:14:36 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 1bbd02327 -> 9262797e8


YARN-5257. Fix unreleased resources and null dereferences (yufeigu via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9262797e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9262797e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9262797e

Branch: refs/heads/trunk
Commit: 9262797e86453fc04b7ca3710b73b21fcdf9e6b4
Parents: 1bbd023
Author: Robert Kanter <rkanter@apache.org>
Authored: Tue Dec 27 14:14:08 2016 -0800
Committer: Robert Kanter <rkanter@apache.org>
Committed: Tue Dec 27 14:14:08 2016 -0800

----------------------------------------------------------------------
 .../unmanagedamlauncher/UnmanagedAMLauncher.java     | 10 +++++-----
 .../org/apache/hadoop/yarn/client/cli/TopCLI.java    | 15 ++++++++-------
 .../yarn/client/api/impl/TimelineClientImpl.java     |  2 +-
 .../java/org/apache/hadoop/yarn/state/Graph.java     |  2 +-
 .../hadoop/yarn/state/VisualizeStateMachine.java     |  4 +++-
 .../hadoop/yarn/util/ProcfsBasedProcessTree.java     |  5 ++++-
 .../application/ApplicationImpl.java                 |  2 +-
 .../containermanager/container/ContainerImpl.java    |  2 +-
 .../linux/resources/TrafficController.java           | 13 +++++++------
 .../localizer/LocalizedResource.java                 |  2 +-
 .../localizer/ResourceLocalizationService.java       |  1 +
 11 files changed, 33 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
index b9c7e80..030a730 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
@@ -192,11 +192,11 @@ public class UnmanagedAMLauncher {
       throw new RuntimeException(ex);
     }
     tokenFile.deleteOnExit();
-    DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile, 
-        true));
-    credentials.writeTokenStorageToStream(os);
-    os.close();
-    
+    try (DataOutputStream os = new DataOutputStream(
+        new FileOutputStream(tokenFile, true))) {
+      credentials.writeTokenStorageToStream(os);
+    }
+
     Map<String, String> env = System.getenv();
     ArrayList<String> envAMList = new ArrayList<String>();
     boolean setClasspath = false;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
index f99dd48..cf7b747 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
@@ -774,13 +774,14 @@ public class TopCLI extends YarnCLI {
 
   private JSONObject getJSONObject(URLConnection conn)
       throws IOException, JSONException {
-    InputStream in = conn.getInputStream();
-    String encoding = conn.getContentEncoding();
-    encoding = encoding == null ? "UTF-8" : encoding;
-    String body = IOUtils.toString(in, encoding);
-    JSONObject obj = new JSONObject(body);
-    JSONObject clusterInfo = obj.getJSONObject("clusterInfo");
-    return clusterInfo;
+    try(InputStream in = conn.getInputStream()) {
+      String encoding = conn.getContentEncoding();
+      encoding = encoding == null ? "UTF-8" : encoding;
+      String body = IOUtils.toString(in, encoding);
+      JSONObject obj = new JSONObject(body);
+      JSONObject clusterInfo = obj.getJSONObject("clusterInfo");
+      return clusterInfo;
+    }
   }
 
   private URL getClusterUrl() throws Exception {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index b275a94..4506c48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -805,7 +805,7 @@ public class TimelineClientImpl extends TimelineClient {
                 error.getErrorCode());
           }
         }
-      } else if (type.equals(DOMAIN_DATA_TYPE)) {
+      } else if (type.equals(DOMAIN_DATA_TYPE) && domains != null) {
         boolean hasError = false;
         for (TimelineDomain domain : domains.getDomains()) {
           try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
index 390deb2..6fb9119 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
@@ -189,7 +189,7 @@ public class Graph {
 
   public void save(String filepath) throws IOException {
     try (OutputStreamWriter fout = new OutputStreamWriter(
-        new FileOutputStream(filepath), Charset.forName("UTF-8"));) {
+        new FileOutputStream(filepath), Charset.forName("UTF-8"))) {
       fout.write(generateGraphViz());
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
index 26e93c6..1620397 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
@@ -49,7 +49,9 @@ public class VisualizeStateMachine {
       if (gname.endsWith("Impl")) {
         gname = gname.substring(0, gname.length()-4);
       }
-      ret.addSubGraph(factory.generateStateGraph(gname));
+      if (ret != null) {
+        ret.addSubGraph(factory.generateStateGraph(gname));
+      }
     }
     return ret;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
index 29bc277..bb2a77f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
@@ -803,7 +803,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree
{
             if (LOG.isDebugEnabled()) {
               LOG.debug("MemInfo : " + key + " : Value  : " + value);
             }
-            memoryMappingInfo.setMemInfo(key, value);
+
+            if (memoryMappingInfo != null) {
+              memoryMappingInfo.setMemInfo(key, value);
+            }
           }
         } catch (Throwable t) {
           LOG

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index b9197c2..112b43a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -603,7 +603,7 @@ public class ApplicationImpl implements Application {
       } catch (InvalidStateTransitionException e) {
         LOG.warn("Can't handle this event at current state", e);
       }
-      if (oldState != newState) {
+      if (newState != null && oldState != newState) {
         LOG.info("Application " + applicationID + " transitioned from "
             + oldState + " to " + newState);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 4a6be32..cae30cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1680,7 +1680,7 @@ public class ContainerImpl implements Container {
             + oldState + "], eventType: [" + event.getType() + "]," +
             " container: [" + containerID + "]", e);
       }
-      if (oldState != newState) {
+      if (newState != null && oldState != newState) {
         LOG.info("Container " + containerID + " transitioned from "
             + oldState
             + " to " + newState);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
index f1468fb..4698eb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
@@ -626,15 +626,16 @@ import java.util.regex.Pattern;
       try {
         File tcCmds = File.createTempFile(TMP_FILE_PREFIX, TMP_FILE_SUFFIX, new
             File(tmpDirPath));
-        Writer writer = new OutputStreamWriter(new FileOutputStream(tcCmds),
-            "UTF-8");
-        PrintWriter printWriter = new PrintWriter(writer);
 
-        for (String command : commands) {
-          printWriter.println(command);
+        try (
+            Writer writer = new OutputStreamWriter(new FileOutputStream(tcCmds),
+                "UTF-8");
+            PrintWriter printWriter = new PrintWriter(writer)) {
+          for (String command : commands) {
+            printWriter.println(command);
+          }
         }
 
-        printWriter.close();
         operation.appendArgs(tcCmds.getAbsolutePath());
 
         return operation;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
index 6d67cac..98f4e21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
@@ -200,7 +200,7 @@ public class LocalizedResource implements EventHandler<ResourceEvent>
{
       } catch (InvalidStateTransitionException e) {
         LOG.warn("Can't handle this event at current state", e);
       }
-      if (oldState != newState) {
+      if (newState != null && oldState != newState) {
         if (LOG.isDebugEnabled()) {
           LOG.debug("Resource " + resourcePath + (localPath != null ?
               "(->" + localPath + ")": "") + " transitioned from " + oldState

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index e4d74f6..37473e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1051,6 +1051,7 @@ public class ResourceLocalizationService extends CompositeService
           LOG.error(
               "Got exception in parsing URL of LocalResource:"
                   + rsrc.getResource(), e);
+          continue;
         }
         LocalizerResourceRequestEvent assoc = scheduled.get(req);
         if (assoc == null) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message