Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id CBEA0200C16 for ; Wed, 4 Jan 2017 09:03:41 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id CA76E160B4A; Wed, 4 Jan 2017 08:03:41 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 96DC9160B44 for ; Wed, 4 Jan 2017 09:03:40 +0100 (CET) Received: (qmail 59456 invoked by uid 500); 4 Jan 2017 08:03:37 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 56779 invoked by uid 99); 4 Jan 2017 08:03:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 04 Jan 2017 08:03:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9BDC4DFC49; Wed, 4 Jan 2017 08:03:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Wed, 04 Jan 2017 08:03:54 -0000 Message-Id: <29a74c6dd7374dc991d11488b87683b8@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [20/50] [abbrv] hadoop git commit: YARN-5257. Fix unreleased resources and null dereferences (yufeigu via rkanter) archived-at: Wed, 04 Jan 2017 08:03:42 -0000 YARN-5257. Fix unreleased resources and null dereferences (yufeigu via rkanter) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9262797e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9262797e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9262797e Branch: refs/heads/YARN-5673 Commit: 9262797e86453fc04b7ca3710b73b21fcdf9e6b4 Parents: 1bbd023 Author: Robert Kanter Authored: Tue Dec 27 14:14:08 2016 -0800 Committer: Robert Kanter Committed: Tue Dec 27 14:14:08 2016 -0800 ---------------------------------------------------------------------- .../unmanagedamlauncher/UnmanagedAMLauncher.java | 10 +++++----- .../org/apache/hadoop/yarn/client/cli/TopCLI.java | 15 ++++++++------- .../yarn/client/api/impl/TimelineClientImpl.java | 2 +- .../java/org/apache/hadoop/yarn/state/Graph.java | 2 +- .../hadoop/yarn/state/VisualizeStateMachine.java | 4 +++- .../hadoop/yarn/util/ProcfsBasedProcessTree.java | 5 ++++- .../application/ApplicationImpl.java | 2 +- .../containermanager/container/ContainerImpl.java | 2 +- .../linux/resources/TrafficController.java | 13 +++++++------ .../localizer/LocalizedResource.java | 2 +- .../localizer/ResourceLocalizationService.java | 1 + 11 files changed, 33 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index b9c7e80..030a730 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -192,11 +192,11 @@ public class UnmanagedAMLauncher { throw new RuntimeException(ex); } tokenFile.deleteOnExit(); - DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile, - true)); - credentials.writeTokenStorageToStream(os); - os.close(); - + try (DataOutputStream os = new DataOutputStream( + new FileOutputStream(tokenFile, true))) { + credentials.writeTokenStorageToStream(os); + } + Map env = System.getenv(); ArrayList envAMList = new ArrayList(); boolean setClasspath = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java index f99dd48..cf7b747 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java @@ -774,13 +774,14 @@ public class TopCLI extends YarnCLI { private JSONObject getJSONObject(URLConnection conn) throws IOException, JSONException { - InputStream in = conn.getInputStream(); - String encoding = conn.getContentEncoding(); - encoding = encoding == null ? "UTF-8" : encoding; - String body = IOUtils.toString(in, encoding); - JSONObject obj = new JSONObject(body); - JSONObject clusterInfo = obj.getJSONObject("clusterInfo"); - return clusterInfo; + try(InputStream in = conn.getInputStream()) { + String encoding = conn.getContentEncoding(); + encoding = encoding == null ? "UTF-8" : encoding; + String body = IOUtils.toString(in, encoding); + JSONObject obj = new JSONObject(body); + JSONObject clusterInfo = obj.getJSONObject("clusterInfo"); + return clusterInfo; + } } private URL getClusterUrl() throws Exception { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index b275a94..4506c48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -805,7 +805,7 @@ public class TimelineClientImpl extends TimelineClient { error.getErrorCode()); } } - } else if (type.equals(DOMAIN_DATA_TYPE)) { + } else if (type.equals(DOMAIN_DATA_TYPE) && domains != null) { boolean hasError = false; for (TimelineDomain domain : domains.getDomains()) { try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java index 390deb2..6fb9119 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java @@ -189,7 +189,7 @@ public class Graph { public void save(String filepath) throws IOException { try (OutputStreamWriter fout = new OutputStreamWriter( - new FileOutputStream(filepath), Charset.forName("UTF-8"));) { + new FileOutputStream(filepath), Charset.forName("UTF-8"))) { fout.write(generateGraphViz()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java index 26e93c6..1620397 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java @@ -49,7 +49,9 @@ public class VisualizeStateMachine { if (gname.endsWith("Impl")) { gname = gname.substring(0, gname.length()-4); } - ret.addSubGraph(factory.generateStateGraph(gname)); + if (ret != null) { + ret.addSubGraph(factory.generateStateGraph(gname)); + } } return ret; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index 29bc277..bb2a77f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -803,7 +803,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { if (LOG.isDebugEnabled()) { LOG.debug("MemInfo : " + key + " : Value : " + value); } - memoryMappingInfo.setMemInfo(key, value); + + if (memoryMappingInfo != null) { + memoryMappingInfo.setMemInfo(key, value); + } } } catch (Throwable t) { LOG http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index b9197c2..112b43a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -603,7 +603,7 @@ public class ApplicationImpl implements Application { } catch (InvalidStateTransitionException e) { LOG.warn("Can't handle this event at current state", e); } - if (oldState != newState) { + if (newState != null && oldState != newState) { LOG.info("Application " + applicationID + " transitioned from " + oldState + " to " + newState); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 4a6be32..cae30cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -1680,7 +1680,7 @@ public class ContainerImpl implements Container { + oldState + "], eventType: [" + event.getType() + "]," + " container: [" + containerID + "]", e); } - if (oldState != newState) { + if (newState != null && oldState != newState) { LOG.info("Container " + containerID + " transitioned from " + oldState + " to " + newState); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java index f1468fb..4698eb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java @@ -626,15 +626,16 @@ import java.util.regex.Pattern; try { File tcCmds = File.createTempFile(TMP_FILE_PREFIX, TMP_FILE_SUFFIX, new File(tmpDirPath)); - Writer writer = new OutputStreamWriter(new FileOutputStream(tcCmds), - "UTF-8"); - PrintWriter printWriter = new PrintWriter(writer); - for (String command : commands) { - printWriter.println(command); + try ( + Writer writer = new OutputStreamWriter(new FileOutputStream(tcCmds), + "UTF-8"); + PrintWriter printWriter = new PrintWriter(writer)) { + for (String command : commands) { + printWriter.println(command); + } } - printWriter.close(); operation.appendArgs(tcCmds.getAbsolutePath()); return operation; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java index 6d67cac..98f4e21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java @@ -200,7 +200,7 @@ public class LocalizedResource implements EventHandler { } catch (InvalidStateTransitionException e) { LOG.warn("Can't handle this event at current state", e); } - if (oldState != newState) { + if (newState != null && oldState != newState) { if (LOG.isDebugEnabled()) { LOG.debug("Resource " + resourcePath + (localPath != null ? "(->" + localPath + ")": "") + " transitioned from " + oldState http://git-wip-us.apache.org/repos/asf/hadoop/blob/9262797e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index e4d74f6..37473e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -1051,6 +1051,7 @@ public class ResourceLocalizationService extends CompositeService LOG.error( "Got exception in parsing URL of LocalResource:" + rsrc.getResource(), e); + continue; } LocalizerResourceRequestEvent assoc = scheduled.get(req); if (assoc == null) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org