Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4339910B18 for ; Fri, 19 Dec 2014 19:39:53 +0000 (UTC) Received: (qmail 50160 invoked by uid 500); 19 Dec 2014 19:39:53 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 50075 invoked by uid 500); 19 Dec 2014 19:39:52 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 50066 invoked by uid 99); 19 Dec 2014 19:39:52 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 19 Dec 2014 19:39:52 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 96E30A2FF4A; Fri, 19 Dec 2014 19:39:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: rkanter@apache.org To: common-commits@hadoop.apache.org Message-Id: <1e82d790af414599ab9f5ce419fa40c3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: MAPREDUCE-6198. NPE from JobTracker#resolveAndAddToTopology in MR1 cause initJob and heartbeat failure. (zxu via rkanter) Date: Fri, 19 Dec 2014 19:39:52 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-1 0860d1a26 -> eb5191195 MAPREDUCE-6198. NPE from JobTracker#resolveAndAddToTopology in MR1 cause initJob and heartbeat failure. (zxu via rkanter) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb519119 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb519119 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb519119 Branch: refs/heads/branch-1 Commit: eb5191195ae142f6678b633af5625a4fce5c9c59 Parents: 0860d1a Author: Robert Kanter Authored: Fri Dec 19 11:39:04 2014 -0800 Committer: Robert Kanter Committed: Fri Dec 19 11:39:04 2014 -0800 ---------------------------------------------------------------------- CHANGES.txt | 3 ++ .../apache/hadoop/net/ScriptBasedMapping.java | 4 +- .../org/apache/hadoop/mapred/JobTracker.java | 9 +++- .../hadoop/mapred/TestJobTrackerTopology.java | 43 ++++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb519119/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 10644bf..d3903f0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -262,6 +262,9 @@ Release 1.3.0 - unreleased MAPREDUCE-6196. Fix BigDecimal ArithmeticException in PiEstimator (rchiang via rkanter) + MAPREDUCE-6198. NPE from JobTracker#resolveAndAddToTopology in MR1 cause + initJob and heartbeat failure. (zxu via rkanter) + Release 1.2.2 - unreleased INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb519119/src/core/org/apache/hadoop/net/ScriptBasedMapping.java ---------------------------------------------------------------------- diff --git a/src/core/org/apache/hadoop/net/ScriptBasedMapping.java b/src/core/org/apache/hadoop/net/ScriptBasedMapping.java index 6834d5d..dc31e49 100644 --- a/src/core/org/apache/hadoop/net/ScriptBasedMapping.java +++ b/src/core/org/apache/hadoop/net/ScriptBasedMapping.java @@ -42,8 +42,8 @@ implements Configurable static final int MIN_ALLOWABLE_ARGS = 1; static final int DEFAULT_ARG_COUNT = 100; - - static final String SCRIPT_FILENAME_KEY = "topology.script.file.name"; + + public static final String SCRIPT_FILENAME_KEY = "topology.script.file.name"; static final String SCRIPT_ARG_COUNT_KEY = "topology.script.number.args"; public ScriptBasedMapping(Configuration conf) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb519119/src/mapred/org/apache/hadoop/mapred/JobTracker.java ---------------------------------------------------------------------- diff --git a/src/mapred/org/apache/hadoop/mapred/JobTracker.java b/src/mapred/org/apache/hadoop/mapred/JobTracker.java index 5e2110f..c8d496c 100644 --- a/src/mapred/org/apache/hadoop/mapred/JobTracker.java +++ b/src/mapred/org/apache/hadoop/mapred/JobTracker.java @@ -2818,7 +2818,14 @@ public class JobTracker implements MRConstants, InterTrackerProtocol, List tmpList = new ArrayList(1); tmpList.add(name); List rNameList = dnsToSwitchMapping.resolve(tmpList); - String rName = rNameList.get(0); + String rName = null; + if (rNameList == null || rNameList.get(0) == null) { + rName = NetworkTopology.DEFAULT_RACK; + LOG.warn("Couldn't resolve " + name + ". Falling back to " + + NetworkTopology.DEFAULT_RACK); + } else { + rName = rNameList.get(0); + } String networkLoc = NodeBase.normalize(rName); return addHostToNodeMapping(name, networkLoc); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb519119/src/test/org/apache/hadoop/mapred/TestJobTrackerTopology.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/hadoop/mapred/TestJobTrackerTopology.java b/src/test/org/apache/hadoop/mapred/TestJobTrackerTopology.java new file mode 100644 index 0000000..0a3530e --- /dev/null +++ b/src/test/org/apache/hadoop/mapred/TestJobTrackerTopology.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +import junit.framework.TestCase; +import org.apache.hadoop.net.ScriptBasedMapping; +import org.junit.Assert; + +/** + * Test {@link JobTracker} w.r.t resolveAndAddToTopology. + */ +public class TestJobTrackerTopology extends TestCase { + + public void testJobTrackerResolveAndAddToTopology() throws Exception { + JobConf conf = new JobConf(); + conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, + "test.sh"); + conf = MiniMRCluster.configureJobConf(conf, "file:///", 0, 0, null); + + JobTracker jt = JobTracker.startTracker(conf); + try { + jt.resolveAndAddToTopology("test.host"); + } catch (NullPointerException e) { + Assert.fail("NullPointerException should not happen"); + } + jt.stopTracker(); + } +} \ No newline at end of file