Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C162F200CDE for ; Tue, 8 Aug 2017 22:03:51 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id BFF67167E40; Tue, 8 Aug 2017 20:03:51 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8D8E6167E3F for ; Tue, 8 Aug 2017 22:03:50 +0200 (CEST) Received: (qmail 47561 invoked by uid 500); 8 Aug 2017 20:03:49 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 47552 invoked by uid 99); 8 Aug 2017 20:03:49 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Aug 2017 20:03:49 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E7779E96B7; Tue, 8 Aug 2017 20:03:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jlowe@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: MAPREDUCE-6927. MR job should only set tracking url if history was successfully written. Contributed by Eric Badger Date: Tue, 8 Aug 2017 20:03:47 +0000 (UTC) archived-at: Tue, 08 Aug 2017 20:03:51 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2.8 bc87a293d -> ea67e1f26 MAPREDUCE-6927. MR job should only set tracking url if history was successfully written. Contributed by Eric Badger (cherry picked from commit 735fce5bec17f4e1799daf922625c475cf588114) Conflicts: hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea67e1f2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea67e1f2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea67e1f2 Branch: refs/heads/branch-2.8 Commit: ea67e1f266406477fec6eb4ff1ec1e1f47974166 Parents: bc87a29 Author: Jason Lowe Authored: Tue Aug 8 14:46:47 2017 -0500 Committer: Jason Lowe Committed: Tue Aug 8 15:03:34 2017 -0500 ---------------------------------------------------------------------- .../jobhistory/JobHistoryEventHandler.java | 27 +++-- .../hadoop/mapreduce/v2/app/AppContext.java | 4 + .../hadoop/mapreduce/v2/app/MRAppMaster.java | 10 ++ .../mapreduce/v2/app/rm/RMCommunicator.java | 4 +- .../jobhistory/TestJobHistoryEventHandler.java | 102 +++++++++++++++++++ .../hadoop/mapreduce/v2/app/MockAppContext.java | 10 ++ .../mapreduce/v2/app/TestRuntimeEstimators.java | 10 ++ .../hadoop/mapreduce/v2/hs/JobHistory.java | 10 ++ 8 files changed, 167 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea67e1f2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index d36462f..43b9470 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -60,6 +60,7 @@ import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; +import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; @@ -1151,7 +1152,12 @@ public class JobHistoryEventHandler extends AbstractService qualifiedDoneFile = doneDirFS.makeQualified(new Path(doneDirPrefixPath, doneJobHistoryFileName)); - moveToDoneNow(qualifiedLogFile, qualifiedDoneFile); + if(moveToDoneNow(qualifiedLogFile, qualifiedDoneFile)) { + String historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithScheme( + getConfig(), context.getApplicationID()); + context.setHistoryUrl(historyUrl); + LOG.info("Set historyUrl to " + historyUrl); + } } // Move confFile to Done Folder @@ -1357,7 +1363,7 @@ public class JobHistoryEventHandler extends AbstractService } } - private void moveTmpToDone(Path tmpPath) throws IOException { + protected void moveTmpToDone(Path tmpPath) throws IOException { if (tmpPath != null) { String tmpFileName = tmpPath.getName(); String fileName = getFileNameFromTmpFN(tmpFileName); @@ -1369,7 +1375,9 @@ public class JobHistoryEventHandler extends AbstractService // TODO If the FS objects are the same, this should be a rename instead of a // copy. - private void moveToDoneNow(Path fromPath, Path toPath) throws IOException { + protected boolean moveToDoneNow(Path fromPath, Path toPath) + throws IOException { + boolean success = false; // check if path exists, in case of retries it may not exist if (stagingDirFS.exists(fromPath)) { LOG.info("Copying " + fromPath.toString() + " to " + toPath.toString()); @@ -1380,13 +1388,18 @@ public class JobHistoryEventHandler extends AbstractService boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath, false, getConfig()); - if (copied) - LOG.info("Copied to done location: " + toPath); - else - LOG.info("copy failed"); doneDirFS.setPermission(toPath, new FsPermission( JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS)); + if (copied) { + LOG.info("Copied from: " + fromPath.toString() + + " to done location: " + toPath.toString()); + success = true; + } else { + LOG.info("Copy failed from: " + fromPath.toString() + + " to done location: " + toPath.toString()); + } } + return success; } boolean pathExists(FileSystem fileSys, Path path) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea67e1f2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java index 4af11c3..16b1760 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java @@ -69,4 +69,8 @@ public interface AppContext { String getNMHostname(); TaskAttemptFinishingMonitor getTaskAttemptFinishingMonitor(); + + String getHistoryUrl(); + + void setHistoryUrl(String historyUrl); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea67e1f2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 779d061..b704c42 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -1046,6 +1046,7 @@ public class MRAppMaster extends CompositeService { private final Configuration conf; private final ClusterInfo clusterInfo = new ClusterInfo(); private final ClientToAMTokenSecretManager clientToAMTokenSecretManager; + private String historyUrl = null; private final TaskAttemptFinishingMonitor taskAttemptFinishingMonitor; @@ -1145,6 +1146,15 @@ public class MRAppMaster extends CompositeService { return taskAttemptFinishingMonitor; } + @Override + public String getHistoryUrl() { + return historyUrl; + } + + @Override + public void setHistoryUrl(String historyUrl) { + this.historyUrl = historyUrl; + } } @SuppressWarnings("unchecked") http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea67e1f2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java index 6cec2f3..a7058e0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java @@ -215,9 +215,7 @@ public abstract class RMCommunicator extends AbstractService } LOG.info("Setting job diagnostics to " + sb.toString()); - String historyUrl = - MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(), - context.getApplicationID()); + String historyUrl = context.getHistoryUrl(); LOG.info("History url is " + historyUrl); FinishApplicationMasterRequest request = FinishApplicationMasterRequest.newInstance(finishState, http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea67e1f2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index 692bd70..255eb22 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -21,6 +21,9 @@ package org.apache.hadoop.mapreduce.jobhistory; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -60,6 +63,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; +import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -899,6 +903,104 @@ public class TestJobHistoryEventHandler { jheh.lastEventHandled.getHistoryEvent() instanceof JobUnsuccessfulCompletionEvent); } + + @Test (timeout=50000) + public void testSetTrackingURLAfterHistoryIsWritten() throws Exception { + TestParams t = new TestParams(true); + Configuration conf = new Configuration(); + + JHEvenHandlerForTest realJheh = + new JHEvenHandlerForTest(t.mockAppContext, 0, false); + JHEvenHandlerForTest jheh = spy(realJheh); + jheh.init(conf); + + try { + jheh.start(); + handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( + t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000, -1))); + verify(jheh, times(0)).processDoneFiles(any(JobId.class)); + verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); + + // Job finishes and successfully writes history + handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), + new Counters(), new Counters()))); + + verify(jheh, times(1)).processDoneFiles(any(JobId.class)); + String historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithScheme( + conf, t.mockAppContext.getApplicationID()); + verify(t.mockAppContext, times(1)).setHistoryUrl(historyUrl); + } finally { + jheh.stop(); + } + } + + @Test (timeout=50000) + public void testDontSetTrackingURLIfHistoryWriteFailed() throws Exception { + TestParams t = new TestParams(true); + Configuration conf = new Configuration(); + + JHEvenHandlerForTest realJheh = + new JHEvenHandlerForTest(t.mockAppContext, 0, false); + JHEvenHandlerForTest jheh = spy(realJheh); + jheh.init(conf); + + try { + jheh.start(); + doReturn(false).when(jheh).moveToDoneNow(any(Path.class), + any(Path.class)); + doNothing().when(jheh).moveTmpToDone(any(Path.class)); + handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( + t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000, -1))); + verify(jheh, times(0)).processDoneFiles(any(JobId.class)); + verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); + + // Job finishes, but doesn't successfully write history + handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), + new Counters(), new Counters()))); + verify(jheh, times(1)).processDoneFiles(any(JobId.class)); + verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); + + } finally { + jheh.stop(); + } + } + @Test (timeout=50000) + public void testDontSetTrackingURLIfHistoryWriteThrows() throws Exception { + TestParams t = new TestParams(true); + Configuration conf = new Configuration(); + + JHEvenHandlerForTest realJheh = + new JHEvenHandlerForTest(t.mockAppContext, 0, false); + JHEvenHandlerForTest jheh = spy(realJheh); + jheh.init(conf); + + try { + jheh.start(); + doThrow(new YarnRuntimeException(new IOException())) + .when(jheh).processDoneFiles(any(JobId.class)); + handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( + t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000, -1))); + verify(jheh, times(0)).processDoneFiles(any(JobId.class)); + verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); + + // Job finishes, but doesn't successfully write history + try { + handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), + new Counters(), new Counters()))); + throw new RuntimeException( + "processDoneFiles didn't throw, but should have"); + } catch (YarnRuntimeException yre) { + // Exception expected, do nothing + } + verify(jheh, times(1)).processDoneFiles(any(JobId.class)); + verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); + } finally { + jheh.stop(); + } + } } class JHEvenHandlerForTest extends JobHistoryEventHandler { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea67e1f2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java index e690f3f..95356b5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java @@ -154,4 +154,14 @@ public class MockAppContext implements AppContext { return null; } + @Override + public String getHistoryUrl() { + return null; + } + + @Override + public void setHistoryUrl(String historyUrl) { + return; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea67e1f2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index 0b7d1b1..c88686e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -895,5 +895,15 @@ public class TestRuntimeEstimators { public TaskAttemptFinishingMonitor getTaskAttemptFinishingMonitor() { return null; } + + @Override + public String getHistoryUrl() { + return null; + } + + @Override + public void setHistoryUrl(String historyUrl) { + return; + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea67e1f2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java index 41bc90a..3466f7a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java @@ -405,4 +405,14 @@ public class JobHistory extends AbstractService implements HistoryContext { public TaskAttemptFinishingMonitor getTaskAttemptFinishingMonitor() { return null; } + + @Override + public String getHistoryUrl() { + return null; + } + + @Override + public void setHistoryUrl(String historyUrl) { + return; + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org