Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E1E3817E8E for ; Mon, 27 Apr 2015 21:26:28 +0000 (UTC) Received: (qmail 84799 invoked by uid 500); 27 Apr 2015 21:26:06 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 84176 invoked by uid 500); 27 Apr 2015 21:26:06 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 81905 invoked by uid 99); 27 Apr 2015 21:26:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 27 Apr 2015 21:26:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3AF03E17BF; Mon, 27 Apr 2015 21:26:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zjshen@apache.org To: common-commits@hadoop.apache.org Date: Mon, 27 Apr 2015 21:26:49 -0000 Message-Id: <51af5fdc261749579ae7e9f89b778d00@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [46/50] [abbrv] hadoop git commit: MAPREDUCE-6252. JobHistoryServer should not fail when encountering a missing directory. Contributed by Craig Welch. MAPREDUCE-6252. JobHistoryServer should not fail when encountering a missing directory. Contributed by Craig Welch. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f017f222 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f017f222 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f017f222 Branch: refs/heads/YARN-2928 Commit: f017f222821ea636eafb61431ef283a1d49cd7c9 Parents: 3044c8d Author: Devaraj K Authored: Mon Apr 27 15:01:42 2015 +0530 Committer: Zhijie Shen Committed: Mon Apr 27 14:18:54 2015 -0700 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../mapreduce/v2/hs/HistoryFileManager.java | 19 ++++++++++------- .../mapreduce/v2/hs/TestHistoryFileManager.java | 22 ++++++++++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f017f222/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index dca42c4..4d217cd 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -364,6 +364,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are broken due to MAPREDUCE-6297. (Siqi Li via gera) + MAPREDUCE-6252. JobHistoryServer should not fail when encountering a + missing directory. (Craig Welch via devaraj) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/f017f222/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 65f8a4f..69f814d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -740,17 +740,22 @@ public class HistoryFileManager extends AbstractService { } } - private static List scanDirectory(Path path, FileContext fc, + @VisibleForTesting + protected static List scanDirectory(Path path, FileContext fc, PathFilter pathFilter) throws IOException { path = fc.makeQualified(path); List jhStatusList = new ArrayList(); - RemoteIterator fileStatusIter = fc.listStatus(path); - while (fileStatusIter.hasNext()) { - FileStatus fileStatus = fileStatusIter.next(); - Path filePath = fileStatus.getPath(); - if (fileStatus.isFile() && pathFilter.accept(filePath)) { - jhStatusList.add(fileStatus); + try { + RemoteIterator fileStatusIter = fc.listStatus(path); + while (fileStatusIter.hasNext()) { + FileStatus fileStatus = fileStatusIter.next(); + Path filePath = fileStatus.getPath(); + if (fileStatus.isFile() && pathFilter.accept(filePath)) { + jhStatusList.add(fileStatus); + } } + } catch (FileNotFoundException fe) { + LOG.error("Error while scanning directory " + path, fe); } return jhStatusList; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f017f222/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index e2e943a..1c5cc5c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -21,13 +21,17 @@ package org.apache.hadoop.mapreduce.v2.hs; import java.io.File; import java.io.FileOutputStream; +import java.io.FileNotFoundException; import java.util.UUID; +import java.util.List; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -45,6 +49,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import static org.mockito.Mockito.*; + public class TestHistoryFileManager { private static MiniDFSCluster dfsCluster = null; private static MiniDFSCluster dfsCluster2 = null; @@ -199,4 +205,20 @@ public class TestHistoryFileManager { testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock); } + @Test + public void testScanDirectory() throws Exception { + + Path p = new Path("any"); + FileContext fc = mock(FileContext.class); + when(fc.makeQualified(p)).thenReturn(p); + when(fc.listStatus(p)).thenThrow(new FileNotFoundException()); + + List lfs = HistoryFileManager.scanDirectory(p, fc, null); + + //primarily, succcess is that an exception was not thrown. Also nice to + //check this + Assert.assertNotNull(lfs); + + } + }