Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BB0EE17233 for ; Tue, 24 Nov 2015 23:47:46 +0000 (UTC) Received: (qmail 9900 invoked by uid 500); 24 Nov 2015 23:47:46 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 9840 invoked by uid 500); 24 Nov 2015 23:47:46 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 9831 invoked by uid 99); 24 Nov 2015 23:47:46 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 24 Nov 2015 23:47:46 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 10D50E0A36; Tue, 24 Nov 2015 23:47:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jlowe@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla Date: Tue, 24 Nov 2015 23:47:46 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 4ac6799d4 -> f5acf94ec YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5acf94e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5acf94e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5acf94e Branch: refs/heads/trunk Commit: f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701 Parents: 4ac6799 Author: Jason Lowe Authored: Tue Nov 24 23:47:10 2015 +0000 Committer: Jason Lowe Committed: Tue Nov 24 23:47:10 2015 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 6 ++++ .../nodelabels/FileSystemNodeLabelsStore.java | 6 ++-- .../TestFileSystemNodeLabelsStore.java | 29 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5acf94e/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4483589..155d5c9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1100,6 +1100,9 @@ Release 2.7.3 - UNRELEASED YARN-4344. NMs reconnecting with changed capabilities can lead to wrong cluster resource calculations (Varun Vasudev via jlowe) + YARN-4365. FileSystemNodeLabelStore should check for root dir existence on + startup (Kuhu Shukla via jlowe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -1960,6 +1963,9 @@ Release 2.6.3 - UNRELEASED YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log files from full disks. (zhihai xu via jlowe) + YARN-4365. FileSystemNodeLabelStore should check for root dir existence on + startup (Kuhu Shukla via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5acf94e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index 20dc67c..c9727a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -88,7 +88,9 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { setFileSystem(conf); // mkdir of root dir path - fs.mkdirs(fsWorkingPath); + if (!fs.exists(fsWorkingPath)) { + fs.mkdirs(fsWorkingPath); + } } @Override @@ -96,7 +98,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { IOUtils.cleanup(LOG, fs, editlogOs); } - private void setFileSystem(Configuration conf) throws IOException { + void setFileSystem(Configuration conf) throws IOException { Configuration confCopy = new Configuration(conf); confCopy.setBoolean("dfs.client.retry.policy.enabled", true); String retryPolicy = http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5acf94e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 4b052c9..4929f95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -24,6 +24,8 @@ import java.util.Arrays; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -33,6 +35,7 @@ import org.junit.Before; import org.junit.Test; import com.google.common.collect.ImmutableMap; +import org.mockito.Mockito; public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { MockNodeLabelManager mgr = null; @@ -317,4 +320,30 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { Arrays.asList("p2", "p4", "p6", "p7", "p8", "p9"))); mgr.stop(); } + + @Test + public void testRootMkdirOnInitStore() throws Exception { + final FileSystem mockFs = Mockito.mock(FileSystem.class); + FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore(mgr) { + void setFileSystem(Configuration conf) throws IOException { + fs = mockFs; + } + }; + mockStore.fs = mockFs; + verifyMkdirsCount(mockStore, true, 0); + verifyMkdirsCount(mockStore, false, 1); + verifyMkdirsCount(mockStore, true, 1); + verifyMkdirsCount(mockStore, false, 2); + } + + private void verifyMkdirsCount(FileSystemNodeLabelsStore store, + boolean existsRetVal, int expectedNumOfCalls) + throws Exception { + Mockito.when(store.fs.exists(Mockito.any( + Path.class))).thenReturn(existsRetVal); + store.init(conf); + Mockito.verify(store.fs,Mockito.times( + expectedNumOfCalls)).mkdirs(Mockito.any(Path + .class)); + } }