Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 658761844C for ; Sat, 19 Dec 2015 00:36:25 +0000 (UTC) Received: (qmail 99196 invoked by uid 500); 19 Dec 2015 00:36:23 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 99001 invoked by uid 500); 19 Dec 2015 00:36:23 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 98620 invoked by uid 99); 19 Dec 2015 00:36:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 19 Dec 2015 00:36:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0FA3BE0972; Sat, 19 Dec 2015 00:36:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Date: Sat, 19 Dec 2015 00:36:27 -0000 Message-Id: <2e5a7de6d9494b3fbd9c1455258cd8a3@git.apache.org> In-Reply-To: <256a19699c804976b63d14308dd1d9d3@git.apache.org> References: <256a19699c804976b63d14308dd1d9d3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [06/20] hadoop git commit: YARN-4405. Support node label store in non-appendable file system. Contributed by Wangda Tan (cherry picked from commit 755dda8dd8bb23864abc752bad506f223fcac010) YARN-4405. Support node label store in non-appendable file system. Contributed by Wangda Tan (cherry picked from commit 755dda8dd8bb23864abc752bad506f223fcac010) (cherry picked from commit be404e168060c03a33a35c8d30925e317a64cad7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f355645c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f355645c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f355645c Branch: refs/heads/branch-2.8 Commit: f355645ca44aaac1bd0b8dbafb4576b495c1878f Parents: 0b88a7f Author: Jian He Authored: Thu Dec 3 17:45:31 2015 -0800 Committer: Wangda Tan Committed: Fri Dec 18 16:34:42 2015 -0800 ---------------------------------------------------------------------- .../conf/TestConfigurationFieldsBase.java | 14 ++--- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/conf/YarnConfiguration.java | 6 ++ .../yarn/conf/TestYarnConfigurationFields.java | 2 + .../nodelabels/CommonNodeLabelsManager.java | 15 ++++- .../nodelabels/FileSystemNodeLabelsStore.java | 65 +++++++++++--------- .../hadoop/yarn/nodelabels/NodeLabelsStore.java | 16 ++--- .../src/main/resources/yarn-default.xml | 8 +++ .../DummyCommonNodeLabelsManager.java | 6 +- .../TestFileSystemNodeLabelsStore.java | 28 +++++++-- .../nodelabels/NullRMNodeLabelsManager.java | 4 +- 11 files changed, 106 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java index 2e4d8b1..e528602 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java @@ -18,27 +18,22 @@ package org.apache.hadoop.conf; -import java.lang.Class; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + import java.lang.reflect.Field; import java.lang.reflect.Modifier; -import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; import static org.junit.Assert.assertTrue; -import org.apache.hadoop.conf.Configuration; - /** * Base class for comparing fields in one or more Configuration classes * against a corresponding .xml file. Usage is intended as follows: @@ -331,6 +326,7 @@ public abstract class TestConfigurationFieldsBase { private static Set compareConfigurationToXmlFields(Map keyMap1, Map keyMap2) { Set retVal = new HashSet(keyMap1.keySet()); retVal.removeAll(keyMap2.keySet()); + return retVal; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 165fa47..727868d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -539,6 +539,9 @@ Release 2.8.0 - UNRELEASED YARN-4292. ResourceUtilization should be a part of NodeInfo REST API. (Sunil G via wangda) + YARN-4405. Support node label store in non-appendable file system. (Wangda + Tan via jianhe) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 1b2565f..521bb3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2075,6 +2075,12 @@ public class YarnConfiguration extends Configuration { */ public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels."; + /** Node label store implementation class */ + public static final String FS_NODE_LABELS_STORE_IMPL_CLASS = NODE_LABELS_PREFIX + + "fs-store.impl.class"; + public static final String DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS = + "org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore"; + /** URI for NodeLabelManager */ public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX + "fs-store.root-dir"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 97fcfa1..dec5cfb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -49,6 +49,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { // Specific properties to skip configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS); + configurationPropsToSkipCompare .add(YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS); configurationPropsToSkipCompare .add(YarnConfiguration.DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index a00c49d..8b26cc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -42,6 +42,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; @@ -224,10 +225,20 @@ public class CommonNodeLabelsManager extends AbstractService { labelCollections.put(NO_LABEL, new RMNodeLabel(NO_LABEL)); } + boolean isCentralizedConfiguration() { + return isCentralizedNodeLabelConfiguration; + } + protected void initNodeLabelStore(Configuration conf) throws Exception { - this.store = new FileSystemNodeLabelsStore(this); + this.store = + ReflectionUtils + .newInstance( + conf.getClass(YarnConfiguration.FS_NODE_LABELS_STORE_IMPL_CLASS, + FileSystemNodeLabelsStore.class, NodeLabelsStore.class), + conf); + this.store.setNodeLabelsManager(this); this.store.init(conf); - this.store.recover(!isCentralizedNodeLabelConfiguration); + this.store.recover(); } // for UT purpose http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index c9727a2..a65349b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -52,11 +52,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOn import com.google.common.collect.Sets; public class FileSystemNodeLabelsStore extends NodeLabelsStore { - - public FileSystemNodeLabelsStore(CommonNodeLabelsManager mgr) { - super(mgr); - } - protected static final Log LOG = LogFactory.getLog(FileSystemNodeLabelsStore.class); protected static final String DEFAULT_DIR_NAME = "node-labels"; @@ -69,8 +64,8 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { Path fsWorkingPath; FileSystem fs; - FSDataOutputStream editlogOs; - Path editLogPath; + private FSDataOutputStream editlogOs; + private Path editLogPath; private String getDefaultFSNodeLabelsRootDir() throws IOException { // default is in local: /tmp/hadoop-yarn-${user}/node-labels/ @@ -160,12 +155,40 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { ensureCloseEditlogFile(); } } + + protected void loadFromMirror(Path newMirrorPath, Path oldMirrorPath) + throws IOException { + // If mirror.new exists, read from mirror.new, + FSDataInputStream is = null; + if (fs.exists(newMirrorPath)) { + is = fs.open(newMirrorPath); + } else if (fs.exists(oldMirrorPath)) { + is = fs.open(oldMirrorPath); + } + + if (null != is) { + List labels = new AddToClusterNodeLabelsRequestPBImpl( + AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is)) + .getNodeLabels(); + mgr.addToCluserNodeLabels(labels); + + if (mgr.isCentralizedConfiguration()) { + // Only load node to labels mapping while using centralized configuration + Map> nodeToLabels = + new ReplaceLabelsOnNodeRequestPBImpl( + ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is)) + .getNodeToLabels(); + mgr.replaceLabelsOnNode(nodeToLabels); + } + is.close(); + } + } /* (non-Javadoc) * @see org.apache.hadoop.yarn.nodelabels.NodeLabelsStore#recover(boolean) */ @Override - public void recover(boolean ignoreNodeToLabelsMappings) throws YarnException, + public void recover() throws YarnException, IOException { /* * Steps of recover @@ -181,31 +204,13 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { // Open mirror from serialized file Path mirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME); Path oldMirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".old"); - - FSDataInputStream is = null; - if (fs.exists(mirrorPath)) { - is = fs.open(mirrorPath); - } else if (fs.exists(oldMirrorPath)) { - is = fs.open(oldMirrorPath); - } - - if (null != is) { - List labels = - new AddToClusterNodeLabelsRequestPBImpl( - AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is)).getNodeLabels(); - Map> nodeToLabels = - new ReplaceLabelsOnNodeRequestPBImpl( - ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is)) - .getNodeToLabels(); - mgr.addToCluserNodeLabels(labels); - mgr.replaceLabelsOnNode(nodeToLabels); - is.close(); - } + + loadFromMirror(mirrorPath, oldMirrorPath); // Open and process editlog editLogPath = new Path(fsWorkingPath, EDITLOG_FILENAME); if (fs.exists(editLogPath)) { - is = fs.open(editLogPath); + FSDataInputStream is = fs.open(editLogPath); while (true) { try { @@ -233,7 +238,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { new ReplaceLabelsOnNodeRequestPBImpl( ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is)) .getNodeToLabels(); - if (!ignoreNodeToLabelsMappings) { + if (mgr.isCentralizedConfiguration()) { /* * In case of Distributed NodeLabels setup, * ignoreNodeToLabelsMappings will be set to true and recover will http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java index 46b94fd..aacb920 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java @@ -31,11 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.exceptions.YarnException; public abstract class NodeLabelsStore implements Closeable { - protected final CommonNodeLabelsManager mgr; - - public NodeLabelsStore(CommonNodeLabelsManager mgr) { - this.mgr = mgr; - } + protected CommonNodeLabelsManager mgr; /** * Store node {@literal ->} label @@ -62,16 +58,14 @@ public abstract class NodeLabelsStore implements Closeable { * ignoreNodeToLabelsMappings will be set to true and recover will be invoked * as RM will collect the node labels from NM through registration/HB * - * @param ignoreNodeToLabelsMappings * @throws IOException * @throws YarnException */ - public abstract void recover(boolean ignoreNodeToLabelsMappings) - throws IOException, YarnException; + public abstract void recover() throws IOException, YarnException; public void init(Configuration conf) throws Exception {} - - public CommonNodeLabelsManager getNodeLabelsManager() { - return mgr; + + public void setNodeLabelsManager(CommonNodeLabelsManager mgr) { + this.mgr = mgr; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f86eab9..0917366 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2465,4 +2465,12 @@ yarn.am.blacklisting.disable-failure-threshold 0.8f + + + + Choose different implementation of node label's storage + + yarn.node-labels.fs-store.impl.class + org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore + http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java index fce663a..64c74c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java @@ -36,10 +36,10 @@ public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager { @Override public void initNodeLabelStore(Configuration conf) { - this.store = new NodeLabelsStore(this) { + this.store = new NodeLabelsStore() { @Override - public void recover(boolean ignoreNodeToLabelsMappings) + public void recover() throws IOException { } @@ -65,6 +65,8 @@ public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager { // do nothing } }; + + this.store.setNodeLabelsManager(this); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 4929f95..82e4e11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.nodelabels; import java.io.File; import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -33,13 +34,17 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.mockito.Mockito; import com.google.common.collect.ImmutableMap; -import org.mockito.Mockito; +@RunWith(Parameterized.class) public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { MockNodeLabelManager mgr = null; Configuration conf = null; + String storeClassName = null; private static class MockNodeLabelManager extends CommonNodeLabelsManager { @@ -59,8 +64,15 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { } } - private FileSystemNodeLabelsStore getStore() { - return (FileSystemNodeLabelsStore) mgr.store; + public TestFileSystemNodeLabelsStore(String className) { + this.storeClassName = className; + } + + @Parameterized.Parameters + public static Collection getParameters() { + return Arrays.asList( + new String[][] { { FileSystemNodeLabelsStore.class.getCanonicalName() }, + { NonAppendableFSNodeLabelStore.class.getCanonicalName() } }); } @Before @@ -68,6 +80,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { mgr = new MockNodeLabelManager(); conf = new Configuration(); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + conf.set(YarnConfiguration.FS_NODE_LABELS_STORE_IMPL_CLASS, storeClassName); File tempDir = File.createTempFile("nlb", ".tmp"); tempDir.delete(); tempDir.mkdirs(); @@ -80,7 +93,11 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { @After public void after() throws IOException { - getStore().fs.delete(getStore().fsWorkingPath, true); + if (mgr.store instanceof FileSystemNodeLabelsStore) { + FileSystemNodeLabelsStore fsStore = + ((FileSystemNodeLabelsStore) mgr.store); + fsStore.fs.delete(fsStore.fsWorkingPath, true); + } mgr.stop(); } @@ -324,11 +341,12 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { @Test public void testRootMkdirOnInitStore() throws Exception { final FileSystem mockFs = Mockito.mock(FileSystem.class); - FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore(mgr) { + FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore() { void setFileSystem(Configuration conf) throws IOException { fs = mockFs; } }; + mockStore.setNodeLabelsManager(mgr); mockStore.fs = mockFs; verifyMkdirsCount(mockStore, true, 0); verifyMkdirsCount(mockStore, false, 1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java index 2e21d26..bb0b45f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java @@ -37,10 +37,10 @@ public class NullRMNodeLabelsManager extends RMNodeLabelsManager { @Override public void initNodeLabelStore(Configuration conf) { - this.store = new NodeLabelsStore(this) { + this.store = new NodeLabelsStore() { @Override - public void recover(boolean ignoreNodeToLabelsMappings) + public void recover() throws IOException { // do nothing }