Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 2DDCB200CD0 for ; Tue, 25 Jul 2017 20:03:09 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 2C42B16623B; Tue, 25 Jul 2017 18:03:09 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A885B1661FD for ; Tue, 25 Jul 2017 20:03:06 +0200 (CEST) Received: (qmail 54749 invoked by uid 500); 25 Jul 2017 18:02:47 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 52669 invoked by uid 99); 25 Jul 2017 18:02:46 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Jul 2017 18:02:46 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 77A40F32F8; Tue, 25 Jul 2017 18:02:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Date: Tue, 25 Jul 2017 18:03:28 -0000 Message-Id: In-Reply-To: <672a888d91d44596bc69aa19fccf9275@git.apache.org> References: <672a888d91d44596bc69aa19fccf9275@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [46/50] [abbrv] hadoop git commit: YARN-6613. Update json validation for new native services providers. Contributed by Billie Rinaldi archived-at: Tue, 25 Jul 2017 18:03:09 -0000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java deleted file mode 100644 index bf6ee2c..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.slider.utils.SliderTestBase; -import org.junit.Test; - -import java.net.URI; - -/** - * Test slider utils. - */ -public class TestMiscSliderUtils extends SliderTestBase { - - - public static final String CLUSTER1 = "cluster1"; - - @Test - public void testPurgeTempDir() throws Throwable { - - Configuration configuration = new Configuration(); - FileSystem fs = FileSystem.get(new URI("file:///"), configuration); - SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); - Path inst = sliderFileSystem.createAppInstanceTempPath(CLUSTER1, "001"); - - assertTrue(fs.exists(inst)); - sliderFileSystem.purgeAppInstanceTempFiles(CLUSTER1); - assertFalse(fs.exists(inst)); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleAppJson.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleAppJson.java new file mode 100644 index 0000000..1700771 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleAppJson.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.conf; + +import org.apache.slider.api.resource.Application; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER; + +/** + * Names of the example configs. + */ +public final class ExampleAppJson { + + public static final String APP_JSON = "app.json"; + public static final String OVERRIDE_JSON = "app-override.json"; + public static final String DEFAULT_JSON = "default.json"; + public static final String EXTERNAL_JSON_0 = "external0.json"; + public static final String EXTERNAL_JSON_1 = "external1.json"; + public static final String EXTERNAL_JSON_2 = "external2.json"; + + public static final String PACKAGE = "/org/apache/slider/core/conf/examples/"; + + + private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON, + DEFAULT_JSON}; + + public static final List ALL_EXAMPLE_RESOURCES = new ArrayList<>(); + static { + for (String example : ALL_EXAMPLES) { + ALL_EXAMPLE_RESOURCES.add(PACKAGE + example); + } + } + + private ExampleAppJson() { + } + + static Application loadResource(String name) throws IOException { + return JSON_SER_DESER.fromResource(PACKAGE + name); + } + + public static String resourceName(String name) { + return "target/test-classes" + PACKAGE + name; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java deleted file mode 100644 index f13fbcc..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.conf; - -import org.apache.slider.api.resource.Application; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER; - -/** - * Names of the example configs. - */ -public final class ExampleConfResources { - - public static final String APP_JSON = "app.json"; - public static final String APP_RES = "app-resolved.json"; - public static final String OVERRIDE_JSON = "app-override.json"; - public static final String OVERRIDE_RES = "app-override-resolved.json"; - - public static final String PACKAGE = "/org/apache/slider/core/conf/examples/"; - - - private static final String[] ALL_EXAMPLES = {APP_JSON, APP_RES, - OVERRIDE_JSON, OVERRIDE_RES}; - - public static final List ALL_EXAMPLE_RESOURCES = new ArrayList<>(); - static { - for (String example : ALL_EXAMPLES) { - ALL_EXAMPLE_RESOURCES.add(PACKAGE + example); - } - } - - private ExampleConfResources() { - } - - static Application loadResource(String name) throws IOException { - return JSON_SER_DESER.fromResource(PACKAGE + name); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java deleted file mode 100644 index 48b0736..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.conf; - -import org.apache.slider.api.resource.Application; -import org.apache.slider.common.tools.SliderUtils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.Arrays; -import java.util.Collection; - -import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER; - -/** - * Test loading example resources. - */ -@RunWith(value = Parameterized.class) -public class TestConfTreeLoadExamples extends Assert { - private String resource; - - public TestConfTreeLoadExamples(String resource) { - this.resource = resource; - } - - @Parameterized.Parameters - public static Collection filenames() { - String[][] stringArray = new String[ExampleConfResources - .ALL_EXAMPLE_RESOURCES.size()][1]; - int i = 0; - for (String s : ExampleConfResources.ALL_EXAMPLE_RESOURCES) { - stringArray[i++][0] = s; - } - return Arrays.asList(stringArray); - } - - @Test - public void testLoadResource() throws Throwable { - try { - Application application = JSON_SER_DESER.fromResource(resource); - SliderUtils.resolve(application); - } catch (Exception e) { - throw new Exception("exception loading " + resource + ":" + e.toString()); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java index 285ddfa..5f5df70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java @@ -18,20 +18,40 @@ package org.apache.slider.core.conf; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.ConfigFile; +import org.apache.slider.api.resource.ConfigFile.TypeEnum; import org.apache.slider.api.resource.Configuration; +import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.persist.JsonSerDeser; +import org.apache.slider.util.ServiceApiUtil; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + import static org.apache.slider.api.InternalKeys.CHAOS_MONKEY_INTERVAL; import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS; import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS; import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES; -import static org.apache.slider.core.conf.ExampleConfResources.APP_JSON; -import static org.apache.slider.core.conf.ExampleConfResources.OVERRIDE_JSON; +import static org.apache.slider.core.conf.ExampleAppJson.APP_JSON; +import static org.apache.slider.core.conf.ExampleAppJson.EXTERNAL_JSON_1; +import static org.apache.slider.core.conf.ExampleAppJson.OVERRIDE_JSON; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; /** * Test global configuration resolution. @@ -42,23 +62,26 @@ public class TestConfigurationResolve extends Assert { @Test public void testOverride() throws Throwable { - - Application orig = ExampleConfResources.loadResource(OVERRIDE_JSON); + Application orig = ExampleAppJson.loadResource(OVERRIDE_JSON); Configuration global = orig.getConfiguration(); assertEquals("a", global.getProperty("g1")); assertEquals("b", global.getProperty("g2")); + assertEquals(2, global.getFiles().size()); Configuration simple = orig.getComponent("simple").getConfiguration(); assertEquals(0, simple.getProperties().size()); + assertEquals(1, simple.getFiles().size()); Configuration master = orig.getComponent("master").getConfiguration(); assertEquals("m", master.getProperty("name")); assertEquals("overridden", master.getProperty("g1")); + assertEquals(0, master.getFiles().size()); Configuration worker = orig.getComponent("worker").getConfiguration(); LOG.info("worker = {}", worker); assertEquals(3, worker.getProperties().size()); + assertEquals(0, worker.getFiles().size()); assertEquals("worker", worker.getProperty("name")); assertEquals("overridden-by-worker", worker.getProperty("g1")); @@ -66,18 +89,36 @@ public class TestConfigurationResolve extends Assert { assertEquals("1000", worker.getProperty("timeout")); // here is the resolution - SliderUtils.resolve(orig); + SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); + FileSystem mockFs = createNiceMock(FileSystem.class); + expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); + expect(sfs.buildClusterDirPath(anyObject())).andReturn( + new Path("cluster_dir_path")).anyTimes(); + replay(sfs, mockFs); + ServiceApiUtil.validateAndResolveApplication(orig, sfs); global = orig.getConfiguration(); LOG.info("global = {}", global); assertEquals("a", global.getProperty("g1")); assertEquals("b", global.getProperty("g2")); + assertEquals(2, global.getFiles().size()); simple = orig.getComponent("simple").getConfiguration(); assertEquals(2, simple.getProperties().size()); assertEquals("a", simple.getProperty("g1")); assertEquals("b", simple.getProperty("g2")); - + assertEquals(2, simple.getFiles().size()); + + Set files = new HashSet<>(); + Map props = new HashMap<>(); + props.put("k1", "overridden"); + props.put("k2", "v2"); + files.add(new ConfigFile().destFile("file1").type(TypeEnum + .PROPERTIES).props(props)); + files.add(new ConfigFile().destFile("file2").type(TypeEnum + .XML).props(Collections.singletonMap("k3", "v3"))); + assertTrue(files.contains(simple.getFiles().get(0))); + assertTrue(files.contains(simple.getFiles().get(1))); master = orig.getComponent("master").getConfiguration(); LOG.info("master = {}", master); @@ -85,6 +126,17 @@ public class TestConfigurationResolve extends Assert { assertEquals("m", master.getProperty("name")); assertEquals("overridden", master.getProperty("g1")); assertEquals("b", master.getProperty("g2")); + assertEquals(2, master.getFiles().size()); + + props.put("k1", "v1"); + files.clear(); + files.add(new ConfigFile().destFile("file1").type(TypeEnum + .PROPERTIES).props(props)); + files.add(new ConfigFile().destFile("file2").type(TypeEnum + .XML).props(Collections.singletonMap("k3", "v3"))); + + assertTrue(files.contains(master.getFiles().get(0))); + assertTrue(files.contains(master.getFiles().get(1))); worker = orig.getComponent("worker").getConfiguration(); LOG.info("worker = {}", worker); @@ -94,13 +146,91 @@ public class TestConfigurationResolve extends Assert { assertEquals("overridden-by-worker", worker.getProperty("g1")); assertEquals("b", worker.getProperty("g2")); assertEquals("1000", worker.getProperty("timeout")); + assertEquals(2, worker.getFiles().size()); + assertTrue(files.contains(worker.getFiles().get(0))); + assertTrue(files.contains(worker.getFiles().get(1))); } @Test - public void testTimeIntervalLoading() throws Throwable { + public void testOverrideExternalConfiguration() throws IOException { + Application orig = ExampleAppJson.loadResource(EXTERNAL_JSON_1); + + Configuration global = orig.getConfiguration(); + assertEquals(0, global.getProperties().size()); + + assertEquals(3, orig.getComponents().size()); - Application orig = ExampleConfResources.loadResource(APP_JSON); + Configuration simple = orig.getComponent("simple").getConfiguration(); + assertEquals(0, simple.getProperties().size()); + + Configuration master = orig.getComponent("master").getConfiguration(); + assertEquals(1, master.getProperties().size()); + assertEquals("is-overridden", master.getProperty("g3")); + + Configuration other = orig.getComponent("other").getConfiguration(); + assertEquals(0, other.getProperties().size()); + + // load the external application + SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); + FileSystem mockFs = createNiceMock(FileSystem.class); + expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); + expect(sfs.buildClusterDirPath(anyObject())).andReturn( + new Path("cluster_dir_path")).anyTimes(); + replay(sfs, mockFs); + Application ext = ExampleAppJson.loadResource(APP_JSON); + ServiceApiUtil.validateAndResolveApplication(ext, sfs); + reset(sfs, mockFs); + + // perform the resolution on original application + JsonSerDeser jsonSerDeser = createNiceMock(JsonSerDeser + .class); + expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); + expect(sfs.buildClusterDirPath(anyObject())).andReturn( + new Path("cluster_dir_path")).anyTimes(); + expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext) + .anyTimes(); + replay(sfs, mockFs, jsonSerDeser); + ServiceApiUtil.setJsonSerDeser(jsonSerDeser); + ServiceApiUtil.validateAndResolveApplication(orig, sfs); + + global = orig.getConfiguration(); + assertEquals(0, global.getProperties().size()); + + assertEquals(4, orig.getComponents().size()); + + simple = orig.getComponent("simple").getConfiguration(); + assertEquals(3, simple.getProperties().size()); + assertEquals("a", simple.getProperty("g1")); + assertEquals("b", simple.getProperty("g2")); + assertEquals("60", + simple.getProperty("internal.chaos.monkey.interval.seconds")); + + master = orig.getComponent("master").getConfiguration(); + assertEquals(5, master.getProperties().size()); + assertEquals("512M", master.getProperty("jvm.heapsize")); + assertEquals("overridden", master.getProperty("g1")); + assertEquals("b", master.getProperty("g2")); + assertEquals("is-overridden", master.getProperty("g3")); + assertEquals("60", + simple.getProperty("internal.chaos.monkey.interval.seconds")); + + Configuration worker = orig.getComponent("worker").getConfiguration(); + LOG.info("worker = {}", worker); + assertEquals(4, worker.getProperties().size()); + assertEquals("512M", worker.getProperty("jvm.heapsize")); + assertEquals("overridden-by-worker", worker.getProperty("g1")); + assertEquals("b", worker.getProperty("g2")); + assertEquals("60", + worker.getProperty("internal.chaos.monkey.interval.seconds")); + + other = orig.getComponent("other").getConfiguration(); + assertEquals(0, other.getProperties().size()); + } + + @Test + public void testTimeIntervalLoading() throws Throwable { + Application orig = ExampleAppJson.loadResource(APP_JSON); Configuration conf = orig.getConfiguration(); long s = conf.getPropertyLong( http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestExampleAppJson.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestExampleAppJson.java new file mode 100644 index 0000000..09096d0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestExampleAppJson.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.conf; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.slider.api.resource.Application; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.util.ServiceApiUtil; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; + +import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + +/** + * Test loading example resources. + */ +@RunWith(value = Parameterized.class) +public class TestExampleAppJson extends Assert { + private String resource; + + public TestExampleAppJson(String resource) { + this.resource = resource; + } + + @Parameterized.Parameters + public static Collection filenames() { + String[][] stringArray = new String[ExampleAppJson + .ALL_EXAMPLE_RESOURCES.size()][1]; + int i = 0; + for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) { + stringArray[i++][0] = s; + } + return Arrays.asList(stringArray); + } + + @Test + public void testLoadResource() throws Throwable { + try { + Application application = JSON_SER_DESER.fromResource(resource); + + SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); + FileSystem mockFs = createNiceMock(FileSystem.class); + expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); + expect(sfs.buildClusterDirPath(anyObject())).andReturn( + new Path("cluster_dir_path")).anyTimes(); + replay(sfs, mockFs); + + ServiceApiUtil.validateAndResolveApplication(application, sfs); + } catch (Exception e) { + throw new Exception("exception loading " + resource + ":" + e.toString()); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java new file mode 100644 index 0000000..162d34c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestAbstractClientProvider.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.ConfigFile; +import org.apache.slider.api.resource.ConfigFile.TypeEnum; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + +/** + * Test the AbstractClientProvider shared methods. + */ +public class TestAbstractClientProvider { + private static final String EXCEPTION_PREFIX = "Should have thrown " + + "exception: "; + private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " + + "exception: "; + + private static class ClientProvider extends AbstractClientProvider { + @Override + public void validateArtifact(Artifact artifact, FileSystem fileSystem) + throws IOException { + } + + @Override + protected void validateConfigFile(ConfigFile configFile, + FileSystem fileSystem) throws IOException { + } + } + + @Test + public void testConfigFiles() throws IOException { + ClientProvider clientProvider = new ClientProvider(); + FileSystem mockFs = createNiceMock(FileSystem.class); + expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes(); + replay(mockFs); + + ConfigFile configFile = new ConfigFile(); + List configFiles = new ArrayList<>(); + configFiles.add(configFile); + + try { + clientProvider.validateConfigFiles(configFiles, mockFs); + Assert.fail(EXCEPTION_PREFIX + "null file type"); + } catch (IllegalArgumentException e) { + } + + configFile.setType(TypeEnum.TEMPLATE); + try { + clientProvider.validateConfigFiles(configFiles, mockFs); + Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template"); + } catch (IllegalArgumentException e) { + } + + configFile.setSrcFile("srcfile"); + try { + clientProvider.validateConfigFiles(configFiles, mockFs); + Assert.fail(EXCEPTION_PREFIX + "empty dest file"); + } catch (IllegalArgumentException e) { + } + + configFile.setDestFile("destfile"); + try { + clientProvider.validateConfigFiles(configFiles, mockFs); + } catch (IllegalArgumentException e) { + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + configFile = new ConfigFile(); + configFile.setType(TypeEnum.JSON); + configFile.setSrcFile(null); + configFile.setDestFile("path/destfile2"); + configFiles.add(configFile); + try { + clientProvider.validateConfigFiles(configFiles, mockFs); + Assert.fail(EXCEPTION_PREFIX + "dest file with multiple path elements"); + } catch (IllegalArgumentException e) { + } + + configFile.setDestFile("/path/destfile2"); + try { + clientProvider.validateConfigFiles(configFiles, mockFs); + } catch (IllegalArgumentException e) { + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + configFile.setDestFile("destfile"); + try { + clientProvider.validateConfigFiles(configFiles, mockFs); + Assert.fail(EXCEPTION_PREFIX + "duplicate dest file"); + } catch (IllegalArgumentException e) { + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java new file mode 100644 index 0000000..6df660d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestBuildApplicationComponent.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.slider.api.resource.Component; +import org.apache.slider.client.SliderClient; +import org.apache.slider.common.params.SliderActions; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.conf.ExampleAppJson; +import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.util.ServiceApiUtil; +import org.apache.slider.utils.YarnZKMiniClusterTestBase; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.slider.common.params.Arguments.ARG_APPDEF; + +/** + * Test for building / resolving components of type APPLICATION. + */ +public class TestBuildApplicationComponent extends YarnZKMiniClusterTestBase { + + private static void checkComponentNames(List components, + Set names) { + assertEquals(names.size(), components.size()); + for (Component comp : components) { + assertTrue(names.contains(comp.getName())); + } + } + + public void buildAndCheckComponents(String appName, String appDef, + SliderFileSystem sfs, Set names) throws Throwable { + ServiceLauncher launcher = createOrBuildCluster( + SliderActions.ACTION_BUILD, appName, Arrays.asList(ARG_APPDEF, + ExampleAppJson.resourceName(appDef)), true, false); + SliderClient sliderClient = launcher.getService(); + addToTeardown(sliderClient); + + // verify the cluster exists + assertEquals(0, sliderClient.actionExists(appName, false)); + // verify generated conf + List components = ServiceApiUtil.getApplicationComponents(sfs, + appName); + checkComponentNames(components, names); + } + + @Test + public void testExternalComponentBuild() throws Throwable { + String clustername = createMiniCluster("", getConfiguration(), 1, true); + + describe("verify external components"); + + SliderFileSystem sfs = createSliderFileSystem(); + + Set nameSet = new HashSet<>(); + nameSet.add("simple"); + nameSet.add("master"); + nameSet.add("worker"); + + buildAndCheckComponents("app-1", ExampleAppJson.APP_JSON, sfs, + nameSet); + buildAndCheckComponents("external-0", ExampleAppJson + .EXTERNAL_JSON_0, sfs, nameSet); + + nameSet.add("other"); + + buildAndCheckComponents("external-1", ExampleAppJson + .EXTERNAL_JSON_1, sfs, nameSet); + + nameSet.add("another"); + + buildAndCheckComponents("external-2", ExampleAppJson + .EXTERNAL_JSON_2, sfs, nameSet); + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java new file mode 100644 index 0000000..f1afe67 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestDefaultProvider.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.slider.api.resource.Application; +import org.apache.slider.client.SliderClient; +import org.apache.slider.common.params.SliderActions; +import org.apache.slider.core.conf.ExampleAppJson; +import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.utils.YarnZKMiniClusterTestBase; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Arrays; + +import static org.apache.slider.common.params.Arguments.ARG_APPDEF; + +/** + * Simple end-to-end test. + */ +public class TestDefaultProvider extends YarnZKMiniClusterTestBase { + + // TODO figure out how to run client commands against minicluster + // (currently errors out unable to find containing jar of AM for upload) + @Ignore + @Test + public void testDefaultProvider() throws Throwable { + createMiniCluster("", getConfiguration(), 1, true); + String appName = "default-1"; + + describe("verify default provider"); + + String appDef = ExampleAppJson.resourceName(ExampleAppJson + .DEFAULT_JSON); + + ServiceLauncher launcher = createOrBuildCluster( + SliderActions.ACTION_CREATE, appName, Arrays.asList(ARG_APPDEF, + appDef), true, true); + SliderClient sliderClient = launcher.getService(); + addToTeardown(sliderClient); + + Application application = sliderClient.actionStatus(appName); + assertEquals(1L, application.getContainers().size()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java index c1f2886..6f4ca42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java @@ -43,7 +43,7 @@ public class BaseMockAppStateAATest extends BaseMockAppStateTest @Override public Application buildApplication() { Application application = factory.newApplication(0, 0, 0) - .name(getTestName()); + .name(getValidTestName()); application.getComponent(ROLE1).getConfiguration().setProperty( COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy .ANTI_AFFINITY_REQUIRED)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java index eb25b40..571e9d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java @@ -362,7 +362,7 @@ public class TestMockAppStateAAPlacement extends BaseMockAppStateAATest // now destroy the app state AppStateBindingInfo bindingInfo = buildBindingInfo(); bindingInfo.application = factory.newApplication(0, 0, desiredAA).name( - getTestName()); + getValidTestName()); bindingInfo.application.getComponent(ROLE2) .getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.ANTI_AFFINITY_REQUIRED)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java index ea0dcf4..9cbda4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java @@ -203,7 +203,7 @@ public class TestMockAppStateContainerFailure extends BaseMockAppStateTest // Update instance definition to allow containers to fail any number of // times AppStateBindingInfo bindingInfo = buildBindingInfo(); - bindingInfo.application.getConfiguration().setProperty( + bindingInfo.application.getComponent(ROLE0).getConfiguration().setProperty( ResourceKeys.CONTAINER_FAILURE_THRESHOLD, "0"); appState = new MockAppState(bindingInfo); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java index 6d8e963..7f7f93a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java @@ -38,6 +38,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.util.Collections; /** @@ -65,7 +66,7 @@ public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest } @Override - public AppStateBindingInfo buildBindingInfo() { + public AppStateBindingInfo buildBindingInfo() throws IOException { AppStateBindingInfo bindingInfo = super.buildBindingInfo(); bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector(); return bindingInfo; @@ -145,7 +146,7 @@ public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest appState = new MockAppState(); AppStateBindingInfo binding2 = buildBindingInfo(); binding2.application = factory.newApplication(0, 0, 0) - .name(getTestName()); + .name(getValidTestName()); binding2.historyPath = historyPath2; appState.buildInstance(binding2); // on this read there won't be the right number of roles http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java index b0634bf..d9c675d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java @@ -70,7 +70,7 @@ public class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest AppStateBindingInfo bindingInfo = buildBindingInfo(); bindingInfo.application = factory.newApplication(r0, r1, r2) - .name(getTestName()); + .name(getValidTestName()); bindingInfo.liveContainers = containers; appState = new MockAppState(bindingInfo); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java index b7e967f..703d65f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java @@ -30,6 +30,7 @@ import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.RoleStatus; import org.junit.Test; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -59,7 +60,7 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest } @Override - public AppStateBindingInfo buildBindingInfo() { + public AppStateBindingInfo buildBindingInfo() throws IOException { AppStateBindingInfo bindingInfo = super.buildBindingInfo(); bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector(); return bindingInfo; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java index d382c8a..4aa5895 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java @@ -40,7 +40,7 @@ public class TestMockContainerResourceAllocations extends BaseMockAppStateTest { @Override public Application buildApplication() { - return factory.newApplication(1, 0, 0).name(getTestName()); + return factory.newApplication(1, 0, 0).name(getValidTestName()); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java index 69abccf..5af87f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java @@ -51,6 +51,7 @@ import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.util.ServiceApiUtil; import org.apache.slider.utils.SliderTestBase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,6 +63,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map.Entry; /** @@ -118,7 +120,7 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements historyPath = new Path(historyWorkDir.toURI()); fs.delete(historyPath, true); appState = new MockAppState(buildBindingInfo()); - stateAccess = new ProviderAppState(getTestName(), appState); + stateAccess = new ProviderAppState(getValidTestName(), appState); } /** @@ -127,9 +129,11 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements * from {@link #buildApplication()} ()} * @return */ - protected AppStateBindingInfo buildBindingInfo() { + protected AppStateBindingInfo buildBindingInfo() throws IOException { AppStateBindingInfo binding = new AppStateBindingInfo(); binding.application = buildApplication(); + ServiceApiUtil.validateAndResolveApplication(binding.application, + sliderFileSystem); //binding.roles = new ArrayList<>(factory.ROLES); binding.fs = fs; binding.historyPath = historyPath; @@ -142,7 +146,7 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements * @return the instance definition */ public Application buildApplication() { - return factory.newApplication(0, 0, 0).name(getTestName()); + return factory.newApplication(0, 0, 0).name(getValidTestName()); } /** @@ -153,6 +157,10 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements return methodName.getMethodName(); } + public String getValidTestName() { + return getTestName().toLowerCase(Locale.ENGLISH); + } + public RoleStatus getRole0Status() { return lookupRole(ROLE0); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java index 2ac5087..8785b92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.Resource; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; @@ -190,6 +191,8 @@ public class MockFactory implements MockRoles { */ public Application newApplication(long r1, long r2, long r3) { Application application = new Application(); + application.setLaunchCommand("sleep 60"); + application.setResource(new Resource().memory("256")); application.getConfiguration().setProperty(ResourceKeys .NODE_FAILURE_THRESHOLD, Integer.toString(NODE_FAILURE_THRESHOLD)); List components = application.getComponents(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java new file mode 100644 index 0000000..9ca3242 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.utils; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.Resource; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.persist.JsonSerDeser; +import org.apache.slider.util.RestApiConstants; +import org.apache.slider.util.RestApiErrorMessages; +import org.apache.slider.util.ServiceApiUtil; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +import static org.apache.slider.util.RestApiConstants.DEFAULT_COMPONENT_NAME; +import static org.apache.slider.util.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME; +import static org.apache.slider.util.RestApiErrorMessages.*; +import static org.apache.slider.util.RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID; +import static org.apache.slider.util.RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Test for ServiceApiUtil helper methods. + */ +public class TestServiceApiUtil { + private static final Logger LOG = LoggerFactory + .getLogger(TestServiceApiUtil.class); + private static final String EXCEPTION_PREFIX = "Should have thrown " + + "exception: "; + private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " + + "exception: "; + + @Test(timeout = 90000) + public void testResourceValidation() throws Exception { + SliderFileSystem sfs = initMock(null); + + Application app = new Application(); + + // no name + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no name"); + } catch (IllegalArgumentException e) { + assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); + } + + // bad format name + String[] badNames = {"4finance", "Finance", "finance@home"}; + for (String badName : badNames) { + app.setName(badName); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName); + } catch (IllegalArgumentException e) { + assertEquals(String.format( + ERROR_APPLICATION_NAME_INVALID_FORMAT, badName), e.getMessage()); + } + } + + // launch command not specified + app.setName("finance_home"); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no launch command"); + } catch (IllegalArgumentException e) { + assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND, + e.getMessage()); + } + + // resource not specified + app.setLaunchCommand("sleep 3600"); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no resource"); + } catch (IllegalArgumentException e) { + assertEquals(String.format( + RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID, + RestApiConstants.DEFAULT_COMPONENT_NAME), e.getMessage()); + } + + // memory not specified + Resource res = new Resource(); + app.setResource(res); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no memory"); + } catch (IllegalArgumentException e) { + assertEquals(String.format( + RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, + RestApiConstants.DEFAULT_COMPONENT_NAME), e.getMessage()); + } + + // invalid no of cpus + res.setMemory("100mb"); + res.setCpus(-2); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail( + EXCEPTION_PREFIX + "application with invalid no of cpus"); + } catch (IllegalArgumentException e) { + assertEquals(String.format( + RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, + RestApiConstants.DEFAULT_COMPONENT_NAME), e.getMessage()); + } + + // number of containers not specified + res.setCpus(2); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no container count"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage() + .contains(ERROR_CONTAINERS_COUNT_INVALID)); + } + + // specifying profile along with cpus/memory raises exception + res.setProfile("hbase_finance_large"); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + + "application with resource profile along with cpus/memory"); + } catch (IllegalArgumentException e) { + assertEquals(String.format(RestApiErrorMessages + .ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED, + RestApiConstants.DEFAULT_COMPONENT_NAME), + e.getMessage()); + } + + // currently resource profile alone is not supported. + // TODO: remove the next test once resource profile alone is supported. + res.setCpus(null); + res.setMemory(null); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with resource profile only"); + } catch (IllegalArgumentException e) { + assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET, + e.getMessage()); + } + + // unset profile here and add cpus/memory back + res.setProfile(null); + res.setCpus(2); + res.setMemory("2gb"); + + // null number of containers + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "null number of containers"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage() + .startsWith(ERROR_CONTAINERS_COUNT_INVALID)); + } + + // negative number of containers + app.setNumberOfContainers(-1L); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "negative number of containers"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage() + .startsWith(ERROR_CONTAINERS_COUNT_INVALID)); + } + + // everything valid here + app.setNumberOfContainers(5L); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + } catch (IllegalArgumentException e) { + LOG.error("application attributes specified should be valid here", e); + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + } + + @Test + public void testArtifacts() throws IOException { + SliderFileSystem sfs = initMock(null); + + Application app = new Application(); + app.setName("name"); + Resource res = new Resource(); + app.setResource(res); + res.setMemory("512M"); + app.setNumberOfContainers(3L); + + // no artifact id fails with default type + Artifact artifact = new Artifact(); + app.setArtifact(artifact); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); + } catch (IllegalArgumentException e) { + assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); + } + + // no artifact id fails with APPLICATION type + artifact.setType(Artifact.TypeEnum.APPLICATION); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); + } catch (IllegalArgumentException e) { + assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); + } + + // no artifact id fails with TARBALL type + artifact.setType(Artifact.TypeEnum.TARBALL); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); + } catch (IllegalArgumentException e) { + assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); + } + + // everything valid here + artifact.setType(Artifact.TypeEnum.DOCKER); + artifact.setId("docker.io/centos:centos7"); + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + } catch (IllegalArgumentException e) { + LOG.error("application attributes specified should be valid here", e); + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + // defaults assigned + assertEquals(app.getComponents().get(0).getName(), + DEFAULT_COMPONENT_NAME); + assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME); + } + + private static Resource createValidResource() { + Resource res = new Resource(); + res.setMemory("512M"); + return res; + } + + private static Component createValidComponent(String compName) { + Component comp = new Component(); + comp.setName(compName); + comp.setResource(createValidResource()); + comp.setNumberOfContainers(1L); + return comp; + } + + private static Application createValidApplication(String compName) { + Application app = new Application(); + app.setLaunchCommand("sleep 3600"); + app.setName("name"); + app.setResource(createValidResource()); + app.setNumberOfContainers(1L); + if (compName != null) { + app.addComponent(createValidComponent(compName)); + } + return app; + } + + private static SliderFileSystem initMock(Application ext) throws IOException { + SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); + FileSystem mockFs = createNiceMock(FileSystem.class); + JsonSerDeser jsonSerDeser = createNiceMock(JsonSerDeser + .class); + expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); + expect(sfs.buildClusterDirPath(anyObject())).andReturn( + new Path("cluster_dir_path")).anyTimes(); + if (ext != null) { + expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext) + .anyTimes(); + } + replay(sfs, mockFs, jsonSerDeser); + ServiceApiUtil.setJsonSerDeser(jsonSerDeser); + return sfs; + } + + @Test + public void testExternalApplication() throws IOException { + Application ext = createValidApplication("comp1"); + SliderFileSystem sfs = initMock(ext); + + Application app = createValidApplication(null); + + Artifact artifact = new Artifact(); + artifact.setType(Artifact.TypeEnum.APPLICATION); + artifact.setId("id"); + app.setArtifact(artifact); + + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + } catch (IllegalArgumentException e) { + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + assertEquals(1, app.getComponents().size()); + assertNotNull(app.getComponent("comp1")); + } + + @Test + public void testDuplicateComponents() throws IOException { + SliderFileSystem sfs = initMock(null); + + String compName = "comp1"; + Application app = createValidApplication(compName); + app.addComponent(createValidComponent(compName)); + + // duplicate component name fails + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + Assert.fail(EXCEPTION_PREFIX + "application with component collision"); + } catch (IllegalArgumentException e) { + assertEquals("Component name collision: " + compName, e.getMessage()); + } + } + + @Test + public void testExternalDuplicateComponent() throws IOException { + Application ext = createValidApplication("comp1"); + SliderFileSystem sfs = initMock(ext); + + Application app = createValidApplication("comp1"); + Artifact artifact = new Artifact(); + artifact.setType(Artifact.TypeEnum.APPLICATION); + artifact.setId("id"); + app.getComponent("comp1").setArtifact(artifact); + + // duplicate component name okay in the case of APPLICATION component + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + } catch (IllegalArgumentException e) { + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + } + + @Test + public void testExternalComponent() throws IOException { + Application ext = createValidApplication("comp1"); + SliderFileSystem sfs = initMock(ext); + + Application app = createValidApplication("comp2"); + Artifact artifact = new Artifact(); + artifact.setType(Artifact.TypeEnum.APPLICATION); + artifact.setId("id"); + app.setArtifact(artifact); + + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + } catch (IllegalArgumentException e) { + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + assertEquals(1, app.getComponents().size()); + // artifact ID not inherited from global + assertNotNull(app.getComponent("comp2")); + + // set APPLICATION artifact id on component + app.getComponent("comp2").setArtifact(artifact); + + try { + ServiceApiUtil.validateAndResolveApplication(app, sfs); + } catch (IllegalArgumentException e) { + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + assertEquals(1, app.getComponents().size()); + // original component replaced by external component + assertNotNull(app.getComponent("comp1")); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java index 746a0ec..5e62fc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnMiniClusterTestBase.java @@ -37,6 +37,7 @@ import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.params.SliderActions; import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; @@ -328,11 +329,8 @@ public abstract class YarnMiniClusterTestBase extends SliderTestBase { */ public void stopRunningClusters() { for (SliderClient client : clustersToTeardown) { - try { - maybeStopCluster(client, "", "Teardown at end of test case", true); - } catch (Exception e) { - LOG.warn("While stopping cluster " + e, e); - } + maybeStopCluster(client, client.getDeployedClusterName(), + "Teardown at end of test case", true); } } @@ -502,6 +500,62 @@ public abstract class YarnMiniClusterTestBase extends SliderTestBase { } /** + * Create or build a cluster (the action is set by the first verb). + * @param action operation to invoke: SliderActions.ACTION_CREATE or + * SliderActions.ACTION_BUILD + * @param clustername cluster name + * @param extraArgs list of extra args to add to the creation command + * @param deleteExistingData should the data of any existing cluster + * of this name be deleted + * @param blockUntilRunning block until the AM is running + * @return launcher which will have executed the command. + */ + public ServiceLauncher createOrBuildCluster(String action, + String clustername, List extraArgs, boolean deleteExistingData, + boolean blockUntilRunning) throws Throwable { + assertNotNull(clustername); + assertNotNull(miniCluster); + // update action should keep existing data + Configuration config = miniCluster.getConfig(); + if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) { + FileSystem dfs = FileSystem.get(new URI(getFsDefaultName()), config); + + SliderFileSystem sliderFileSystem = new SliderFileSystem(dfs, config); + Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername); + LOG.info("deleting instance data at {}", clusterDir); + //this is a safety check to stop us doing something stupid like deleting / + assertTrue(clusterDir.toString().contains("/.slider/")); + rigorousDelete(sliderFileSystem, clusterDir, 60000); + } + + + List argsList = new ArrayList<>(); + argsList.addAll(Arrays.asList( + action, clustername, + Arguments.ARG_MANAGER, getRMAddr(), + Arguments.ARG_FILESYSTEM, getFsDefaultName(), + Arguments.ARG_DEBUG)); + + argsList.addAll(getExtraCLIArgs()); + + if (extraArgs != null) { + argsList.addAll(extraArgs); + } + ServiceLauncher launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(config), + //varargs list of command line params + argsList + ); + assertEquals(0, launcher.getServiceExitCode()); + SliderClient client = launcher.getService(); + if (blockUntilRunning) { + client.monitorAppToRunning(new Duration(CLUSTER_GO_LIVE_TIME)); + } + return launcher; + } + + /** * Delete with some pauses and backoff; designed to handle slow delete * operation in windows. */ @@ -652,28 +706,6 @@ public abstract class YarnMiniClusterTestBase extends SliderTestBase { return getTestConfiguration().getTrimmed(getApplicationHomeKey()); } - public List getImageCommands() { - if (switchToImageDeploy) { - // its an image that had better be defined - assertNotNull(getArchivePath()); - if (!imageIsRemote) { - // its not remote, so assert it exists - File f = new File(getArchivePath()); - assertTrue(f.exists()); - return Arrays.asList(Arguments.ARG_IMAGE, f.toURI().toString()); - } else { - assertNotNull(remoteImageURI); - - // if it is remote, then its whatever the archivePath property refers to - return Arrays.asList(Arguments.ARG_IMAGE, remoteImageURI.toString()); - } - } else { - assertNotNull(getApplicationHome()); - assertTrue(new File(getApplicationHome()).exists()); - return Arrays.asList(Arguments.ARG_APP_HOME, getApplicationHome()); - } - } - /** * Get the resource configuration dir in the source tree. * @@ -746,14 +778,23 @@ public abstract class YarnMiniClusterTestBase extends SliderTestBase { SliderClient sliderClient, String clustername, String message, - boolean force) throws IOException, YarnException { + boolean force) { if (sliderClient != null) { if (SliderUtils.isUnset(clustername)) { clustername = sliderClient.getDeployedClusterName(); } //only stop a cluster that exists if (SliderUtils.isSet(clustername)) { - return clusterActionFreeze(sliderClient, clustername, message, force); + try { + clusterActionFreeze(sliderClient, clustername, message, force); + } catch (Exception e) { + LOG.warn("While stopping cluster " + e, e); + } + try { + sliderClient.actionDestroy(clustername); + } catch (Exception e) { + LOG.warn("While destroying cluster " + e, e); + } } } return 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java index 322b346..cf9e616 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/YarnZKMiniClusterTestBase.java @@ -22,7 +22,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.zk.BlockingZKWatcher; import org.apache.slider.core.zk.ZKIntegration; import org.slf4j.Logger; @@ -109,9 +108,7 @@ public abstract class YarnZKMiniClusterTestBase extends int numLogDirs, boolean startZK, boolean startHDFS) throws IOException { - if (SliderUtils.isUnset(name)) { - name = methodName.getMethodName(); - } + name = buildClustername(name); createMicroZKCluster("-" + name, conf); conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true); conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, getZKBinding()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json deleted file mode 100644 index e2a21ea..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override-resolved.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "app-1", - "lifetime": "3600", - "configuration": { - "properties": { - "g1": "a", - "g2": "b" - } - }, - "resource": { - "cpus": 1, - "memory": "512" - }, - "number_of_containers": 2, - "components": [ - { - "name": "simple", - "configuration": { - "properties": { - "g1": "a", - "g2": "b" - } - } - }, - { - "name": "master", - "configuration": { - "properties": { - "g1": "overridden", - "g2": "b" - } - } - }, - { - "name": "worker", - "resource": { - "cpus": 1, - "memory": "1024" - }, - "configuration": { - "properties": { - "g1": "overridden-by-worker", - "g2": "b", - "timeout": "1000" - } - } - } - ] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json index 552cdef..d7e2fd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-override.json @@ -1,11 +1,29 @@ { "name": "app-1", "lifetime": "3600", + "launch_command": "sleep 3600", "configuration": { "properties": { "g1": "a", "g2": "b" - } + }, + "files": [ + { + "type": "PROPERTIES", + "dest_file": "file1", + "props": { + "k1": "v1", + "k2": "v2" + } + }, + { + "type": "XML", + "dest_file": "file2", + "props": { + "k3": "v3" + } + } + ] }, "resource": { "cpus": 1, @@ -14,7 +32,18 @@ "number_of_containers": 2, "components": [ { - "name": "simple" + "name": "simple", + "configuration": { + "files": [ + { + "type": "PROPERTIES", + "dest_file": "file1", + "props": { + "k1": "overridden" + } + } + ] + } }, { "name": "master", http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json deleted file mode 100644 index cd1ab6f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app-resolved.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "name": "zk-app-1", - "lifetime": "3600", - "configuration": { - "properties": { - "internal.chaos.monkey.interval.seconds": "60", - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0" - } - }, - "resource": { - "cpus": 1, - "memory": "512" - }, - "number_of_containers": 2, - "components": [ - { - "name": "simple", - "number_of_containers": 2, - "configuration": { - "properties": { - "g1": "a", - "g2": "b" - } - } - }, - { - "name": "master", - "number_of_containers": 1, - "resource": { - "cpus": 1, - "memory": "512" - }, - "configuration": { - "properties": { - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0", - "jvm.heapsize": "512M" - } - } - }, - { - "name": "worker", - "number_of_containers": 5, - "resource": { - "cpus": 1, - "memory": "1024" - }, - "configuration": { - "properties": { - "g1": "overridden-by-worker", - "g2": "b", - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0", - "jvm.heapsize": "512M" - } - } - } - ] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57dddc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json index 90857db..b1d73c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/core/conf/examples/app.json @@ -1,20 +1,12 @@ { "name": "app-1", "lifetime": "3600", + "launch_command": "sleep 3600", "configuration": { "properties": { "g1": "a", "g2": "b", - "internal.chaos.monkey.interval.seconds": "60", - "zookeeper.port": "2181", - "zookeeper.path": "/yarnapps_small_cluster", - "zookeeper.hosts": "zoo1,zoo2,zoo3", - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.fs.defaultFS": "hdfs://cluster:8020", - "site.fs.default.name": "hdfs://cluster:8020", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.info.port": "0" + "internal.chaos.monkey.interval.seconds": "60" } }, "resource": { @@ -32,6 +24,7 @@ "configuration": { "properties": { "g1": "overridden", + "g3": "will-be-overridden", "jvm.heapsize": "512M" } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org