Return-Path: X-Original-To: apmail-myriad-commits-archive@minotaur.apache.org Delivered-To: apmail-myriad-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A0D6018ACD for ; Wed, 28 Oct 2015 16:07:59 +0000 (UTC) Received: (qmail 87703 invoked by uid 500); 28 Oct 2015 16:07:59 -0000 Delivered-To: apmail-myriad-commits-archive@myriad.apache.org Received: (qmail 87675 invoked by uid 500); 28 Oct 2015 16:07:59 -0000 Mailing-List: contact commits-help@myriad.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@myriad.incubator.apache.org Delivered-To: mailing list commits@myriad.incubator.apache.org Received: (qmail 87666 invoked by uid 99); 28 Oct 2015 16:07:59 -0000 Received: from Unknown (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 28 Oct 2015 16:07:59 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id ED89C181181 for ; Wed, 28 Oct 2015 16:07:58 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 0.79 X-Spam-Level: X-Spam-Status: No, score=0.79 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, T_RP_MATCHES_RCVD=-0.01] autolearn=disabled Received: from mx1-eu-west.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id Bipz2LFlKZde for ; Wed, 28 Oct 2015 16:07:40 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-eu-west.apache.org (ASF Mail Server at mx1-eu-west.apache.org) with SMTP id 7E27E256F7 for ; Wed, 28 Oct 2015 16:07:34 +0000 (UTC) Received: (qmail 86212 invoked by uid 99); 28 Oct 2015 16:07:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 28 Oct 2015 16:07:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8643FE0921; Wed, 28 Oct 2015 16:07:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: smarella@apache.org To: commits@myriad.incubator.apache.org Date: Wed, 28 Oct 2015 16:07:41 -0000 Message-Id: <86b4974fc50b45688a8541fc7ee95766@git.apache.org> In-Reply-To: <9e0979bdf7b245518e53d541fb7fbe68@git.apache.org> References: <9e0979bdf7b245518e53d541fb7fbe68@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [09/20] incubator-myriad git commit: com.ebay => org.apache http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java deleted file mode 100644 index 3567c3f..0000000 --- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.scheduler; - -import com.ebay.myriad.scheduler.yarn.MyriadFairScheduler; -import java.io.IOException; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.AsyncDispatcher; -//import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -//import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -//import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; -//import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; -//import org.apache.hadoop.yarn.util.resource.Resources; -import org.junit.After; -import org.junit.Before; -//import org.junit.Test; - -//import static org.junit.Assert.assertEquals; - -/** - * Tests myriad scheduler. - */ -public class TestMyriadScheduler { - protected Configuration conf; - protected FairScheduler scheduler; - protected ResourceManager resourceManager; - - @Before - public void setUp() throws IOException { - scheduler = new MyriadFairScheduler(); - conf = createConfiguration(); - resourceManager = new ResourceManager(); - resourceManager.init(conf); - - // TODO: This test should really be using MockRM. For now starting stuff - // that is needed at a bare minimum. - ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start(); - resourceManager.getRMContext().getStateStore().start(); - - // to initialize the master key - resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); - - scheduler.setRMContext(resourceManager.getRMContext()); - scheduler.init(conf); - scheduler.start(); - } - - @After - public void tearDown() { - if (scheduler != null) { - scheduler.stop(); - scheduler = null; - } - - if (resourceManager != null) { - resourceManager.stop(); - resourceManager = null; - } - } - - public Configuration createConfiguration() { - Configuration conf = new YarnConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); - conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1024); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); - return conf; - } -/* - @Test - public void testClusterMemory() throws Exception { - // Add a node - RMNode node1 = - MockNodes - .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); - NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); - scheduler.handle(nodeEvent1); - assertEquals(1024, scheduler.getClusterResource().getMemory()); - - // Add another node - RMNode node2 = - MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2"); - NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); - scheduler.handle(nodeEvent2); - assertEquals(1536, scheduler.getClusterResource().getMemory()); - - // Remove the first node - NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1); - scheduler.handle(nodeEvent3); - assertEquals(512, scheduler.getClusterResource().getMemory()); - } - */ -} - http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java deleted file mode 100644 index f785f2e..0000000 --- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.scheduler; - -import static org.junit.Assert.*; - -import org.apache.mesos.Protos.CommandInfo; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.ebay.myriad.configuration.MyriadConfiguration; -import com.ebay.myriad.scheduler.TaskFactory.NMTaskFactoryImpl; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - -/** - * Class to test CommandLine generation - */ -public class TestServiceCommandLine { - - static MyriadConfiguration cfg; - - static String toJHSCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . &&" + - " cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\";" + - "sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver"; - - static String toCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;"; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class); - - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - } - - @Test - public void testJHSCommandLineGeneration() throws Exception { - ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null); - String executorCmd = "$YARN_HOME/bin/mapred historyserver"; - ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0); - - CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd); - - assertTrue(cInfo.getValue().startsWith(toCompare)); - } - - @Test - public void testNMCommandLineGeneration() throws Exception { - Long[] ports = new Long[]{1L, 2L, 3L, 4L}; - NMPorts nmPorts = new NMPorts(ports); - - ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L, 15L), 3.0, 5.0); - - ExecutorCommandLineGenerator clGenerator = new DownloadNMExecutorCLGenImpl(cfg, "hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz"); - NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator); - - CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts); - - assertTrue(cInfo.getValue().startsWith(toCompare)); - - } -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java deleted file mode 100644 index f296f63..0000000 --- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.ebay.myriad.scheduler; - -import static org.junit.Assert.*; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.ebay.myriad.configuration.MyriadBadConfigurationException; -import com.ebay.myriad.configuration.MyriadConfiguration; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -/** - * Tests for TaskUtils - */ -public class TestTaskUtils { - - static MyriadConfiguration cfg; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - } - - @Test - public void testGetResource() { - TaskUtils taskUtils = new TaskUtils(cfg); - - NMProfile fooProfile = new NMProfile("abc", 1L, 1000L); - try { - taskUtils.getAuxTaskCpus(fooProfile, "foo"); - fail("Should not complete sucessfully for foo"); - } catch (MyriadBadConfigurationException e) { - // success - } - - try { - double cpu = taskUtils.getAuxTaskCpus(fooProfile, "jobhistory"); - assertTrue(cpu > 0.0); - } catch (MyriadBadConfigurationException e) { - fail("cpu should be defined for jobhistory"); - } - } - - @Test - public void testServiceResourceProfile() throws Exception { - // testing custom deserializer - - Gson gson = new GsonBuilder().registerTypeAdapter(ServiceResourceProfile.class, new ServiceResourceProfile.CustomDeserializer()).create(); - - - ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0); - - String parentStr = gson.toJson(parentProfile); - ServiceResourceProfile processedProfile = gson.fromJson(parentStr, ServiceResourceProfile.class); - - assertTrue(processedProfile.getClass().equals(ServiceResourceProfile.class)); - assertTrue(processedProfile.toString().equalsIgnoreCase(parentStr)); - - ServiceResourceProfile childProfile = new ExtendedResourceProfile(new NMProfile("bcd", 5L, 15L), 2.0, 7.0); - - String childStr = gson.toJson(childProfile); - ServiceResourceProfile processedChildProfile = gson.fromJson(childStr, ServiceResourceProfile.class); - - assertTrue(processedChildProfile instanceof ExtendedResourceProfile); - assertTrue(processedChildProfile.toString().equalsIgnoreCase(childStr)); - } - - @Test - - public void testStackTrace() { - - new Throwable().printStackTrace(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/constraints/LikeConstraintSpec.groovy ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/constraints/LikeConstraintSpec.groovy b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/constraints/LikeConstraintSpec.groovy deleted file mode 100644 index 497564e..0000000 --- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/constraints/LikeConstraintSpec.groovy +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.scheduler.constraints - -import com.google.common.collect.Lists -import org.apache.mesos.Protos -import spock.lang.Specification - -import static org.apache.mesos.Protos.Value.Text -import static org.apache.mesos.Protos.Value.Type.TEXT - -/** - * - * Test for LikeConstraint - * - */ -class LikeConstraintSpec extends Specification { - - def "is matching host name"() { - given: - def constraint = new LikeConstraint("hostname", "host-[0-9]*.example.com") - - expect: - returnValue == constraint.matchesHostName(inputHostName) - - where: - inputHostName | returnValue - null | false - "" | false - "blah-blue" | false - "host-12.example.com" | true - "host-1.example.com" | true - "host-2.example.com" | true - } - - def "is matching dfs attribute"() { - given: - def constraint = new LikeConstraint("dfs", "true") - - expect: - returnValue == constraint.matchesSlaveAttributes(attributes) - - where: - attributes | returnValue - null | false - Lists.newArrayList() | false - Lists.newArrayList(getTextAttribute("dfs", "")) | false - Lists.newArrayList(getTextAttribute("dfs", "false")) | false - Lists.newArrayList(getTextAttribute("Distributed FS", "true")) | false - Lists.newArrayList(getTextAttribute("dfs", "true")) | true - Lists.newArrayList(getTextAttribute("dfs", "true"), - getTextAttribute("random", "random value")) | true - } - - def "equals"() { - given: - def constraint1 = new LikeConstraint("hostname", "perfnode13[3-4].perf.lab") - def constraint2 = new LikeConstraint("hostname", "perfnode13[3-4].perf.lab") - def constraint3 = new LikeConstraint("hostname", "perfnode133.perf.lab") - - expect: - constraint1.equals(constraint2) - !constraint1.equals(constraint3) - !constraint2.equals(constraint3) - } - - private static Protos.Attribute getTextAttribute(String name, String value) { - Protos.Attribute.newBuilder() - .setName(name) - .setType(TEXT) - .setText(Text.newBuilder() - .setValue(value)) - .build() - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/FGSTestBaseSpec.groovy ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/FGSTestBaseSpec.groovy b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/FGSTestBaseSpec.groovy deleted file mode 100644 index 166d0f6..0000000 --- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/FGSTestBaseSpec.groovy +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.scheduler.fgs - -import com.ebay.myriad.configuration.MyriadConfiguration -import com.ebay.myriad.scheduler.MyriadDriver -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import org.apache.hadoop.yarn.api.records.* -import org.apache.hadoop.yarn.event.Dispatcher -import org.apache.hadoop.yarn.event.EventHandler -import org.apache.hadoop.yarn.server.resourcemanager.MockNodes -import org.apache.hadoop.yarn.server.resourcemanager.RMContext -import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter -import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode -import org.apache.hadoop.yarn.util.resource.Resources -import org.apache.mesos.Protos -import org.apache.mesos.SchedulerDriver -import spock.lang.Specification - -import java.util.concurrent.ConcurrentHashMap - -/** - * - * Base class for testing Fine Grained Scaling. - * - */ -class FGSTestBaseSpec extends Specification { - def nodeStore = new NodeStore() - def mesosDriver = Mock(SchedulerDriver) - def myriadDriver = new MyriadDriver(mesosDriver) - def offerLifecycleManager = new OfferLifecycleManager(nodeStore, myriadDriver) - - def cfg = new MyriadConfiguration() - - void setup() { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - cfg = mapper.readValue( - Thread.currentThread().getContextClassLoader().getResource("myriad-config-default.yml"), - MyriadConfiguration.class) - } -/******************* Nodes Related ****************/ - - def rmNodes = new ConcurrentHashMap() - - RMNode getRMNode(int cpu, int mem, String host, Protos.SlaveID slaveId) { - RMNode rmNode = MockNodes.newNodeInfo(0, Resources.createResource(mem, cpu), 0, host) - if (rmNodes[rmNode.getNodeID()]) { - throw new IllegalArgumentException("Node with hostname: " + host + " already exists") - } - rmNodes.put(rmNode.getNodeID(), rmNode) - nodeStore.add(getSchedulerNode(rmNode)) - def node = nodeStore.getNode(host) - node.setSlaveId(slaveId) - - return rmNode - } - - SchedulerNode getSchedulerNode(RMNode rmNode) { - SchedulerNode schedulerNode = new SchedulerNode(rmNode, false) { - - @Override - void reserveResource(SchedulerApplicationAttempt attempt, Priority priority, RMContainer container) { - } - - @Override - void unreserveResource(SchedulerApplicationAttempt attempt) { - } - } - return schedulerNode - } - - /******************* RMContext Related ****************/ - - def publisher = Mock(SystemMetricsPublisher) {} - def writer = Mock(RMApplicationHistoryWriter) {} - def handler = Mock(EventHandler) {} - - def dispatcher = Mock(Dispatcher) { - getEventHandler() >> handler - } - - def rmContext = Mock(RMContext) { - getDispatcher() >> dispatcher - getRMApplicationHistoryWriter() >> writer - getSystemMetricsPublisher() >> publisher - getRMNodes() >> rmNodes - } - - /******************* Offers Related ****************/ - - Protos.Offer addOfferToFeed(Protos.SlaveID slaveID, String host, int cpu, int mem) { - def offer = Protos.Offer.newBuilder() - .setId(Protos.OfferID.newBuilder().setValue("test_offer_id")) - .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("test_framework_id")) - .setSlaveId(slaveID) - .setHostname(host) - .addResources(Protos.Resource.newBuilder() - .setName("cpus") - .setScalar(Protos.Value.Scalar.newBuilder().setValue(cpu)) - .setType(Protos.Value.Type.SCALAR).build()) - .addResources(Protos.Resource.newBuilder() - .setName("mem") - .setScalar(Protos.Value.Scalar.newBuilder().setValue(mem)) - .setType(Protos.Value.Type.SCALAR).build()) - .build() - offerLifecycleManager.addOffers(offer) - return offer - } - - /******************* Containers Related ****************/ - - class FGSContainer { - ContainerId containerId - Container container - RMContainer rmContainer - ContainerStatus containerStatus - } - - def fgsContainers = new HashMap<>() - - AbstractYarnScheduler yarnScheduler = Mock(AbstractYarnScheduler) { - getRMContainer(_ as ContainerId) >> { ContainerId cid -> fgsContainers.get(cid).rmContainer } - } - - FGSContainer getFGSContainer(RMNode node, int cid, int cpu, int mem, ContainerState state) { - FGSContainer fgsContainer = createFGSContainer(node, cid, cpu, mem, state) - if (!fgsContainers[fgsContainer.containerId]) { - fgsContainers[fgsContainer.containerId] = fgsContainer - } - return fgsContainer - } - - private FGSContainer createFGSContainer(RMNode node, int cid, int cpu, int mem, ContainerState state) { - ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance( - ApplicationId.newInstance(123456789, 1), 1), cid) - FGSContainer fgsContainer = new FGSContainer() - fgsContainer.containerId = containerId - fgsContainer.container = Container.newInstance(containerId, node.getNodeID(), node.getHttpAddress(), - Resources.createResource(mem, cpu), null, null) - fgsContainer.rmContainer = new RMContainerImpl(fgsContainer.container, containerId.getApplicationAttemptId(), - node.getNodeID(), "user1", rmContext) - nodeStore.getNode(node.getNodeID().getHost()).getNode().allocateContainer(fgsContainer.rmContainer) - fgsContainer.containerStatus = ContainerStatus.newInstance(containerId, state, "", 0) - return fgsContainer - } - -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/NMHeartBeatHandlerSpec.groovy ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/NMHeartBeatHandlerSpec.groovy b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/NMHeartBeatHandlerSpec.groovy deleted file mode 100644 index a37b9bf..0000000 --- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/NMHeartBeatHandlerSpec.groovy +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.scheduler.fgs - -import com.ebay.myriad.scheduler.yarn.interceptor.InterceptorRegistry -import com.ebay.myriad.state.SchedulerState -import org.apache.hadoop.yarn.api.records.ContainerState -import org.apache.hadoop.yarn.api.records.ContainerStatus -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStartedEvent -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent -import org.apache.hadoop.yarn.util.resource.Resources -import org.apache.mesos.Protos -import org.slf4j.Logger - -/** - * - * Tests for NMHeartBeatHandler - * - */ -class NMHeartBeatHandlerSpec extends FGSTestBaseSpec { - - def "Node Manager registration"() { - given: - def hbHandler = getNMHeartBeatHandler() - hbHandler.logger = Mock(Logger) - - def nonZeroNM = getRMNode(2, 2048, "test_host1", null) - def zeroNM = getRMNode(0, 0, "test_host2", null) - - when: - hbHandler.beforeRMNodeEventHandled(getNMRegistrationEvent(nonZeroNM), rmContext) - - then: - 1 * hbHandler.logger.warn('FineGrainedScaling feature got invoked for a NM with non-zero capacity. ' + - 'Host: {}, Mem: {}, CPU: {}. Setting the NM\'s capacity to (0G,0CPU)', 'test_host1', 2048, 2) - nonZeroNM.getTotalCapability().getMemory() == 0 - nonZeroNM.getTotalCapability().getVirtualCores() == 0 - - when: - hbHandler.beforeRMNodeEventHandled(getNMRegistrationEvent(zeroNM), rmContext) - - then: - 0 * hbHandler.logger.warn(_) // no logger.warn invoked - nonZeroNM.getTotalCapability().getMemory() == 0 - nonZeroNM.getTotalCapability().getVirtualCores() == 0 - } - - def "Node Manager HeartBeat"() { - given: - def host = "test_host" - def slaveId = Protos.SlaveID.newBuilder().setValue(host + "_slave_id").build() - def zeroNM = getRMNode(0, 0, host, slaveId) - - def fgsContainer1 = getFGSContainer(zeroNM, 1, 1, 1024, ContainerState.RUNNING) - def fgsContainer2 = getFGSContainer(zeroNM, 2, 1, 1024, ContainerState.COMPLETE) - def fgsContainer3 = getFGSContainer(zeroNM, 3, 1, 1024, ContainerState.RUNNING) - - addOfferToFeed(slaveId, host, 2, 2048) - - def yarnNodeCapacityManager = Mock(YarnNodeCapacityManager) - def hbHandler = getNMHeartBeatHandler(yarnNodeCapacityManager) - - when: - hbHandler.handleStatusUpdate( - getHBEvent( - zeroNM, - fgsContainer1.containerStatus, - fgsContainer2.containerStatus, - fgsContainer3.containerStatus), - rmContext) - - then: - nodeStore.getNode(host).getContainerSnapshot().size() == 3 - 1 * yarnNodeCapacityManager.setNodeCapacity(zeroNM, Resources.createResource(4096, 4)) - } - - - RMNodeStartedEvent getNMRegistrationEvent(RMNode node) { - new RMNodeStartedEvent(node.getNodeID(), null, null) - } - - RMNodeStatusEvent getHBEvent(RMNode node, ContainerStatus... statuses) { - return new RMNodeStatusEvent(node.getNodeID(), null, Arrays.asList(statuses), null, null) - } - - NMHeartBeatHandler getNMHeartBeatHandler() { - return getNMHeartBeatHandler(Mock(YarnNodeCapacityManager)) - } - - NMHeartBeatHandler getNMHeartBeatHandler(YarnNodeCapacityManager yarnNodeCapacityMgr) { - def registry = Mock(InterceptorRegistry) - def state = Mock(SchedulerState) - return new NMHeartBeatHandler(registry, yarnScheduler, myriadDriver, - yarnNodeCapacityMgr, offerLifecycleManager, nodeStore, state) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/YarnNodeCapacityManagerSpec.groovy ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/YarnNodeCapacityManagerSpec.groovy b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/YarnNodeCapacityManagerSpec.groovy deleted file mode 100644 index c63ff9a..0000000 --- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/YarnNodeCapacityManagerSpec.groovy +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.scheduler.fgs - -import com.ebay.myriad.configuration.NodeManagerConfiguration -import com.ebay.myriad.scheduler.TaskFactory -import com.ebay.myriad.scheduler.TaskUtils -import com.ebay.myriad.scheduler.yarn.interceptor.InterceptorRegistry -import com.ebay.myriad.state.NodeTask -import com.ebay.myriad.state.SchedulerState -import org.apache.hadoop.yarn.api.records.ContainerState -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent -import org.apache.hadoop.yarn.util.resource.Resources -import org.apache.mesos.Protos - -/** - * - * Tests for YarnNodeCapacityManager - * - */ -class YarnNodeCapacityManagerSpec extends FGSTestBaseSpec { - - def "No Containers Allocated Due To Mesos Offers"() { - given: - def yarnNodeCapacityMgr = getYarnNodeCapacityManager() - - def host = "test_host" - def slaveId = Protos.SlaveID.newBuilder().setValue(host + "_slave_id").build() - def zeroNM = getRMNode(0, 0, host, slaveId) - - // have a mesos offer before HB - def offer = addOfferToFeed(slaveId, host, 4, 4096) - offerLifecycleManager.markAsConsumed(offer) - - // 2 containers before HB. - def fgsContainer1 = getFGSContainer(zeroNM, 1, 1, 1024, ContainerState.RUNNING) - def fgsContainer2 = getFGSContainer(zeroNM, 2, 1, 1024, ContainerState.RUNNING) - nodeStore.getNode(host).snapshotRunningContainers() - - // Node's capacity set to match the size of 2 containers + mesos offers - yarnNodeCapacityMgr.setNodeCapacity(zeroNM, Resources.createResource(6144, 6)) - - // no new container allocations - - when: - yarnNodeCapacityMgr.handleContainerAllocation(zeroNM) - - then: - nodeStore.getNode(host).getNode().getRunningContainers().size() == 2 // 2 containers still running - 1 * mesosDriver.declineOffer(offer.getId()) // offer rejected, as it's not used to allocate more containers - zeroNM.getTotalCapability().getVirtualCores() == 2 // capacity returns back to match size of running containers - zeroNM.getTotalCapability().getMemory() == 2048 - nodeStore.getNode(host).getContainerSnapshot() == null // container snapshot is released - } - - def "Containers Allocated Due To Mesos Offers"() { - given: - def yarnNodeCapacityMgr = getYarnNodeCapacityManager() - - def host = "test_host" - def slaveId = Protos.SlaveID.newBuilder().setValue(host + "_slave_id").build() - def zeroNM = getRMNode(0, 0, host, slaveId) - - // have a mesos offer before HB - def offer = addOfferToFeed(slaveId, host, 4, 4096) - offerLifecycleManager.markAsConsumed(offer) - - // 2 containers before HB. - def fgsContainer1 = getFGSContainer(zeroNM, 1, 1, 1024, ContainerState.RUNNING) - def fgsContainer2 = getFGSContainer(zeroNM, 2, 1, 1024, ContainerState.RUNNING) - nodeStore.getNode(host).snapshotRunningContainers() - - // Node's capacity set to match the size of 2 running containers + mesos offers - yarnNodeCapacityMgr.setNodeCapacity(zeroNM, Resources.createResource(6144, 6)) - - // 2 new containers allocated after HB - def fgsContainer3 = getFGSContainer(zeroNM, 3, 1, 1024, ContainerState.NEW) - def fgsContainer4 = getFGSContainer(zeroNM, 4, 1, 1024, ContainerState.NEW) - - when: - yarnNodeCapacityMgr.handleContainerAllocation(zeroNM) - - then: - nodeStore.getNode(host).getNode().getRunningContainers().size() == 4 // 2 running + 2 new - 1 * mesosDriver.launchTasks(_ as Collection, _ as List) // for place holder tasks - zeroNM.getTotalCapability().getVirtualCores() == 4 // capacity equals size of running + new containers - zeroNM.getTotalCapability().getMemory() == 4096 - nodeStore.getNode(host).getContainerSnapshot() == null // container snapshot is released - } - - def "Set Node Capacity"() { - given: - def zeroNM = getRMNode(0, 0, "test_host", null) - def yarnNodeCapacityMgr = getYarnNodeCapacityManager() - - when: - yarnNodeCapacityMgr.setNodeCapacity(zeroNM, Resources.createResource(2048, 2)) - - then: - zeroNM.getTotalCapability().getMemory() == 2048 - zeroNM.getTotalCapability().getVirtualCores() == 2 - 1 * rmContext.getDispatcher().getEventHandler().handle(_ as NodeResourceUpdateSchedulerEvent) - } - - YarnNodeCapacityManager getYarnNodeCapacityManager() { - def registry = Mock(InterceptorRegistry) - def executorInfo = Protos.ExecutorInfo.newBuilder() - .setExecutorId(Protos.ExecutorID.newBuilder().setValue("some_id")) - .setCommand(Protos.CommandInfo.newBuilder()) - .build() - def nodeTask = Mock(NodeTask) { - getExecutorInfo() >> executorInfo - } - def state = Mock(SchedulerState) { - getNodeTask(_, NodeManagerConfiguration.NM_TASK_PREFIX) >> nodeTask - } - return new YarnNodeCapacityManager(registry, yarnScheduler, rmContext, - myriadDriver, offerLifecycleManager, nodeStore, state) - - } -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/MesosModule.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/MesosModule.java b/myriad-scheduler/src/test/java/org/apache/myriad/MesosModule.java new file mode 100644 index 0000000..aee04a8 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/MesosModule.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad; + +import java.util.concurrent.FutureTask; + +import org.apache.mesos.Protos; +import org.apache.mesos.Protos.Status; +import org.apache.mesos.SchedulerDriver; +import org.apache.mesos.state.State; +import org.apache.mesos.state.Variable; +import org.mockito.Mockito; + +import org.apache.myriad.configuration.MyriadConfiguration; +import org.apache.myriad.state.MyriadState; +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; + +/** + * Guice Module for Mesos objects. + */ +public class MesosModule extends AbstractModule { + public MesosModule() { + } + + @Override + protected void configure() { + bind(org.apache.myriad.scheduler.MyriadDriver.class).in(Scopes.SINGLETON); + } + + @Provides + @Singleton + SchedulerDriver providesSchedulerDriver(org.apache.myriad.scheduler.MyriadScheduler scheduler, MyriadConfiguration cfg, org.apache.myriad.state.SchedulerState schedulerState) { + + SchedulerDriver driver = Mockito.mock(SchedulerDriver.class); + Mockito.when(driver.start()).thenReturn(Status.DRIVER_RUNNING); + Mockito.when(driver.abort()).thenReturn(Status.DRIVER_ABORTED); + + return driver; + } + + @Provides + @Singleton + State providesStateStore(MyriadConfiguration cfg) { + State stateStore = Mockito.mock(State.class); + + Runnable dummyTask = new Runnable() { + public void run() { + } + }; + + Variable var = Mockito.mock(Variable.class); + Protos.FrameworkID id = Protos.FrameworkID.newBuilder().setValue("1").build(); + + Mockito.when(var.value()).thenReturn(id.toByteArray()); + FutureTask futureTask = new FutureTask(dummyTask, var); + futureTask.run(); + Mockito.when(stateStore.fetch(MyriadState.KEY_FRAMEWORK_ID)).thenReturn(futureTask); + + return stateStore; + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java new file mode 100644 index 0000000..66ddb15 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.myriad; + +import static org.junit.Assert.*; + +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.myriad.scheduler.TaskFactory; +import com.google.inject.Guice; +import com.google.inject.Injector; + +/** + * Test for Multibindings + */ +public class MultiBindingsTest { + + private static Injector injector; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MyriadTestModule myriadModule = new MyriadTestModule(); + injector = Guice.createInjector(myriadModule); + + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void multiBindingsTest() { + + + MultiBindingsUsage myinstance = injector.getInstance(MultiBindingsUsage.class); + + Map taskMap = myinstance.getMap(); + assertNotNull(taskMap); + assertEquals(3, taskMap.size()); + + taskMap = myinstance.getMap(); + for (Map.Entry entry : taskMap.entrySet()) { + String keyName = entry.getKey(); + TaskFactory taskFactory = entry.getValue(); + System.out.println(taskFactory); + } + + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsUsage.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsUsage.java b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsUsage.java new file mode 100644 index 0000000..ecb7894 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsUsage.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.myriad; + +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.myriad.scheduler.TaskFactory; + +/** + * Helper class to test multibindings + */ +public class MultiBindingsUsage { + + @Inject + private Map taskFactoryMap; + + public Map getMap() { + return taskFactoryMap; + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/MyriadTestModule.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/MyriadTestModule.java b/myriad-scheduler/src/test/java/org/apache/myriad/MyriadTestModule.java new file mode 100644 index 0000000..315d055 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/MyriadTestModule.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.myriad; + +import java.io.IOException; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.myriad.configuration.ServiceConfiguration; +import org.apache.myriad.configuration.MyriadConfiguration; +import org.apache.myriad.configuration.MyriadExecutorConfiguration; +import org.apache.myriad.configuration.NodeManagerConfiguration; +import org.apache.myriad.scheduler.TaskFactory.NMTaskFactoryImpl; +import org.apache.myriad.scheduler.NMExecutorCLGenImpl; +import org.apache.myriad.scheduler.TaskFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import com.google.inject.multibindings.MapBinder; + +/** + * AbstractModule extension for UnitTests + */ +public class MyriadTestModule extends AbstractModule { + + private static final Logger LOGGER = LoggerFactory.getLogger(MyriadTestModule.class); + + private MyriadConfiguration cfg; + + @SuppressWarnings("unchecked") + @Override + protected void configure() { + + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + try { + cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class); + } catch (IOException e1) { + LOGGER.error("IOException", e1); + return; + } + + if (cfg == null) { + return; + } + + bind(MyriadConfiguration.class).toInstance(cfg); + + MapBinder mapBinder = MapBinder.newMapBinder(binder(), String.class, TaskFactory.class); + mapBinder.addBinding(NodeManagerConfiguration.NM_TASK_PREFIX).to(NMTaskFactoryImpl.class).in(Scopes.SINGLETON); + Map auxServicesConfigs = cfg.getServiceConfigurations(); + for (Map.Entry entry : auxServicesConfigs.entrySet()) { + String taskFactoryClass = entry.getValue().getTaskFactoryImplName().orNull(); + if (taskFactoryClass != null) { + try { + Class implClass = (Class) Class.forName(taskFactoryClass); + mapBinder.addBinding(entry.getKey()).to(implClass).in(Scopes.SINGLETON); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } else { + mapBinder.addBinding(entry.getKey()).to(org.apache.myriad.scheduler.ServiceTaskFactoryImpl.class).in(Scopes.SINGLETON); + } + } + } + + @Provides + @Singleton + org.apache.myriad.scheduler.ExecutorCommandLineGenerator providesCLIGenerator(MyriadConfiguration cfg) { + org.apache.myriad.scheduler.ExecutorCommandLineGenerator cliGenerator = null; + MyriadExecutorConfiguration myriadExecutorConfiguration = cfg.getMyriadExecutorConfiguration(); + if (myriadExecutorConfiguration.getNodeManagerUri().isPresent()) { + cliGenerator = new org.apache.myriad.scheduler.DownloadNMExecutorCLGenImpl(cfg, myriadExecutorConfiguration.getNodeManagerUri().get()); + } else { + cliGenerator = new NMExecutorCLGenImpl(cfg); + } + return cliGenerator; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadBadConfigurationExceptionTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadBadConfigurationExceptionTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadBadConfigurationExceptionTest.java new file mode 100644 index 0000000..cc4124c --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadBadConfigurationExceptionTest.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.myriad.configuration; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Class to test MyriadBadConfigurationException + */ +public class MyriadBadConfigurationExceptionTest { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void myriadExceptionTest() { + final String testStr = "org.apache.myriad.configuration.MyriadBadConfigurationException: Bad configuration exception"; + MyriadBadConfigurationException exp = new MyriadBadConfigurationException("Bad configuration exception"); + + assertEquals(testStr, exp.toString()); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java new file mode 100644 index 0000000..06f44b6 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.myriad.configuration; + +import static org.junit.Assert.*; + +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * AuxServices/tasks test + */ +public class MyriadConfigurationTest { + + static MyriadConfiguration cfg; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class); + + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void additionalPropertiestest() throws Exception { + + Map auxConfigs = cfg.getServiceConfigurations(); + + assertNotNull(auxConfigs); + assertEquals(auxConfigs.size(), 2); + + for (Map.Entry entry : auxConfigs.entrySet()) { + String taskName = entry.getKey(); + ServiceConfiguration config = entry.getValue(); + String outTaskname = config.getTaskName(); + assertEquals(taskName, outTaskname); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsSpec.groovy ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsSpec.groovy b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsSpec.groovy new file mode 100644 index 0000000..190db32 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsSpec.groovy @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.scheduler + +import org.apache.myriad.configuration.NodeManagerConfiguration +import org.apache.myriad.state.NodeTask +import org.apache.myriad.state.SchedulerState +import org.apache.mesos.Protos +import spock.lang.Specification + +/** + * + * @author kensipe + */ +class SchedulerUtilsSpec extends Specification { + + def "is unique host name"() { + given: + def offer = Mock(Protos.OfferOrBuilder) + offer.getHostname() >> "hostname" + + expect: + returnValue == SchedulerUtils.isUniqueHostname(offer, launchTask, tasks) + + where: + tasks | launchTask | returnValue + [] | null | true + null | null | true + createNodeTaskList("hostname") | createNodeTask("hostname") | false + createNodeTaskList("missinghost") | createNodeTask("hostname") | true + createNodeTaskList("missinghost1", "missinghost2") | createNodeTask("missinghost3") | true + createNodeTaskList("missinghost1", "hostname") | createNodeTask("hostname") | false + + } + + def "is eligible for Fine Grained Scaling"() { + given: + def state = Mock(SchedulerState) + def tasks = [] + def fgsNMTask = new NodeTask(new ExtendedResourceProfile(new NMProfile("zero", 0, 0), 1.0, 2.0), null) + def cgsNMTask = new NodeTask(new ExtendedResourceProfile(new NMProfile("low", 2, 4096), 1.0, 2.0), null) + fgsNMTask.setHostname("test_fgs_hostname") + cgsNMTask.setHostname("test_cgs_hostname") + tasks << fgsNMTask << cgsNMTask + state.getActiveTasksByType(NodeManagerConfiguration.NM_TASK_PREFIX) >> tasks + + expect: + returnValue == SchedulerUtils.isEligibleForFineGrainedScaling(hostName, state) + + where: + hostName | returnValue + "test_fgs_hostname" | true + "test_cgs_hostname" | false + "blah" | false + "" | false + null | false + } + + ArrayList createNodeTaskList(String... hostnames) { + def list = [] + hostnames.each { hostname -> + list << createNodeTask(hostname) + } + return list + } + + + NodeTask createNodeTask(String hostname) { + def node = new NodeTask(new ExtendedResourceProfile(new NMProfile("", 1, 1), 1.0, 1.0), null) + node.hostname = hostname + node.taskPrefix = "nm" + node + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TMSTaskFactoryImpl.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TMSTaskFactoryImpl.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TMSTaskFactoryImpl.java new file mode 100644 index 0000000..89edfe9 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TMSTaskFactoryImpl.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.myriad.scheduler; + +import javax.inject.Inject; + +import org.apache.mesos.Protos.CommandInfo; +import org.apache.mesos.Protos.ExecutorInfo; +import org.apache.mesos.Protos.FrameworkID; +import org.apache.mesos.Protos.Offer; +import org.apache.mesos.Protos.TaskID; +import org.apache.mesos.Protos.TaskInfo; + +import org.apache.myriad.configuration.MyriadConfiguration; + +/** + * Test implementation of TaskFactory + */ +public class TMSTaskFactoryImpl implements TaskFactory { + + private MyriadConfiguration cfg; + private TaskUtils taskUtils; + + @Inject + public TMSTaskFactoryImpl(MyriadConfiguration cfg, TaskUtils taskUtils) { + this.setCfg(cfg); + this.setTaskUtils(taskUtils); + } + + @Override + public TaskInfo createTask(Offer offer, FrameworkID frameworkId, TaskID taskId, org.apache.myriad.state.NodeTask nodeTask) { + return null; + } + + public MyriadConfiguration getCfg() { + return cfg; + } + + public void setCfg(MyriadConfiguration cfg) { + this.cfg = cfg; + } + + public TaskUtils getTaskUtils() { + return taskUtils; + } + + public void setTaskUtils(TaskUtils taskUtils) { + this.taskUtils = taskUtils; + } + + @Override + public ExecutorInfo getExecutorInfoForSlave(FrameworkID frameworkId, Offer offer, CommandInfo commandInfo) { + return null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestMyriadScheduler.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestMyriadScheduler.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestMyriadScheduler.java new file mode 100644 index 0000000..d7abb3d --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestMyriadScheduler.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.scheduler; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +//import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +//import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +//import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; +//import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +//import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.After; +import org.junit.Before; +//import org.junit.Test; + +//import static org.junit.Assert.assertEquals; + +/** + * Tests myriad scheduler. + */ +public class TestMyriadScheduler { + protected Configuration conf; + protected FairScheduler scheduler; + protected ResourceManager resourceManager; + + @Before + public void setUp() throws IOException { + scheduler = new org.apache.myriad.scheduler.yarn.MyriadFairScheduler(); + conf = createConfiguration(); + resourceManager = new ResourceManager(); + resourceManager.init(conf); + + // TODO: This test should really be using MockRM. For now starting stuff + // that is needed at a bare minimum. + ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start(); + resourceManager.getRMContext().getStateStore().start(); + + // to initialize the master key + resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); + + scheduler.setRMContext(resourceManager.getRMContext()); + scheduler.init(conf); + scheduler.start(); + } + + @After + public void tearDown() { + if (scheduler != null) { + scheduler.stop(); + scheduler = null; + } + + if (resourceManager != null) { + resourceManager.stop(); + resourceManager = null; + } + } + + public Configuration createConfiguration() { + Configuration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); + conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); + return conf; + } +/* + @Test + public void testClusterMemory() throws Exception { + // Add a node + RMNode node1 = + MockNodes + .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + assertEquals(1024, scheduler.getClusterResource().getMemory()); + + // Add another node + RMNode node2 = + MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2"); + NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); + scheduler.handle(nodeEvent2); + assertEquals(1536, scheduler.getClusterResource().getMemory()); + + // Remove the first node + NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1); + scheduler.handle(nodeEvent3); + assertEquals(512, scheduler.getClusterResource().getMemory()); + } + */ +} + http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java new file mode 100644 index 0000000..080a76f --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.scheduler; + +import static org.junit.Assert.*; + +import org.apache.mesos.Protos.CommandInfo; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.myriad.configuration.MyriadConfiguration; +import org.apache.myriad.scheduler.TaskFactory.NMTaskFactoryImpl; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * Class to test CommandLine generation + */ +public class TestServiceCommandLine { + + static MyriadConfiguration cfg; + + static String toJHSCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . &&" + + " cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\";" + + "sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver"; + + static String toCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;"; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class); + + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void testJHSCommandLineGeneration() throws Exception { + ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null); + String executorCmd = "$YARN_HOME/bin/mapred historyserver"; + ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0); + + CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd); + + assertTrue(cInfo.getValue().startsWith(toCompare)); + } + + @Test + public void testNMCommandLineGeneration() throws Exception { + Long[] ports = new Long[]{1L, 2L, 3L, 4L}; + NMPorts nmPorts = new NMPorts(ports); + + ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L, 15L), 3.0, 5.0); + + ExecutorCommandLineGenerator clGenerator = new DownloadNMExecutorCLGenImpl(cfg, "hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz"); + NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator); + + CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts); + + assertTrue(cInfo.getValue().startsWith(toCompare)); + + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java new file mode 100644 index 0000000..f58e1ba --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.myriad.scheduler; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.myriad.configuration.MyriadBadConfigurationException; +import org.apache.myriad.configuration.MyriadConfiguration; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * Tests for TaskUtils + */ +public class TestTaskUtils { + + static MyriadConfiguration cfg; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void testGetResource() { + TaskUtils taskUtils = new TaskUtils(cfg); + + NMProfile fooProfile = new NMProfile("abc", 1L, 1000L); + try { + taskUtils.getAuxTaskCpus(fooProfile, "foo"); + fail("Should not complete sucessfully for foo"); + } catch (MyriadBadConfigurationException e) { + // success + } + + try { + double cpu = taskUtils.getAuxTaskCpus(fooProfile, "jobhistory"); + assertTrue(cpu > 0.0); + } catch (MyriadBadConfigurationException e) { + fail("cpu should be defined for jobhistory"); + } + } + + @Test + public void testServiceResourceProfile() throws Exception { + // testing custom deserializer + + Gson gson = new GsonBuilder().registerTypeAdapter(ServiceResourceProfile.class, new ServiceResourceProfile.CustomDeserializer()).create(); + + + ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0); + + String parentStr = gson.toJson(parentProfile); + ServiceResourceProfile processedProfile = gson.fromJson(parentStr, ServiceResourceProfile.class); + + assertTrue(processedProfile.getClass().equals(ServiceResourceProfile.class)); + assertTrue(processedProfile.toString().equalsIgnoreCase(parentStr)); + + ServiceResourceProfile childProfile = new ExtendedResourceProfile(new NMProfile("bcd", 5L, 15L), 2.0, 7.0); + + String childStr = gson.toJson(childProfile); + ServiceResourceProfile processedChildProfile = gson.fromJson(childStr, ServiceResourceProfile.class); + + assertTrue(processedChildProfile instanceof ExtendedResourceProfile); + assertTrue(processedChildProfile.toString().equalsIgnoreCase(childStr)); + } + + @Test + + public void testStackTrace() { + + new Throwable().printStackTrace(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/constraints/LikeConstraintSpec.groovy ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/constraints/LikeConstraintSpec.groovy b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/constraints/LikeConstraintSpec.groovy new file mode 100644 index 0000000..b56d23b --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/constraints/LikeConstraintSpec.groovy @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.scheduler.constraints + +import com.google.common.collect.Lists +import org.apache.mesos.Protos +import spock.lang.Specification + +import static org.apache.mesos.Protos.Value.Text +import static org.apache.mesos.Protos.Value.Type.TEXT + +/** + * + * Test for LikeConstraint + * + */ +class LikeConstraintSpec extends Specification { + + def "is matching host name"() { + given: + def constraint = new LikeConstraint("hostname", "host-[0-9]*.example.com") + + expect: + returnValue == constraint.matchesHostName(inputHostName) + + where: + inputHostName | returnValue + null | false + "" | false + "blah-blue" | false + "host-12.example.com" | true + "host-1.example.com" | true + "host-2.example.com" | true + } + + def "is matching dfs attribute"() { + given: + def constraint = new LikeConstraint("dfs", "true") + + expect: + returnValue == constraint.matchesSlaveAttributes(attributes) + + where: + attributes | returnValue + null | false + Lists.newArrayList() | false + Lists.newArrayList(getTextAttribute("dfs", "")) | false + Lists.newArrayList(getTextAttribute("dfs", "false")) | false + Lists.newArrayList(getTextAttribute("Distributed FS", "true")) | false + Lists.newArrayList(getTextAttribute("dfs", "true")) | true + Lists.newArrayList(getTextAttribute("dfs", "true"), + getTextAttribute("random", "random value")) | true + } + + def "equals"() { + given: + def constraint1 = new LikeConstraint("hostname", "perfnode13[3-4].perf.lab") + def constraint2 = new LikeConstraint("hostname", "perfnode13[3-4].perf.lab") + def constraint3 = new LikeConstraint("hostname", "perfnode133.perf.lab") + + expect: + constraint1.equals(constraint2) + !constraint1.equals(constraint3) + !constraint2.equals(constraint3) + } + + private static Protos.Attribute getTextAttribute(String name, String value) { + Protos.Attribute.newBuilder() + .setName(name) + .setType(TEXT) + .setText(Text.newBuilder() + .setValue(value)) + .build() + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy new file mode 100644 index 0000000..afd02be --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.scheduler.fgs + +import org.apache.myriad.configuration.MyriadConfiguration +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import org.apache.hadoop.yarn.api.records.* +import org.apache.hadoop.yarn.event.Dispatcher +import org.apache.hadoop.yarn.event.EventHandler +import org.apache.hadoop.yarn.server.resourcemanager.MockNodes +import org.apache.hadoop.yarn.server.resourcemanager.RMContext +import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter +import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode +import org.apache.hadoop.yarn.util.resource.Resources +import org.apache.mesos.Protos +import org.apache.mesos.SchedulerDriver +import org.apache.myriad.scheduler.MyriadDriver +import spock.lang.Specification + +import java.util.concurrent.ConcurrentHashMap + +/** + * + * Base class for testing Fine Grained Scaling. + * + */ +class FGSTestBaseSpec extends Specification { + def nodeStore = new NodeStore() + def mesosDriver = Mock(SchedulerDriver) + def myriadDriver = new MyriadDriver(mesosDriver) + def offerLifecycleManager = new OfferLifecycleManager(nodeStore, myriadDriver) + + def cfg = new MyriadConfiguration() + + void setup() { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) + cfg = mapper.readValue( + Thread.currentThread().getContextClassLoader().getResource("myriad-config-default.yml"), + MyriadConfiguration.class) + } +/******************* Nodes Related ****************/ + + def rmNodes = new ConcurrentHashMap() + + RMNode getRMNode(int cpu, int mem, String host, Protos.SlaveID slaveId) { + RMNode rmNode = MockNodes.newNodeInfo(0, Resources.createResource(mem, cpu), 0, host) + if (rmNodes[rmNode.getNodeID()]) { + throw new IllegalArgumentException("Node with hostname: " + host + " already exists") + } + rmNodes.put(rmNode.getNodeID(), rmNode) + nodeStore.add(getSchedulerNode(rmNode)) + def node = nodeStore.getNode(host) + node.setSlaveId(slaveId) + + return rmNode + } + + SchedulerNode getSchedulerNode(RMNode rmNode) { + SchedulerNode schedulerNode = new SchedulerNode(rmNode, false) { + + @Override + void reserveResource(SchedulerApplicationAttempt attempt, Priority priority, RMContainer container) { + } + + @Override + void unreserveResource(SchedulerApplicationAttempt attempt) { + } + } + return schedulerNode + } + + /******************* RMContext Related ****************/ + + def publisher = Mock(SystemMetricsPublisher) {} + def writer = Mock(RMApplicationHistoryWriter) {} + def handler = Mock(EventHandler) {} + + def dispatcher = Mock(Dispatcher) { + getEventHandler() >> handler + } + + def rmContext = Mock(RMContext) { + getDispatcher() >> dispatcher + getRMApplicationHistoryWriter() >> writer + getSystemMetricsPublisher() >> publisher + getRMNodes() >> rmNodes + } + + /******************* Offers Related ****************/ + + Protos.Offer addOfferToFeed(Protos.SlaveID slaveID, String host, int cpu, int mem) { + def offer = Protos.Offer.newBuilder() + .setId(Protos.OfferID.newBuilder().setValue("test_offer_id")) + .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("test_framework_id")) + .setSlaveId(slaveID) + .setHostname(host) + .addResources(Protos.Resource.newBuilder() + .setName("cpus") + .setScalar(Protos.Value.Scalar.newBuilder().setValue(cpu)) + .setType(Protos.Value.Type.SCALAR).build()) + .addResources(Protos.Resource.newBuilder() + .setName("mem") + .setScalar(Protos.Value.Scalar.newBuilder().setValue(mem)) + .setType(Protos.Value.Type.SCALAR).build()) + .build() + offerLifecycleManager.addOffers(offer) + return offer + } + + /******************* Containers Related ****************/ + + class FGSContainer { + ContainerId containerId + Container container + RMContainer rmContainer + ContainerStatus containerStatus + } + + def fgsContainers = new HashMap<>() + + AbstractYarnScheduler yarnScheduler = Mock(AbstractYarnScheduler) { + getRMContainer(_ as ContainerId) >> { ContainerId cid -> fgsContainers.get(cid).rmContainer } + } + + FGSContainer getFGSContainer(RMNode node, int cid, int cpu, int mem, ContainerState state) { + FGSContainer fgsContainer = createFGSContainer(node, cid, cpu, mem, state) + if (!fgsContainers[fgsContainer.containerId]) { + fgsContainers[fgsContainer.containerId] = fgsContainer + } + return fgsContainer + } + + private FGSContainer createFGSContainer(RMNode node, int cid, int cpu, int mem, ContainerState state) { + ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(123456789, 1), 1), cid) + FGSContainer fgsContainer = new FGSContainer() + fgsContainer.containerId = containerId + fgsContainer.container = Container.newInstance(containerId, node.getNodeID(), node.getHttpAddress(), + Resources.createResource(mem, cpu), null, null) + fgsContainer.rmContainer = new RMContainerImpl(fgsContainer.container, containerId.getApplicationAttemptId(), + node.getNodeID(), "user1", rmContext) + nodeStore.getNode(node.getNodeID().getHost()).getNode().allocateContainer(fgsContainer.rmContainer) + fgsContainer.containerStatus = ContainerStatus.newInstance(containerId, state, "", 0) + return fgsContainer + } + +}