myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dar...@apache.org
Subject [1/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the process). -Refactor TackFactory classes as necessary to wo
Date Fri, 12 Aug 2016 13:32:58 GMT
Repository: incubator-myriad
Updated Branches:
  refs/heads/master df4cbc0e9 -> 7aea259cf


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
index 87aee56..b6806a2 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
@@ -18,53 +18,106 @@
  */
 package org.apache.myriad.scheduler;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.mesos.Protos;
 import static org.junit.Assert.assertTrue;
 
 import org.apache.mesos.Protos.CommandInfo;
+import org.apache.myriad.configuration.ServiceConfiguration;
 import org.apache.myriad.BaseConfigurableTest;
-import org.apache.myriad.scheduler.TaskFactory.NMTaskFactoryImpl;
 import org.junit.Test;
 
+import java.util.*;
+
 /**
  * Class to test CommandLine generation
  */
 public class TestServiceCommandLine extends BaseConfigurableTest {
+
+  public static final String KEY_NM_ADDRESS = "myriad.yarn.nodemanager.address";
+  public static final String KEY_NM_LOCALIZER_ADDRESS = "myriad.yarn.nodemanager.localizer.address";
+  public static final String KEY_NM_WEBAPP_ADDRESS = "myriad.yarn.nodemanager.webapp.address";
+  public static final String KEY_NM_SHUFFLE_PORT = "myriad.mapreduce.shuffle.port";
+
+  public static final String KEY_JHS_WEBAPP_ADDRESS = "myriad.mapreduce.jobhistory.webapp.address";
+  public static final String KEY_JHS_ADMIN_ADDRESS = "myriad.mapreduce.jobhistory.admin.address";
+  public static final String KEY_JHS_ADDRESS = "myriad.mapreduce.jobhistory.address";
+
+  private static final String msgFormat = System.lineSeparator() + "%s" + System.lineSeparator()
+ "!="
+      + System.lineSeparator() + "%s";
+  protected static final String CMD_FORMAT = "echo \"%1$s\" && %1$s";
   static String toJHSCompare =
-      "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz &&  sudo  cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;
" +
-      "export TASK_DIR=`basename $PWD`; sudo  chmod +wx /sys/fs/cgroup/cpu/mesos/$TASK_DIR;"
+
-      "sudo -E -u hduser -H  $YARN_HOME/bin/mapred historyserver\"; sudo tar -zxpf hadoop-2.5.0.tar.gz
&&  sudo  cp" +
-      " conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred
historyserver";
+      " sudo tar -zxpf hadoop-2.7.0.tar.gz &&  sudo  cp yarnConfiguration /usr/local/hadoop/etc/hadoop/yarn-site.xml
&&  " +
+          "sudo -E -u hduser -H  bin/mapred historyserver";
   static String toCompare =
-      "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz &&  sudo  cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;";
+      " sudo tar -zxpf hadoop-2.7.0.tar.gz &&  sudo  cp yarnConfiguration /usr/local/hadoop/etc/hadoop/yarn-site.xml
&&  " +
+          "sudo -E -u hduser -H  $YARN_HOME/bin/yarn nodemanager";
 
   @Test
   public void testJHSCommandLineGeneration() throws Exception {
-    ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null);
-    String executorCmd = "$YARN_HOME/bin/mapred historyserver";
-    ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0);
+    Map<String, Long> portsMap = new TreeMap<>();
+    portsMap.put(KEY_JHS_ADDRESS, 0L);
+    portsMap.put(KEY_JHS_WEBAPP_ADDRESS, 3L);
+    portsMap.put(KEY_JHS_ADMIN_ADDRESS, 0L);
+
+    ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0,
portsMap);
+    ServiceConfiguration serviceConfiguration = cfg.getServiceConfiguration("jobhistory").get();
+    ServiceCommandLineGenerator serviceCommandLineGenerator = new ServiceCommandLineGenerator(cfg);
+    List<Long> ports = new ArrayList<>();
+    ports.add(2L);
+    ports.add(1L);
+    ports.add(3L);
 
-    CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd);
-    System.out.println(toJHSCompare);
-    System.out.println(cInfo.getValue());
+    CommandInfo cInfo = serviceCommandLineGenerator.generateCommandLine(profile,
+        serviceConfiguration,
+        ports);
+    String testVal =  String.format(CMD_FORMAT, toJHSCompare);
+    assertTrue(String.format(msgFormat, cInfo.getValue(), testVal),
+        cInfo.getValue().equals(testVal));
 
-    assertTrue(cInfo.getValue().startsWith(toCompare));
+    List<Protos.Environment.Variable> environmentList = cInfo.getEnvironment().getVariablesList();
+    String yarnOpts = "";
+    for (Protos.Environment.Variable variable: environmentList) {
+      if (variable.getName().equals(ServiceCommandLineGenerator.ENV_HADOOP_OPTS)){
+        yarnOpts = variable.getValue();
+      }
+    }
+    assertTrue("Environment contains " + ServiceCommandLineGenerator.ENV_HADOOP_OPTS, StringUtils.isNotEmpty(yarnOpts));
+    System.out.println(yarnOpts);
+    assertTrue(ServiceCommandLineGenerator.ENV_HADOOP_OPTS + " must contain -D" + KEY_JHS_WEBAPP_ADDRESS
+
+        "=0.0.0.0:3", yarnOpts.contains(KEY_JHS_WEBAPP_ADDRESS + "=0.0.0.0:3"));
   }
 
   @Test
   public void testNMCommandLineGeneration() throws Exception {
     Long[] ports = new Long[]{1L, 2L, 3L, 4L};
-    NMPorts nmPorts = new NMPorts(ports);
+    List<Long> nmPorts = Arrays.asList(ports);
+    Map<String, Long> portsMap = new TreeMap<>();
+    portsMap.put(KEY_NM_ADDRESS, 0L);
+    portsMap.put(KEY_NM_WEBAPP_ADDRESS, 0L);
+    portsMap.put(KEY_NM_LOCALIZER_ADDRESS, 0L);
+    portsMap.put(KEY_NM_SHUFFLE_PORT, 0L);
 
-    ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L,
15L), 3.0, 5.0);
+    ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L,
15L), 3.0, 5.0, portsMap);
 
-    ExecutorCommandLineGenerator clGenerator = new DownloadNMExecutorCLGenImpl(cfg,
-        "hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz");
-    NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator);
+    ExecutorCommandLineGenerator clGenerator = new NMExecutorCommandLineGenerator(cfg);
 
-    CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts);
-    System.out.println(toCompare);
-    System.out.println(cInfo.getValue());
-    assertTrue(cInfo.getValue().startsWith(toCompare));
+    CommandInfo cInfo = clGenerator.generateCommandLine(profile, null, nmPorts);
+    String testVal =  String.format(CMD_FORMAT, toCompare);
+    assertTrue(String.format(msgFormat, cInfo.getValue(), testVal),
+        cInfo.getValue().equals(testVal));
 
+    List<Protos.Environment.Variable> environmentList = cInfo.getEnvironment().getVariablesList();
+    String yarnOpts = "";
+    for (Protos.Environment.Variable variable: environmentList) {
+      if (variable.getName().equals(NMExecutorCommandLineGenerator.ENV_YARN_NODEMANAGER_OPTS)){
+        yarnOpts = variable.getValue();
+      }
+    }
+    System.out.println(yarnOpts);
+    assertTrue("Environment contains " + NMExecutorCommandLineGenerator.ENV_YARN_NODEMANAGER_OPTS,
StringUtils.isNotEmpty(yarnOpts));
+    assertTrue(NMExecutorCommandLineGenerator.ENV_YARN_NODEMANAGER_OPTS + " must contain
-D" + KEY_NM_SHUFFLE_PORT +
+        "=1", yarnOpts.contains(KEY_NM_SHUFFLE_PORT + "=1"));
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceTaskFactory.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceTaskFactory.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceTaskFactory.java
new file mode 100644
index 0000000..bdd40e6
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceTaskFactory.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad.scheduler;
+
+import org.apache.mesos.Protos;
+import org.apache.myriad.BaseConfigurableTest;
+import org.apache.myriad.configuration.ServiceConfiguration;
+import org.apache.myriad.scheduler.offer.OfferBuilder;
+import org.apache.myriad.scheduler.resource.ResourceOfferContainer;
+import org.apache.myriad.state.NodeTask;
+import org.junit.Test;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Tests for ServiceTaskFactory Class
+ */
+public class TestServiceTaskFactory extends BaseConfigurableTest {
+  static Protos.FrameworkID frameworkId = Protos.FrameworkID.newBuilder().setValue("test").build();
+
+  @Test
+  public void testServiceTaskFactory() {
+    ServiceCommandLineGenerator clGenerator = new ServiceCommandLineGenerator(cfgWithDocker);
+    TaskUtils taskUtils = new TaskUtils(cfgWithDocker);
+    Protos.Offer offer = new OfferBuilder("test.com")
+        .addScalarResource("cpus", 10.0)
+        .addScalarResource("mem", 16000)
+        .addRangeResource("ports", 3400, 3410)
+        .build();
+    Map<String, ServiceConfiguration> stringServiceConfigurationMap = cfgWithDocker.getServiceConfigurations();
+    System.out.print(stringServiceConfigurationMap);
+    ServiceConfiguration serviceConfiguration = cfgWithDocker.getServiceConfigurations().get("jobhistory");
+    ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", serviceConfiguration.getCpus(),
+        serviceConfiguration.getJvmMaxMemoryMB(), serviceConfiguration.getPorts());
+    NodeTask nodeTask = new NodeTask(profile, null);
+    nodeTask.setTaskPrefix("jobhistory");
+    ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile, null);
+    System.out.print(roc.getPorts());
+    ServiceTaskFactory taskFactory = new ServiceTaskFactory(cfgWithDocker, taskUtils, clGenerator);
+    Protos.TaskInfo taskInfo = taskFactory.createTask(roc, frameworkId, makeTaskId("jobhistory"),
nodeTask);
+    assertTrue("taskInfo should have a container", taskInfo.hasContainer());
+    assertFalse("The container should not have an executor", taskInfo.hasExecutor());
+    Protos.ContainerInfo containerInfo = taskInfo.getContainer();
+    assertTrue("There should be two volumes", containerInfo.getVolumesCount() == 2);
+    assertTrue("The first volume should be read only", containerInfo.getVolumes(0).getMode().equals(Protos.Volume.Mode.RO));
+    assertTrue("The first volume should be read write", containerInfo.getVolumes(1).getMode().equals(Protos.Volume.Mode.RW));
+    assertTrue("There should be a docker image", containerInfo.getDocker().hasImage());
+    assertTrue("The docker image should be mesos/myraid", containerInfo.getDocker().getImage().equals("mesos/myriad"));
+    assertTrue("Should be using host networking", containerInfo.getDocker().getNetwork().equals(Protos.ContainerInfo.DockerInfo.Network.HOST));
+    assertTrue("There should be two parameters", containerInfo.getDocker().getParametersList().size()
== 2);
+    assertTrue("Privledged mode should be false", containerInfo.getDocker().getPrivileged()
== false);
+  }
+
+  private Protos.TaskID makeTaskId(String taskId) {
+    return Protos.TaskID.newBuilder().setValue(taskId).build();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
index d543c29..21b5ad0 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
@@ -17,15 +17,18 @@
  */
 package org.apache.myriad.scheduler;
 
+import static org.junit.Assert.assertTrue;
+import org.apache.mesos.Protos;
+import org.apache.myriad.BaseConfigurableTest;
+import org.junit.Test;
+
 import com.google.common.collect.Range;
 import com.google.common.collect.Ranges;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
-import org.apache.mesos.Protos;
-import org.apache.myriad.BaseConfigurableTest;
-import org.junit.Test;
 
-import static org.junit.Assert.assertTrue;
+import java.util.HashMap;
+
 
 /**
  * Tests for TaskUtils
@@ -40,7 +43,7 @@ public class TestTaskUtils extends BaseConfigurableTest {
     Gson gson = new GsonBuilder().registerTypeAdapter(ServiceResourceProfile.class, new ServiceResourceProfile.CustomDeserializer())
         .create();
 
-    ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0);
+    ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0,
new HashMap<String, Long>());
 
     String parentStr = gson.toJson(parentProfile);
     ServiceResourceProfile processedProfile = gson.fromJson(parentStr, ServiceResourceProfile.class);
@@ -48,7 +51,8 @@ public class TestTaskUtils extends BaseConfigurableTest {
     assertTrue(processedProfile.getClass().equals(ServiceResourceProfile.class));
     assertTrue(processedProfile.toString().equalsIgnoreCase(parentStr));
 
-    ServiceResourceProfile childProfile = new ExtendedResourceProfile(new NMProfile("bcd",
5L, 15L), 2.0, 7.0);
+    ServiceResourceProfile childProfile = new ExtendedResourceProfile(new NMProfile("bcd",
5L, 15L), 2.0, 7.0,
+        cfg.getNodeManagerConfiguration().getPorts());
 
     String childStr = gson.toJson(childProfile);
     ServiceResourceProfile processedChildProfile = gson.fromJson(childStr, ServiceResourceProfile.class);
@@ -124,25 +128,4 @@ public class TestTaskUtils extends BaseConfigurableTest {
     checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus",
0.5, 1.5), "cpus", 0.0, 0.5);
   }
 
-  @Test
-  public void testContainerInfo() {
-    TaskUtils taskUtils = new TaskUtils(cfgWithDocker);
-    Protos.ContainerInfo containerInfo = taskUtils.getContainerInfo();
-    assertTrue("The container should have a docker", containerInfo.hasDocker());
-    assertTrue("There should be two volumes", containerInfo.getVolumesCount() == 2);
-    assertTrue("The first volume should be read only", containerInfo.getVolumes(0).getMode().equals(Protos.Volume.Mode.RO));
-    assertTrue("The first volume should be read write", containerInfo.getVolumes(1).getMode().equals(Protos.Volume.Mode.RW));
-  }
-
-  @Test public void testDockerInfo() {
-    TaskUtils taskUtils = new TaskUtils(cfgWithDocker);
-    Protos.ContainerInfo containerInfo = taskUtils.getContainerInfo();
-    assertTrue("The container should have a docker", containerInfo.hasDocker());
-    assertTrue("There should be two volumes", containerInfo.getVolumesList().size() == 2);
-    assertTrue("There should be a docker image", containerInfo.getDocker().hasImage());
-    assertTrue("The docker image should be mesos/myraid", containerInfo.getDocker().getImage().equals("mesos/myriad"));
-    assertTrue("Should be using host networking", containerInfo.getDocker().getNetwork().equals(Protos.ContainerInfo.DockerInfo.Network.HOST));
-    assertTrue("There should be two parameters", containerInfo.getDocker().getParametersList().size()
== 2);
-    assertTrue("Privledged mode should be false", containerInfo.getDocker().getPrivileged()
== false);
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java
index e1f4eb5..a4f4268 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java
@@ -4,6 +4,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.util.Set;
+import java.util.TreeMap;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeId;
@@ -62,7 +63,8 @@ public class YarnNodeCapacityManagerTest extends BaseConfigurableTest {
   
   private Set<NodeTask> getNodeTasks() {
     Constraint cZero = new LikeConstraint("0.0.0.1", "host-[0-9]*.example.com");
-    ServiceResourceProfile zProfile = new ServiceResourceProfile("zProfile", 0.0, 0.0, 0.0,
0.0);
+    TreeMap<String, Long> ports = new TreeMap<>();
+    ServiceResourceProfile zProfile = new ServiceResourceProfile("zProfile", 0.0, 0.0, ports);
 
     ntZero = new NodeTask(zProfile, cZero);
     ntZero.setTaskPrefix("nm");

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/offer/OfferBuilder.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/offer/OfferBuilder.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/offer/OfferBuilder.java
new file mode 100644
index 0000000..6772409
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/offer/OfferBuilder.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.myriad.scheduler.offer;
+
+import org.apache.mesos.Protos;
+
+import java.util.Collections;
+
+
+/**
+ * Simple builder to make Offers for tests
+ */
+public class OfferBuilder {
+  Protos.Offer.Builder offer = Protos.Offer.newBuilder();
+
+  public OfferBuilder(String offerId, String hostname, String slaveId) {
+    offer.setHostname(hostname);
+    offer.setId(Protos.OfferID.newBuilder().setValue(offerId));
+    offer.setSlaveId(Protos.SlaveID.newBuilder().setValue(slaveId));
+    offer.setFrameworkId(Protos.FrameworkID.newBuilder().setValue("Myriad"));
+  }
+  public OfferBuilder(String offerId, String hostname) {
+    offer.setHostname(hostname);
+    offer.setId(Protos.OfferID.newBuilder().setValue(offerId));
+    offer.setSlaveId(Protos.SlaveID.newBuilder().setValue("agent"));
+    offer.setFrameworkId(Protos.FrameworkID.newBuilder().setValue("Myriad"));
+  }
+  public OfferBuilder(String hostname) {
+    offer.setHostname(hostname);
+    offer.setId(Protos.OfferID.newBuilder().setValue("test"));
+    offer.setSlaveId(Protos.SlaveID.newBuilder().setValue("agent"));
+    offer.setFrameworkId(Protos.FrameworkID.newBuilder().setValue("Myriad"));
+  }
+
+  public OfferBuilder addScalarResource(String name, double value) {
+    offer.addAllResources(Collections.singleton(createScalarResource(name, value)));
+    return this;
+  }
+
+  public OfferBuilder addScalarResource(String name, String role, double value) {
+    offer.addAllResources(Collections.singleton(createScalarResource(name, role, value)));
+    return this;
+  }
+
+  public OfferBuilder addRangeResource(String name, long begin, long end) {
+    offer.addAllResources(Collections.singleton(createRangeResource(name, begin, end)));
+    return this;
+  }
+
+  public OfferBuilder addRangeResource(String name, String role, long begin, long end) {
+    offer.addAllResources(Collections.singleton(createRangeResource(name, role, begin, end)));
+    return this;
+  }
+
+  public Protos.Offer build() {
+    return offer.build();
+  }
+
+  private static Protos.Resource createScalarResource(String name, String role, double value)
{
+    return Protos.Resource.newBuilder()
+        .setScalar(Protos.Value.Scalar.newBuilder().setValue(value))
+        .setType(Protos.Value.Type.SCALAR)
+        .setRole(role)
+        .setName(name)
+        .build();
+  }
+
+  private static Protos.Resource createScalarResource(String name, double value) {
+    return Protos.Resource.newBuilder()
+        .setScalar(Protos.Value.Scalar.newBuilder().setValue(value))
+        .setType(Protos.Value.Type.SCALAR)
+        .setName(name)
+        .build();
+  }
+
+  private static Protos.Resource createRangeResource(String name, long begin, long end) {
+    return Protos.Resource.newBuilder()
+        .setName(name)
+        .setType(Protos.Value.Type.RANGES)
+        .setRanges(Protos.Value.Ranges.newBuilder()
+            .addRange(Protos.Value.Range.newBuilder()
+                .setBegin(begin)
+                .setEnd(end)
+                .build())
+            .build())
+        .build();
+  }
+
+  private static Protos.Resource createRangeResource(String name, String role, long begin,
long end) {
+    return Protos.Resource.newBuilder()
+        .setName(name)
+        .setType(Protos.Value.Type.RANGES)
+        .setRole(role)
+        .setRanges(Protos.Value.Ranges.newBuilder()
+            .addRange(Protos.Value.Range.newBuilder()
+                .setBegin(begin)
+                .setEnd(end)
+                .build())
+            .build())
+        .build();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/resource/TestResourceOfferContainer.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/resource/TestResourceOfferContainer.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/resource/TestResourceOfferContainer.java
new file mode 100644
index 0000000..d54cc7a
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/resource/TestResourceOfferContainer.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad.scheduler.resource;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.mesos.Protos;
+import org.apache.myriad.configuration.MyriadConfiguration;
+import org.apache.myriad.scheduler.ExtendedResourceProfile;
+import org.apache.myriad.scheduler.NMProfile;
+import org.apache.myriad.scheduler.ServiceResourceProfile;
+import org.apache.myriad.scheduler.offer.OfferBuilder;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for ResourceOfferContainerClass
+ */
+public class TestResourceOfferContainer {
+  static MyriadConfiguration cfg;
+  static double epsilon = .0001;
+
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+    cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"),
+        MyriadConfiguration.class);
+  }
+
+  @Test
+  public void testResouceOfferContainerForNMWithOutRole() {
+    Protos.Offer offer = new OfferBuilder("test.com")
+        .addScalarResource("cpus", 4.0)
+        .addScalarResource("mem", 8000)
+        .addRangeResource("ports", 3500, 3600)
+        .build();
+    Map<String, Long> ports = new HashMap<>(4);
+    ports.put("test1.address", 0L);
+    ports.put("test2.address", 0L);
+    ports.put("test3.address", 0L);
+    ports.put("test4.port", 3501L);
+    ServiceResourceProfile profile1 = new ExtendedResourceProfile(new NMProfile("small",
2L, 6000L), .2, 1024.0, ports);
+    ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile1, "");
+    System.out.print(roc.getCpus());
+    System.out.print(roc.getMem());
+    System.out.print(roc.getPorts());
+    assertTrue(roc.getHostName().equals("test.com"));
+    assertTrue("Should be satisfied if offer contains request", roc.satisfies(profile1));
+    ServiceResourceProfile profile2 = new ExtendedResourceProfile(new NMProfile("tooMuchCpu",
7L, 8000L), .2, 1024.0, ports);
+    roc = new ResourceOfferContainer(offer, profile2, "");
+    assertFalse("Should be unsatisfied if too much cpu requested", roc.satisfies(profile2));
+    ServiceResourceProfile profile3 = new ExtendedResourceProfile(new NMProfile("tooMuchMem",
3L, 50000L), .2, 1024.0, ports);
+    roc = new ResourceOfferContainer(offer, profile3, "");
+    assertFalse("Should be unsatisfied if too much memory requested", roc.satisfies(profile3));
+    ports.put("test.bad.address", 1500L);
+    ServiceResourceProfile profile4 = new ExtendedResourceProfile(new NMProfile("portOutOfRange",
3L, 50000L), .2, 1024.0, ports);
+    roc = new ResourceOfferContainer(offer, profile4, "");
+    assertFalse("Should be unsatisfied if port not in range", roc.satisfies(profile4));
+    List<Protos.Resource> resourcesCpu = roc.consumeCpus(3.0);
+    assertTrue("Should get a list of resources of size 1", resourcesCpu.size() == 1.0);
+    assertTrue("Cpus should be decreased", roc.getCpus() == 1.0);
+    List<Protos.Resource> resourcesMem = roc.consumeMem(7000.0);
+    assertTrue("Should get a list of resources of size 1", resourcesMem.size() == 1);
+    assertTrue("Mem should be decreased", roc.getMem() == 1000.0);
+  }
+
+  @Test
+  public void testResouceOfferContainerForNMWithRole() {
+    Protos.Offer offer = new OfferBuilder("test.com")
+        .addScalarResource("cpus", 2.0)
+        .addScalarResource("mem", 8000)
+        .addScalarResource("cpus", "test", 4.0)
+        .addScalarResource("mem", "test", 32000.0)
+        .addRangeResource("ports", 3500, 3600)
+        .addRangeResource("ports", "test", 1500, 1600)
+        .build();
+    Map<String, Long> ports = new HashMap<>(4);
+    ports.put("test1.address", 0L);
+    ports.put("test2.address", 0L);
+    ports.put("test3.address", 1500L);
+    ports.put("test4.port", 3502L);
+    ServiceResourceProfile profile1 = new ExtendedResourceProfile(new NMProfile("small",
2L, 8000L), .2, 1024.0, ports);
+    ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile1, "test");
+    assertTrue(roc.getHostName().equals("test.com"));
+    assertTrue("Should be satisfied if offer contains request", roc.satisfies(profile1));
+    ServiceResourceProfile profile2 = new ExtendedResourceProfile(new NMProfile("tooMuchCpu",
7L, 8000L), .2, 1024.0, ports);
+    roc = new ResourceOfferContainer(offer, profile2, "test");
+    assertFalse("Should be unsatisfied if too much cpu requested", roc.satisfies(profile2));
+    ServiceResourceProfile profile3 = new ExtendedResourceProfile(new NMProfile("tooMuchMem",
3L, 50000L), .2, 1024.0, ports);
+    roc = new ResourceOfferContainer(offer, profile3, "test");
+    assertFalse("Should be unsatisfied if too much memory requested", roc.satisfies(profile3));
+    ports.put("test.bad.address", 32000L);
+    ServiceResourceProfile profile4 = new ExtendedResourceProfile(new NMProfile("portOutOfRange",
3L, 50000L), .2, 1024.0, ports);
+    roc = new ResourceOfferContainer(offer, profile4, "test");
+    assertFalse("Should be unsatisfied if port not in range", roc.satisfies(profile4));
+    List<Protos.Resource> resources = roc.consumeCpus(4.5);
+    assertTrue("Resource List should be of size to when requesting 4.1 cpus", (resources.size()
== 2));
+    assertTrue("Cpus should be decreased", roc.getCpus() <= 1.5);
+    List<Protos.Resource> resources1 = roc.consumeCpus(1.5);
+    assertTrue("Resource List should be of size 1", resources1.size() == 1);
+    assertTrue("All cpu resources should be consumed", roc.getCpus() <= 0.0);
+  }
+
+  @Test
+  public void testResourceOfferContainerForAuxServiceWithOutRole() {
+    Protos.Offer offer = new OfferBuilder("test.com")
+        .addScalarResource("cpus", 2.0)
+        .addScalarResource("mem", 8000)
+        .addRangeResource("ports", 3500, 3600)
+        .build();
+    Map<String, Long> ports = new HashMap<>(4);
+    ports.put("test1.address", 0L);
+    ports.put("test2.address", 0L);
+    ports.put("test3.address", 0L);
+    ports.put("test4.port", 3501L);
+    ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 2.0, 8000.0,
ports);
+    ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile, null);
+    assertTrue(roc.getHostName().equals("test.com"));
+    assertTrue("Should be satisfied if offer contains request", roc.satisfies(profile));
+  }
+
+  @Test
+  public void testResourceOfferContainerForAuxServiceWithRole() {
+    Protos.Offer offer = new OfferBuilder("test.com")
+        .addScalarResource("cpus", 2.0)
+        .addScalarResource("mem", 8000)
+        .addScalarResource("cpus", "test", 4.0)
+        .addScalarResource("mem", "test", 32000.0)
+        .addRangeResource("ports", 3500, 3600)
+        .addRangeResource("ports", "test", 1500, 1600)
+        .build();
+    Map<String, Long> ports = new HashMap<>(4);
+    ports.put("test1.address", 0L);
+    ports.put("test2.address", 0L);
+    ports.put("test3.address", 1500L);
+    ports.put("test4.port", 3501L);
+    ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 2.0, 8000.0,
ports);
+    ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile, null);
+    assertTrue(roc.getHostName().equals("test.com"));
+    assertTrue("Should be satisfied if offer contains request", roc.satisfies(profile));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java
index 69954b7..09145a2 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java
@@ -7,6 +7,8 @@ import org.apache.myriad.scheduler.constraints.LikeConstraint;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.TreeMap;
+
 /**
  * Unit tests for Cluster
  */
@@ -16,15 +18,16 @@ public class ClusterTest {
 
   @Before
   public void setUp() throws Exception {
+    TreeMap<String, Long> ports = new TreeMap<>();
     cluster = new Cluster();
     cluster.setClusterName("test-cluster");
     cluster.setMinQuota(5.0);
     cluster.setResourceManagerHost("localhost");
     cluster.setResourceManagerPort("8192");
 
-    task1 = new NodeTask(new ServiceResourceProfile("profile1", 0.1, 1024.0), new LikeConstraint("hostname1",
"host-[0-9]*.example1.com"));
-    task2 = new NodeTask(new ServiceResourceProfile("profile2", 0.2, 1024.0), new LikeConstraint("hostname2",
"host-[0-9]*.example2.com"));
-    task3 = new NodeTask(new ServiceResourceProfile("profile3", 0.3, 1024.0), new LikeConstraint("hostname3",
"host-[0-9]*.example3.com"));
+    task1 = new NodeTask(new ServiceResourceProfile("profile1", 0.1, 1024.0, ports), new
LikeConstraint("hostname1", "host-[0-9]*.example1.com"));
+    task2 = new NodeTask(new ServiceResourceProfile("profile2", 0.2, 1024.0, ports), new
LikeConstraint("hostname2", "host-[0-9]*.example2.com"));
+    task3 = new NodeTask(new ServiceResourceProfile("profile3", 0.3, 1024.0, ports), new
LikeConstraint("hostname3", "host-[0-9]*.example3.com"));
   }
 
   private void resetCluster() throws Exception {

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java
index d3cf2fe..3f486c0 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java
@@ -6,6 +6,8 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.TreeMap;
+
 /**
  * Unit test cases for NodeTask
  */
@@ -13,12 +15,13 @@ public class NodeTaskTest {
   NodeTask task;
 
   @Before
-  public void setUp() throws Exception {    
-    task = new NodeTask(new ServiceResourceProfile("profile", 0.1, 1024.0), new LikeConstraint("hostname",
"host-[0-9]*.example.com"));
+  public void setUp() throws Exception {
+    TreeMap<String, Long> ports = new TreeMap<>();
+    task = new NodeTask(new ServiceResourceProfile("profile", 0.1, 1024.0, ports), new LikeConstraint("hostname",
"host-[0-9]*.example.com"));
     
     task.setHostname("localhost");
     task.setTaskPrefix("prefix");
-    task.setProfile(new ServiceResourceProfile("ServiceResourceProfile", 0.1, 1024.0, 0.1,
512.0));
+    task.setProfile(new ServiceResourceProfile("ServiceResourceProfile", 0.1, 1024.0, ports));
   }
   
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java
b/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java
index 59e2d41..cd4a12e 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java
@@ -5,6 +5,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 
 import java.util.Set;
+import java.util.TreeMap;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore;
@@ -38,8 +39,8 @@ public class SchedulerStateTest {
   
   @Test
   public void testAddAndRemoveTask() throws Exception {
-    NodeTask task1 = new NodeTask(new ServiceResourceProfile("profile1", 0.1, 1024.0), new
LikeConstraint("hostname", "host-[0-9]*.example.com"));
-    NodeTask task2 = new NodeTask(new ServiceResourceProfile("profile2", 0.1, 1024.0), new
LikeConstraint("hostname", "host-[0-9]*.example.com"));
+    NodeTask task1 = new NodeTask(new ServiceResourceProfile("profile1", 0.1, 1024.0, new
TreeMap<String, Long>()), new LikeConstraint("hostname", "host-[0-9]*.example.com"));
+    NodeTask task2 = new NodeTask(new ServiceResourceProfile("profile2", 0.1, 1024.0, new
TreeMap<String, Long>()), new LikeConstraint("hostname", "host-[0-9]*.example.com"));
     TaskID id1 = TaskID.newBuilder().setValue("mock-task-1").build();
     TaskID id2 = TaskID.newBuilder().setValue("mock-task-2").build();
     

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java
b/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java
index 6794ec2..9d4dac5 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java
@@ -5,7 +5,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayOutputStream;
 import java.nio.ByteBuffer;
-import java.util.List;
+import java.util.*;
 
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.mesos.Protos;
@@ -33,7 +33,8 @@ public class ByteBufferSupportTest {
 
   @Before
   public void setUp() throws Exception {
-    task = new NodeTask(new ServiceResourceProfile("profile", 0.1, 1024.0), new LikeConstraint("hostname",
"host-[0-9]*.example.com"));    
+    task = new NodeTask(new ServiceResourceProfile("profile", 0.1, 1024.0, new TreeMap<String,
Long>()),
+        new LikeConstraint("hostname", "host-[0-9]*.example.com"));
     task.setHostname("localhost");
     task.setTaskPrefix("prefix"); 
     task.setExecutorInfo(getExecutorInfo());

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
index 4bdb6e2..cd72c18 100644
--- a/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
+++ b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
@@ -63,6 +63,7 @@ services:
      myriad.mapreduce.jobhistory.webapp.address: 0
    envSettings: -Dcluster.name.prefix=/mycluster
    taskName: jobhistory
+   command: bin/mapred historyserver
  timelineserver:
    jvmMaxMemoryMB: 1024
    cpus: 1

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/resources/myriad-config-test-default.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default.yml b/myriad-scheduler/src/test/resources/myriad-config-test-default.yml
index f6f6b58..d07a318 100644
--- a/myriad-scheduler/src/test/resources/myriad-config-test-default.yml
+++ b/myriad-scheduler/src/test/resources/myriad-config-test-default.yml
@@ -60,18 +60,20 @@ yarnEnvironment:
 mesosAuthenticationPrincipal:
 mesosAuthenticationSecretFilename:
 services:
-   jobhistory:  
-     maxInstances: 1
-     jvmMaxMemoryMB: 1024
-     cpus: 1
-     ports:
-       myriad.mapreduce.jobhistory.admin.address: 0
-       myriad.mapreduce.jobhistory.address: 0
-       myriad.mapreduce.jobhistory.webapp.address: 0
-     envSettings: -Dcluster.name.prefix=/mycluster
-     taskName: jobhistory
-   timelineserver:
-     jvmMaxMemoryMB: 1024
-     cpus: 1
-     envSettings: -Dcluster.name.prefix=/mycluster2
-     taskName: timelineserver	
+  jobhistory:
+    maxInstances: 1
+    jvmMaxMemoryMB: 1024
+    cpus: 1
+    ports:
+      myriad.mapreduce.jobhistory.admin.address: 0
+      myriad.mapreduce.jobhistory.address: 0
+      myriad.mapreduce.jobhistory.webapp.address: 1
+    command: bin/mapred historyserver
+    envSettings: -Dcluster.name.prefix=/mycluster
+    taskName: jobhistory
+  timelineserver:
+    jvmMaxMemoryMB: 1024
+    cpus: 1
+    envSettings: -Dcluster.name.prefix=/mycluster2
+    taskName: timelineserver
+    command: bin/mapred historyserver



Mime
View raw message