myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smare...@apache.org
Subject [2/4] incubator-myriad git commit: Myriad-Issue-60 Added ability to launch JHS and other services that fit under Myriad umbrella from MyriadScheduler using default Mesos slave executor
Date Fri, 16 Oct 2015 20:39:54 GMT
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/46573816/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java
b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java
new file mode 100644
index 0000000..f05072d
--- /dev/null
+++ b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java
@@ -0,0 +1,68 @@
+package com.ebay.myriad.scheduler;
+
+import static org.junit.Assert.*;
+
+import org.apache.mesos.Protos.CommandInfo;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.ebay.myriad.configuration.MyriadConfiguration;
+import com.ebay.myriad.scheduler.TaskFactory.NMTaskFactoryImpl;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ * Class to test CommandLine generation
+ *
+ */
+public class TestServiceCommandLine {
+
+  static MyriadConfiguration cfg;
+  
+  static String toJHSCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo
chown hduser . &&" +
+      " cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred
historyserver\";" +
+      "sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf
/usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver";
+  
+  static String toCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown
hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;";
+  
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+    cfg = mapper.readValue(
+            Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"),
+            MyriadConfiguration.class);
+
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+  }
+
+  @Test
+  public void testJHSCommandLineGeneration() throws Exception {
+    ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null);
+    String executorCmd = "$YARN_HOME/bin/mapred historyserver";
+    ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0);
+    
+    CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd);
+     
+    assertTrue(cInfo.getValue().startsWith(toCompare));
+  }
+
+  @Test
+  public void testNMCommandLineGeneration() throws Exception {
+    Long [] ports = new Long [] {1L, 2L, 3L, 4L};
+    NMPorts nmPorts = new NMPorts(ports);
+    
+    ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L,
15L), 3.0, 5.0);
+    
+    ExecutorCommandLineGenerator clGenerator = new DownloadNMExecutorCLGenImpl(cfg, "hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz");
+    NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator);
+    
+    CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts);
+    
+    assertTrue(cInfo.getValue().startsWith(toCompare));
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/46573816/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java
new file mode 100644
index 0000000..ad2542b
--- /dev/null
+++ b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java
@@ -0,0 +1,103 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package com.ebay.myriad.scheduler;
+
+import static org.junit.Assert.*;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.ebay.myriad.configuration.MyriadBadConfigurationException;
+import com.ebay.myriad.configuration.MyriadConfiguration;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+/**
+ * Tests for TaskUtils
+ *
+ */
+public class TestTaskUtils {
+
+  static MyriadConfiguration cfg;
+  
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+    cfg = mapper.readValue(
+            Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"),
+            MyriadConfiguration.class);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+  }
+
+  @Test
+  public void testGetResource() {
+    TaskUtils taskUtils = new TaskUtils(cfg);
+    
+    NMProfile fooProfile = new NMProfile("abc", 1L, 1000L);
+    try {
+      taskUtils.getAuxTaskCpus(fooProfile, "foo");
+      fail("Should not complete sucessfully for foo");
+    } catch (MyriadBadConfigurationException e) {
+      // success
+    }
+    
+    try {
+      double cpu = taskUtils.getAuxTaskCpus(fooProfile, "jobhistory");
+      assertTrue(cpu > 0.0);
+    } catch (MyriadBadConfigurationException e) {
+      fail("cpu should be defined for jobhistory");
+    }    
+  }
+  
+  @Test
+  public void testServiceResourceProfile() throws Exception {
+    // testing custom deserializer
+    
+    Gson gson = new GsonBuilder().registerTypeAdapter(ServiceResourceProfile.class, new ServiceResourceProfile.CustomDeserializer()).create();
+    
+
+    ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0);
+
+    String parentStr = gson.toJson(parentProfile);
+    ServiceResourceProfile processedProfile = gson.fromJson(parentStr, ServiceResourceProfile.class);
+    
+    assertTrue(processedProfile.getClass().equals(ServiceResourceProfile.class));
+    assertTrue(processedProfile.toString().equalsIgnoreCase(parentStr));
+    
+    ServiceResourceProfile childProfile = new ExtendedResourceProfile(new NMProfile("bcd",
5L, 15L), 2.0, 7.0);
+    
+    String childStr = gson.toJson(childProfile);
+    ServiceResourceProfile processedChildProfile = gson.fromJson(childStr, ServiceResourceProfile.class);
+
+    assertTrue(processedChildProfile instanceof ExtendedResourceProfile);
+    assertTrue(processedChildProfile.toString().equalsIgnoreCase(childStr));
+  }
+
+  @Test
+  
+  public void testStackTrace() {
+    
+    new Throwable().printStackTrace();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/46573816/myriad-scheduler/src/test/resources/myriad-config-test-default.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default.yml b/myriad-scheduler/src/test/resources/myriad-config-test-default.yml
new file mode 100644
index 0000000..9698e14
--- /dev/null
+++ b/myriad-scheduler/src/test/resources/myriad-config-test-default.yml
@@ -0,0 +1,55 @@
+mesosMaster: 10.0.2.15:5050
+checkpoint: false
+frameworkFailoverTimeout: 43200000
+frameworkName: MyriadTest
+frameworkRole: 
+frameworkUser: hduser     # User the Node Manager runs as, required if nodeManagerURI set,
otherwise defaults to the user
+                          # running the resource manager.
+frameworkSuperUser: root  # To be depricated, currently permissions need set by a superuser
due to Mesos-1790.  Must be
+                          # root or have passwordless sudo. Required if nodeManagerURI set,
ignored otherwise.
+nativeLibrary: /usr/local/lib/libmesos.so
+zkServers: localhost:2181
+zkTimeout: 20000
+restApiPort: 8192
+profiles:
+  small:
+    cpu: 1
+    mem: 1100
+  medium:
+    cpu: 2
+    mem: 2048
+  large:
+    cpu: 4
+    mem: 4096
+rebalancer: false
+nodemanager:
+  jvmMaxMemoryMB: 1024
+  cpus: 0.2
+  cgroups: false
+executor:
+  jvmMaxMemoryMB: 256
+  path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar
+  #The following should be used for a remotely distributed URI, hdfs assumed but other URI
types valid.
+  nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz
+  #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.0.1.jar
+yarnEnvironment:
+  YARN_HOME: /usr/local/hadoop
+  #YARN_HOME: hadoop-2.5.0 #this should be relative if nodeManagerUri is set
+  #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary
+mesosAuthenticationPrincipal:
+mesosAuthenticationSecretFilename:
+services:
+   jobhistory:
+     jvmMaxMemoryMB: 1024
+     cpus: 1
+     ports:
+       myriad.mapreduce.jobhistory.admin.address: -1
+       myriad.mapreduce.jobhistory.address: -1
+       myriad.mapreduce.jobhistory.webapp.address: -1
+     envSettings: -Dcluster.name.prefix=/mycluster
+     taskName: jobhistory
+   timelineserver:
+     jvmMaxMemoryMB: 1024
+     cpus: 1
+     envSettings: -Dcluster.name.prefix=/mycluster2
+     taskName: timelineserver	
\ No newline at end of file


Mime
View raw message