myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dar...@apache.org
Subject incubator-myriad git commit: Added Docker Support, see the README.md in docker directory for quick start guide.
Date Thu, 12 May 2016 01:25:27 GMT
Repository: incubator-myriad
Updated Branches:
  refs/heads/master 519d9b421 -> bcd1fc8c6


Added Docker Support, see the README.md in docker directory for quick start guide.

Code changes:
- Added classes necessary to pass configuration to TaskFactory
- Modified classes necessary to consume the configuration changes
- Edits to docker/scripts mostly to break up the build

@Todo
- add support for MesosInfo (for AppC support), will be done as more information about the MesosInfo object appears.
- add configuration Uri to avoid rebuilding images (done, separate PR).
- clean up docker scripts more, including the ability to pass build arguments, will be incrementally done.

JIRA:
  [Myriad-136] https://issues.apache.org/jira/browse/MYRIAD-136
Pull Request:
  Closes #67
Author:
  DarinJ <darinj@apache.org>

Code changes:
- Added classes necessary to pass configuration to TaskFactory
- Modified classes necessary to consume the configuration changes
- Edits to docker/scripts mostly to break up the build

@Todo
- add support for MesosInfo (for AppC support), will be done as more information about the MesosInfo object appears.
- add configuration Uri to avoid rebuilding images (done, separate PR).
- clean up docker scripts more, including the ability to pass build arguments, will be incrementally done.


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/bcd1fc8c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/bcd1fc8c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/bcd1fc8c

Branch: refs/heads/master
Commit: bcd1fc8c6f6ea184522bc9a2c58610710a4be9b3
Parents: 519d9b4
Author: darinj <darinj.work@gmail.com>
Authored: Mon May 2 15:08:20 2016 -0400
Committer: darinj <darinj@apache.org>
Committed: Wed May 11 21:18:36 2016 -0400

----------------------------------------------------------------------
 build.gradle                                    |   8 +
 docker/Dockerfile                               |  21 ++-
 docker/README.md                                |  29 ++--
 docker/build-myriad.sh                          |   3 +-
 docker/build.gradle                             |  18 ++-
 docker/myriad-bin/configure-yarn.sh             |  60 +++++++
 docker/myriad-bin/create-user.sh                |  34 ----
 docker/myriad-bin/install-yarn.sh               |  54 ++-----
 docker/myriad-etc/mapred-site.xml.template      |  29 ++++
 .../myriad-config-default.yml.template          | 113 +++++++++++++
 docker/myriad-etc/yarn-site.xml.template        | 162 +++++++++++++++++++
 myriad-scheduler/build.gradle                   |   1 -
 .../configuration/MyriadConfiguration.java      |  30 +++-
 .../MyriadContainerConfiguration.java           |  54 +++++++
 .../MyriadDockerConfiguration.java              |  72 +++++++++
 .../scheduler/ServiceTaskFactoryImpl.java       |   3 +
 .../apache/myriad/scheduler/TaskFactory.java    |  11 +-
 .../org/apache/myriad/scheduler/TaskUtils.java  |  95 ++++++++++-
 .../apache/myriad/scheduler/TestTaskUtils.java  |  94 ++++++++++-
 ...iad-config-test-default-with-docker-info.yml |  90 +++++++++++
 ...-config-test-default-with-framework-role.yml |  70 ++++++++
 21 files changed, 934 insertions(+), 117 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 54e3eac..ac416ec 100644
--- a/build.gradle
+++ b/build.gradle
@@ -77,6 +77,13 @@ subprojects {
 
     configurations.create('myriadExecutorConf')
 
+    //Keeping guava at the level of hadoop 2.6.0 so we don't get all the
+    //awesomeness in our IDE and fail at runtime.
+    configurations.all {
+        resolutionStrategy {
+            force 'com.google.guava:guava:11.0.2'
+        }
+    }
     configurations {
         provided
         capsule
@@ -97,6 +104,7 @@ subprojects {
         compile 'com.google.code.gson:gson:2.3.1'        // marshalling between the scheduler and executor
 
         testCompile 'org.apache.zookeeper:zookeeper:3.4.6' // to resolve temporary mavenlocal issue
+        runtime 'com.google.guava:guava:11.0.2'
         testCompile 'junit:junit:4.12'
         testCompile 'commons-collections:commons-collections:3.2.1'
     }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 52ad7f3..4f344a6 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -24,6 +24,10 @@ MAINTAINER Apache Myriad dev@myriad.incubator.apache.org
 
 ENV HADOOP_USER="yarn"
 ENV HADOOP_VER="2.7.0"
+ENV HADOOP_HOME=/usr/local/hadoop
+ENV JAVA_HOME=/usr
+ENV MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so
+
 
 # Setup mesosphere repositories
 RUN apt-get -y update
@@ -32,24 +36,17 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF
 RUN DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') CODENAME=$(lsb_release -cs) && echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | tee /etc/apt/sources.list.d/mesosphere.list
 RUN apt-get -y update
 
-ADD myriad-bin/create-user.sh /create-user.sh
-RUN sh /create-user.sh
-
 # Install Mesos
 RUN apt-get install -y mesos curl tar
-
 # Run local YARN install
 ADD myriad-bin/install-yarn.sh /install-yarn.sh
 RUN sh /install-yarn.sh
-
+ADD myriad-bin/configure-yarn.sh /configure-yarn.sh
+RUN /configure-yarn.sh
 # Copy over myriad libraries
 ADD ./libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
-
-
 # Initialize hadoop confs with env vars
-ADD myriad-bin/run-myriad.sh /run-myriad.sh
-RUN mkdir /myriad-conf/
-
-# Run the YARN resource manager
+ADD myriad-etc/* /usr/local/hadoop/etc/hadoop/
+RUN chown -R root:root /usr/local/hadoop/etc
 USER yarn
-CMD sh /run-myriad.sh 
+CMD /usr/local/hadoop/bin/yarn resourcemanager

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/README.md
----------------------------------------------------------------------
diff --git a/docker/README.md b/docker/README.md
index 7ce77ac..1650691 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -2,25 +2,36 @@
 Docker is the easiest way to from 0 to Yarn on Mesos within minutes. 
 
 ## Building the Resource Manager Docker
-Run the following command:
-```./gradlew buildRMDocker```
-This will build the ResourceManager from src and save the image as *mesos/myriad*.
 
 #Configuration Guide#
 
-In order for the ResourceManager to operate correctly, you will need to provide 2 configuration files. These files will need to mounted from a directory into */myriad-conf* within the docker container.
+In order for the ResourceManager to operate correctly, you will need to provide 5 configuration files. These files will 
+need to mounted from a directory into `./myriad-etc` of this folder.
 
-* [myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml)
-* modified [yarn-site.xml](https://github.com/mesos/myriad/blob/phase1/docs/myriad-dev.md)
+* [myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml) (template provided)
+* [yarn-site.xml](https://github.com/mesos/myriad/blob/phase1/docs/myriad-dev.md) (template provided)
+* mapred-site.xml (template provided)
+* hdfs-site.xml (used for hdfs)
+* core-site.xml (used for hdfs)
 
+It is assumed you already have hdfs and Mesos running.  For more information about Apache Mesos visit the [website](http://mesos.apache.org). 
+If you need to setup hdfs, consider using the [hdfs-mesos framework](https://github.com/mesosphere/hdfs).  Copy the hdfs.xml and 
+core-site.xml from your hdfs configuration into myriad-etc.  Additional, files maybe necessary such as rack.sh, log4j.properties, 
+hadoop-env.sh, and yarn-env.sh depending on your configuration.
+
+Run the following commands:
+```bash
+./gradlew -P username/myriad buildRMDocker
+docker push username/myriad
+```
+This will build the ResourceManager from src, save, and push the image as *username/myriad*.
 
 ## Running the Resource Manager Docker
 
 ```bash
 docker run --net=host --name='myriad-resourcemanager' -t \
-  -v /path/to/configs:/myriad-conf \
-  mesos/myriad
-  ```
+  <username>/myriad
+```
 
 #Environment Variables#
 * *ALTERNATE_HADOOP_URL* : Optional - Allows user to override the hadoop distribution used by Myriad. This will download the *.tar.gz file to be used as the hadoop distribution of choice for Myriad. 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/build-myriad.sh
----------------------------------------------------------------------
diff --git a/docker/build-myriad.sh b/docker/build-myriad.sh
index d3b0e74..b0638d0 100644
--- a/docker/build-myriad.sh
+++ b/docker/build-myriad.sh
@@ -36,6 +36,7 @@ if [ ! -d "libs" ]; then
 fi
 
 # Copy over the Java Libraries
-cp -rf ../myriad-scheduler/build/libs/* libs/
 
+cp -rf ../myriad-scheduler/build/libs/* libs/
+cp -rf ../myriad-executor/build/libs/* libs/
 echo "Init complete! " #Modify config/myriad-default-config.yml to your liking before building the docker image"

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/build.gradle
----------------------------------------------------------------------
diff --git a/docker/build.gradle b/docker/build.gradle
index 4f35494..38f61e6 100644
--- a/docker/build.gradle
+++ b/docker/build.gradle
@@ -23,15 +23,25 @@ import com.bmuschko.gradle.docker.tasks.image.Dockerfile
 import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
 
 task copySchedulerJars(type:Copy){
+    dependsOn(":myriad-scheduler:build")
     def subdir = new File(project.buildDir, "libs")
     if( !subdir.exists() ) { subdir.mkdirs() }
     from '../myriad-scheduler/build/libs'
     into 'libs'
 }
 
-task copyMyriadJars(dependsOn: ":myriad-scheduler:build") {
+task copyExecutorJars(type:Copy){
     dependsOn(":myriad-executor:build")
+    def subdir = new File(project.buildDir, "libs")
+    if( !subdir.exists() ) { subdir.mkdirs() }
+    from '../myriad-executor/build/libs'
+    into 'libs'
+}
+
+
+task copyMyriadJars(dependsOn: ":myriad-scheduler:build") {
     dependsOn(copySchedulerJars)
+    dependsOn(copyExecutorJars)
 }
 
 task buildRMDocker(type: DockerBuildImage) {
@@ -46,5 +56,9 @@ task buildRMDocker(type: DockerBuildImage) {
         url = 'unix:///var/run/docker.sock'
     }  
     inputDir = file('.')
-    tag = 'mesos/myriad'
+    if(!project.hasProperty('dockerTag')) {
+        tag = 'mesos/myriad'
+    } else {
+        tag = dockerTag
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-bin/configure-yarn.sh
----------------------------------------------------------------------
diff --git a/docker/myriad-bin/configure-yarn.sh b/docker/myriad-bin/configure-yarn.sh
new file mode 100755
index 0000000..ff85ba9
--- /dev/null
+++ b/docker/myriad-bin/configure-yarn.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+: '
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+'
+##
+# YARN installer script for Apache Myriad Deployment
+##
+
+# Put in env defaults if they are missing
+export HADOOP_GROUP=${HADOOP_GROUP:='hadoop'}
+
+export YARN_USER=${YARN_USER:='yarn'}
+export USER_UID=${USER_UID:='107'}
+export YARN_GROUP=${YARN_GROUP:='yarn'}
+export HADOOP_GID=${HADOOP_GID:='112'}
+export YARN_GID=${YARN_GID:='113'}
+export HADOOP_HOME=${HADOOP_HOME:='/usr/local/hadoop'}
+
+# Add hduser user
+groupadd ${HADOOP_GROUP} -g ${HADOOP_GID}
+groupadd ${YARN_GROUP} -g ${YARN_GID}
+useradd ${YARN_USER} -g ${YARN_GROUP} -G ${HADOOP_GROUP} -u ${USER_UID} -s /bin/bash
+mkdir /home/${HADOOP_USER}
+chown -R ${YARN_USER}:${YARN_GROUP} /home/${YARN_USER}
+
+#set permissions
+chown -R root:root ${HADOOP_HOME}
+chmod -R g-w /usr/local/
+chown root:${YARN_GROUP} ${HADOOP_HOME}/bin/container-executor
+chmod 6050 ${HADOOP_HOME}/bin/container-executor
+
+# Init bashrc with hadoop env variables
+sh -c 'echo export JAVA_HOME=/usr >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/bin >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/sbin >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_MAPRED_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_COMMON_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_HDFS_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export YARN_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_HOME\}/lib/native >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\${HADOOP_HOME}/lib\" >> /home/${HADOOP_USER}/.bashrc'
+
+
+echo "end of configure-yarn.sh script"

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-bin/create-user.sh
----------------------------------------------------------------------
diff --git a/docker/myriad-bin/create-user.sh b/docker/myriad-bin/create-user.sh
deleted file mode 100755
index c6c1bf6..0000000
--- a/docker/myriad-bin/create-user.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-: '
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
-'
-# Put in env defaults if they are missing
-export HADOOP_GROUP=${HADOOP_GROUP:='hadoop'}
-export HADOOP_USER=${HADOOP_USER:='yarn'}
-export HADOOP_HOME=${HADOOP_HOME:='/usr/local/hadoop'}
-export USER_UID=${USER_UID:='113'}
-export GROUP_UID=${GROUP_GID:='112'}
-
-# Add hduser user
-echo "Creating $HADOOP_USER user.."
-groupadd $HADOOP_GROUP -g ${GROUP_UID}
-useradd $HADOOP_USER -g $HADOOP_GROUP -u ${USER_UID} -s /bin/bash -d /home/${HADOOP_USER}
-mkdir /home/${HADOOP_USER}
-chown -R $HADOOP_USER:$HADOOP_GROUP /home/${HADOOP_USER}
-
-echo "end of create-user.sh script"

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-bin/install-yarn.sh
----------------------------------------------------------------------
diff --git a/docker/myriad-bin/install-yarn.sh b/docker/myriad-bin/install-yarn.sh
index 41b23ed..90c2762 100755
--- a/docker/myriad-bin/install-yarn.sh
+++ b/docker/myriad-bin/install-yarn.sh
@@ -21,60 +21,28 @@ under the License.
 # YARN installer script for Apache Myriad Deployment
 ##
 HADOOP_VER="2.7.0"
-HADOOP_TARBALL_URL=http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz
 
 echo "Installing Yarn...."
-if [ ! -z "$1" ];then
-  HADOOP_TARBALL_URL=$1
-  echo "Deleting previous hadoop home"
-  rm -rf ${HADOOP_HOME}
+if [ ! -z $1 ];then
+  HADOOP_URL=$1
+else
+  HADOOP_URL=http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz
 fi
-
-# Download the tarball
-wget -O /opt/hadoop.tgz ${HADOOP_TARBALL_URL}
-HADOOP_BASENAME=`basename ${HADOOP_TARBALL_URL} .tar.gz`
-
-# Put in env defaults if they are missing
-export HADOOP_GROUP=${HADOOP_GROUP:='hadoop'}
-export HADOOP_USER=${HADOOP_USER:='yarn'}
+HADOOP_TGZ=`basename ${HADOOP_URL}`
+HADOOP_BASENAME=`basename ${HADOOP_URL} .tar.gz`
 export HADOOP_HOME=${HADOOP_HOME:='/usr/local/hadoop'}
-export USER_UID=${USER_UID:='113'}
-export GROUP_UID=${GROUP_GID:='112'}
-
-# Add hduser user
-groupadd $HADOOP_GROUP -g ${GROUP_UID}
-useradd $HADOOP_USER -g $HADOOP_GROUP -u ${USER_UID} -s /bin/bash
-#mkdir /home/${HADOOP_USER}
-chown -R $HADOOP_USER:$HADOOP_GROUP /home/${HADOOP_USER}
 
 # Extract Hadoop
-tar vxzf /opt/hadoop.tgz -C /tmp
-#mv /tmp/hadoop-${HADOOP_VER} ${HADOOP_HOME}
-echo "Moving /tmp/hadoop-${HADOOP_BASENAME} to ${HADOOP_HOME}"
+echo "Downloading ${HADOOP_TGZ} from ${HADOOP_URL}"
+wget ${HADOOP_URL}
+tar xzf ${HADOOP_TGZ} -C /tmp
 mv /tmp/${HADOOP_BASENAME} ${HADOOP_HOME}
-ls -lath ${HADOOP_HOME}
-
-mkdir /home/$HADOOP_USER
-chown -R ${HADOOP_USER}:${HADOOP_GROUP} ${HADOOP_HOME}
-
-# Init bashrc with hadoop env variables
-sh -c 'echo export JAVA_HOME=/usr >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/bin >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/sbin >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_MAPRED_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_COMMON_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_HDFS_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export YARN_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_HOME\}/lib/native >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\${HADOOP_HOME}/lib\" >> /home/${HADOOP_USER}/.bashrc'
+#Remove tarball
+rm -f ${HADOOP_TGZ}
 
 # Link Mesos Libraries
-touch ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
 echo "export JAVA_HOME=/usr" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
 echo "export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
-
 # Ensure the hadoop-env is executable
 chmod +x ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
-
 echo "end of install-yarn.sh script"

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-etc/mapred-site.xml.template
----------------------------------------------------------------------
diff --git a/docker/myriad-etc/mapred-site.xml.template b/docker/myriad-etc/mapred-site.xml.template
new file mode 100644
index 0000000..b21660f
--- /dev/null
+++ b/docker/myriad-etc/mapred-site.xml.template
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+<!--s option enables dynamic port assignment by mesos -->
+<configuration>
+<property>
+  <name>mapreduce.shuffle.port</name>
+  <value>${myriad.mapreduce.shuffle.port}</value>
+</property>
+<property>
+  <name>mapreduce.framework.name</name>
+  <value>yarn</value>
+</property>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-etc/myriad-config-default.yml.template
----------------------------------------------------------------------
diff --git a/docker/myriad-etc/myriad-config-default.yml.template b/docker/myriad-etc/myriad-config-default.yml.template
new file mode 100644
index 0000000..6a1e52a
--- /dev/null
+++ b/docker/myriad-etc/myriad-config-default.yml.template
@@ -0,0 +1,113 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##
+
+# Address of the mesos master - <IP:port> or ZooKeeper path
+mesosMaster: zk://CHANGEME:2181/mesos
+#Container information for the node managers
+containerInfo:
+    type: DOCKER
+    dockerInfo:
+        image: CHANGEME/myriad
+    volume:
+        -
+          containerPath: /tmp
+          hostPath: /tmp
+# Whether to check point myriad's mesos framework or not
+checkpoint: false
+# Myriad's mesos framework failover timeout in milliseconds. This tells mesos
+# to expect myriad would failover within this time interval.
+frameworkFailoverTimeout: 60000 #10 minutes
+# Myriad's mesos framework name.
+frameworkName: MyriadAlpha
+# Myriad's mesos framework role.
+#frameworkRole: myriad
+# User the Node Manager will run as (Defaults to user running the resource manager if absent,  necessary for remote distribution).
+frameworkUser: yarn
+# User that gets the nodeManagerUri and sets up the directories for Node Manager, must have passwordless sudo (Necessary only for remote distribution, otherwise ignored).
+# frameworkSuperUser: root
+# Myriad's REST-ful services port mapping.
+restApiPort: 8192
+# Address of the ZK ensemble (separate by comma, if multiple zk servers are used)
+#zkServers: localhost:2181
+# ZK Session timeout
+zkTimeout: 20000
+haEnabled: True
+# The node manager profiles. The REST API to flex up expects one of the profiles defined here.
+# Admin can define custom profiles (requires restart of Resource Manager)
+nmInstances:
+  medium: 1
+profiles:
+  zero:
+    cpu: 0
+    mem: 0
+  small:
+    cpu: 1
+    mem: 1024
+  medium:
+    cpu: 2
+    mem: 2048
+  large:
+    cpu: 4
+    mem: 4096
+  extralarge:
+    cpu: 6
+    mem: 6144
+# Whether to turn on myriad's auto-rebalancer feature.
+# Currently it's work-in-progress and should be set to 'false'.   
+rebalancer: false
+# Properties for the Node Manager process that's launched by myriad as a result of 'flex up' REST call.
+nodemanager:
+  jvmMaxMemoryMB: 1024  # Xmx for NM JVM process.
+  cpus: 0.2             # CPU needed by NM process.
+#  cgroups: true         # Whether NM should support CGroups. If set to 'true', myriad automatically
+                        # configures yarn-site.xml to attach YARN's cgroups under Mesos' cgroup hierarchy.
+executor:
+  jvmMaxMemoryMB: 256   # Xmx for myriad's executor that launches Node Manager.
+  #path: hdfs://172.31.2.176:54310/dist/myriad-executor-runnable-x.x.x.jar # Path for the myriad's executor binary.
+  nodeManagerUri: hdfs://172.31.2.167:54310/dist/hadoop-2.6.0.tgz # the uri to d/l hadoop from   # Path to the Hadoop tarball
+# Environment variables required to launch Node Manager process. Admin can also pass other environment variables to NodeManager.
+yarnEnvironment:
+  YARN_HOME: /usr/local/hadoop # Or /usr/local/hadoop if using MapR's Hadoop
+  #YARN_HOME: hadoop-2.6.0 # Should be relative nodeManagerUri is set
+  #YARN_NODEMANAGER_OPTS: -Dnodemanager.resource.io-spindles=4.0 # Required only if using MapR's Hadoop
+  JAVA_HOME: /usr/ # System dependent, but sometimes necessary
+services:
+   jobhistory:
+     command: $YARN_HOME/bin/mapred historyserver
+     jvmMaxMemoryMB: 1024
+     cpus: 1
+     maxInstances: 1
+     #ports:
+      # myriad.mapreduce.jobhistory.admin.address: -1
+      # myriad.mapreduce.jobhistory.address: -1
+      # myriad.mapreduce.jobhistory.webapp.address: -1
+     envSettings: -Dcluster.name.prefix=/mycluster
+     taskName: jobhistory
+  timelineserver:
+     command: $YARN_HOME/bin/yarn timelineserver
+     jvmMaxMemoryMB: 1024
+     cpus: 1
+     envSettings: -Dcluster.name.prefix=/mycluster2
+     taskName: timelineserver
+
+
+# Authentication principal for Myriad's mesos framework
+#mesosAuthenticationPrincipal: some_principal
+# Authentication secret filename for Myriad's mesos framework
+#mesosAuthenticationSecretFilename: /path/to/secret/filename

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-etc/yarn-site.xml.template
----------------------------------------------------------------------
diff --git a/docker/myriad-etc/yarn-site.xml.template b/docker/myriad-etc/yarn-site.xml.template
new file mode 100644
index 0000000..c81b124
--- /dev/null
+++ b/docker/myriad-etc/yarn-site.xml.template
@@ -0,0 +1,162 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+
+
+<configuration>
+
+  <!--The hostname of the Resource Manager, this MUST be changed! -->
+  <!--If you plan to deploy via marathon, use <taskId>.marathon.mesos where taskId is the name of your marathon task-->
+  <property>
+    <name>yarn.resourcemanager.hostname</name>
+    <value>CHANGEME</value>
+  </property>
+
+  <!-- Necessary for log aggregation -->
+  <property>
+    <name>yarn.log-aggregation-enable</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <description>Where to aggregate logs to.</description>
+    <name>yarn.nodemanager.remote-app-log-dir</name>
+    <value>/var/log/hadoop-yarn/apps</value>
+  </property>
+
+  <!-- Aux Services needed for Map/Reduce and Myriad -->
+  <property>
+    <name>yarn.nodemanager.aux-services</name>
+    <value>mapreduce_shuffle,myriad_executor</value>
+  </property>
+
+  <property>
+    <name>yarn.resourcemanager.store.class</name>
+    <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore</value>
+  </property>
+
+  <property>
+    <name>yarn.resourcemanager.fs.state-store.uri</name>
+    <value>/var/yarn/rm/system</value>
+  </property>
+
+  <property>
+    <name>yarn.resourcemanager.recovery.enabled</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
+    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services.myriad_executor.class</name>
+    <value>org.apache.myriad.executor.MyriadExecutorAuxService</value>
+  </property>
+
+  <!-- Myriad specific YARN configuration properties-->
+  <property>
+    <name>yarn.nodemanager.resource.cpu-vcores</name>
+    <value>${nodemanager.resource.cpu-vcores}</value>
+   </property>
+   <property>
+     <name>yarn.scheduler.minimum-allocation-vcores</name>
+     <value>0</value>
+   </property>
+
+  <property>
+    <name>yarn.nodemanager.resource.memory-mb</name>
+    <value>${nodemanager.resource.memory-mb}</value>
+  </property>
+  <property>
+    <name>yarn.scheduler.minimum-allocation-mb</name>
+    <value>0</value>
+  </property>
+
+  <!-- Dynamic Port Assignment enabled by Mesos -->
+  <property>
+    <name>yarn.nodemanager.address</name>
+    <value>${myriad.yarn.nodemanager.address}</value>
+  </property>
+  <property>
+    <name>yarn.nodemanager.webapp.address</name>
+    <value>${myriad.yarn.nodemanager.webapp.address}</value>
+  </property>
+  <property>
+    <name>yarn.nodemanager.webapp.https.address</name>
+    <value>${myriad.yarn.nodemanager.webapp.address}</value>
+  </property>
+  <property>
+    <name>yarn.nodemanager.localizer.address</name>
+    <value>${myriad.yarn.nodemanager.localizer.address}</value>
+  </property>
+
+  <!-- Myriad Scheduler configuration -->
+  <property>
+    <name>yarn.resourcemanager.scheduler.class</name>
+    <value>org.apache.myriad.scheduler.yarn.MyriadFairScheduler</value>
+  </property>
+
+  <!-- Cgroups configuration -->
+  <!--
+  <property>
+    <description>who will execute(launch) the containers.</description>
+    <name>yarn.nodemanager.container-executor.class</name>
+    <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
+  </property>
+
+  <property>
+    <description>The class which should help the LCE handle resources.</description>
+    <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
+    <value>org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler</value>
+  </property>
+
+  <property>
+    <description>The class which should help the LCE handle resources.</description>
+    <name>yarn.nodemanager.linux-container-executor.group</name>
+    <value>yarn</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
+    <value></value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
+    <value>${yarn.nodemanager.linux-container-executor.cgroups.mount-path}</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.group</name>
+    <value>root</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.path</name>
+    <value>${yarn.home}/bin/container-executor</value>
+  </property>
+-->
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/build.gradle
----------------------------------------------------------------------
diff --git a/myriad-scheduler/build.gradle b/myriad-scheduler/build.gradle
index fc7516d..3e88c03 100644
--- a/myriad-scheduler/build.gradle
+++ b/myriad-scheduler/build.gradle
@@ -38,7 +38,6 @@ dependencies {
     compile "org.apache.commons:commons-lang3:3.4"
     compile 'com.google.inject.extensions:guice-multibindings:3.0'
     testCompile "org.apache.hadoop:hadoop-yarn-server-resourcemanager:${hadoopVer}:tests"
-
 }
 
 configurations {

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
index 67c8232..716a20a 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Optional;
 import com.google.common.base.Strings;
 import java.util.Map;
+
 import org.codehaus.jackson.map.annotate.JsonSerialize;
 import org.hibernate.validator.constraints.NotEmpty;
 
@@ -53,8 +54,27 @@ import org.hibernate.validator.constraints.NotEmpty;
  * executor:
  * jvmMaxMemoryMB: 256
  * path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar
+ * containerInfo:
+ *   DockerInfo:
+ *     image: mesos/myriad
+ *     forcePullImage: false
+ *     parameters:
+ *       -
+ *         key: key
+ *         value: value
+ *       -
+ *         key: key
+ *         value: value
+ *   volumes:
+ *     -
+ *       containerPath: path
+ *       hostPath: path
+ *       mode: RW
+ *     -
+ *       containerPath: path
+ *       hostPath: path
  * yarnEnvironment:
- * YARN_HOME: /usr/local/hadoop
+ *   YARN_HOME: /usr/local/hadoop
  */
 public class MyriadConfiguration {
   /**
@@ -93,6 +113,9 @@ public class MyriadConfiguration {
   private Boolean checkpoint;
 
   @JsonProperty
+  private MyriadContainerConfiguration containerInfo;
+
+  @JsonProperty
   private Double frameworkFailoverTimeout;
 
   @JsonProperty
@@ -160,7 +183,6 @@ public class MyriadConfiguration {
   public MyriadConfiguration() {
   }
 
-
   public String getMesosMaster() {
     return mesosMaster;
   }
@@ -169,6 +191,10 @@ public class MyriadConfiguration {
     return this.checkpoint != null ? checkpoint : DEFAULT_CHECKPOINT;
   }
 
+  public Optional<MyriadContainerConfiguration> getContainerInfo() {
+    return Optional.fromNullable(containerInfo);
+  }
+
   public Double getFrameworkFailoverTimeout() {
     return this.frameworkFailoverTimeout != null ? this.frameworkFailoverTimeout : DEFAULT_FRAMEWORK_FAILOVER_TIMEOUT_MS;
   }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadContainerConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadContainerConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadContainerConfiguration.java
new file mode 100644
index 0000000..6a6a8bc
--- /dev/null
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadContainerConfiguration.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+import org.hibernate.validator.constraints.NotEmpty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * MyriadContainerConfiguration
+ * Provides deserialization support for containerInfo data stored in yaml config
+ */
+public class MyriadContainerConfiguration {
+  @JsonProperty
+  @NotEmpty
+  private String type;
+  @JsonProperty
+  private MyriadDockerConfiguration dockerInfo;
+  @JsonProperty
+  private List<Map<String, String>> volumes;
+
+  @JsonProperty
+  public String getType() {
+    return type;
+  }
+
+  public Optional<MyriadDockerConfiguration> getDockerInfo() {
+    return Optional.fromNullable(dockerInfo);
+  }
+
+  public Iterable<Map<String, String>> getVolumes() {
+    return volumes == null ? new ArrayList<Map<String, String>>() : volumes;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadDockerConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadDockerConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadDockerConfiguration.java
new file mode 100644
index 0000000..38a2f0f
--- /dev/null
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadDockerConfiguration.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.hibernate.validator.constraints.NotEmpty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * MyriadDockerConfiguration
+ * Provides deserialization support for dockerInfo data stored in yaml config
+ */
+public class MyriadDockerConfiguration {
+  @JsonProperty
+  @NotEmpty
+  String image;
+
+  @JsonProperty
+  String network;
+
+  @JsonProperty
+  Boolean privledged;
+
+  @JsonProperty
+  Boolean forcePullImage;
+
+  @JsonProperty
+  List<Map<String, String>> parameters;
+
+  public String getImage() {
+    return image;
+  }
+
+  public Boolean getForcePullImage() {
+    return forcePullImage != null ? forcePullImage : false;
+  }
+
+  public String getNetwork() {
+    return network != null ? network : "HOST";
+  }
+
+  public Boolean getPrivledged() {
+    return privledged != null ? privledged : false;
+  }
+
+  public List<Map<String, String>> getParameters() {
+    if (parameters == null) {
+      return new ArrayList<>();
+    } else {
+      return parameters;
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java
index 076ebbb..dd86ac0 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java
@@ -140,6 +140,9 @@ public class ServiceTaskFactoryImpl implements TaskFactory {
       taskBuilder.addResources(Resource.newBuilder().setName("ports").setType(Value.Type.RANGES).setRanges(valueRanger.build()));
     }
     taskBuilder.setCommand(commandInfo);
+    if (cfg.getContainerInfo().isPresent()) {
+      taskBuilder.setContainer(taskUtils.getContainerInfo());
+    }
     return taskBuilder.build();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
index ecf0276..9ed54a7 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
@@ -201,10 +201,13 @@ public interface TaskFactory {
       ExecutorID executorId = ExecutorID.newBuilder()
           .setValue(EXECUTOR_PREFIX + frameworkId.getValue() + offer.getId().getValue() + offer.getSlaveId().getValue())
           .build();
-      return ExecutorInfo.newBuilder().setCommand(commandInfo).setName(EXECUTOR_NAME).setExecutorId(executorId)
-          .addAllResources(taskUtils.getScalarResource(offer, "cpus", taskUtils.getExecutorCpus(), 0.0))
-          .addAllResources(taskUtils.getScalarResource(offer, "mem", taskUtils.getExecutorMemory(), 0.0))
-          .build();
+      ExecutorInfo.Builder executorInfo = ExecutorInfo.newBuilder().setCommand(commandInfo).setName(EXECUTOR_NAME).setExecutorId(executorId)
+              .addAllResources(taskUtils.getScalarResource(offer, "cpus", taskUtils.getExecutorCpus(), 0.0))
+              .addAllResources(taskUtils.getScalarResource(offer, "mem", taskUtils.getExecutorMemory(), 0.0));
+      if (cfg.getContainerInfo().isPresent()) {
+        executorInfo.setContainer(taskUtils.getContainerInfo());
+      }
+      return executorInfo.build();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
index 845c3be..1d9c518 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
@@ -18,11 +18,15 @@
  */
 package org.apache.myriad.scheduler;
 
+import com.google.common.base.Function;
 import com.google.common.base.Optional;
+import com.google.common.collect.Iterables;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.StringWriter;
-import java.util.ArrayList;
+import java.util.*;
+import javax.annotation.Nullable;
 import javax.inject.Inject;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -40,12 +44,9 @@ import javax.xml.xpath.XPathExpression;
 import javax.xml.xpath.XPathExpressionException;
 import javax.xml.xpath.XPathFactory;
 
+import com.google.common.base.Preconditions;
 import org.apache.mesos.Protos;
-import org.apache.myriad.configuration.MyriadBadConfigurationException;
-import org.apache.myriad.configuration.MyriadConfiguration;
-import org.apache.myriad.configuration.MyriadExecutorConfiguration;
-import org.apache.myriad.configuration.NodeManagerConfiguration;
-import org.apache.myriad.configuration.ServiceConfiguration;
+import org.apache.myriad.configuration.*;
 import org.apache.myriad.executor.MyriadExecutorDefaults;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -60,11 +61,19 @@ import org.xml.sax.SAXException;
  */
 public class TaskUtils {
   private static final Logger LOGGER = LoggerFactory.getLogger(TaskUtils.class);
-
   private static final String YARN_NODEMANAGER_RESOURCE_CPU_VCORES = "yarn.nodemanager.resource.cpu-vcores";
   private static final String YARN_NODEMANAGER_RESOURCE_MEMORY_MB = "yarn.nodemanager.resource.memory-mb";
+  private static final String CONTAINER_PATH_KEY = "containerPath";
+  private static final String HOST_PATH_KEY = "hostPath";
+  private static final String RW_MODE = "mode";
+  private static final String CONTAINER_PORT_KEY = "containerPort";
+  private static final String HOST_PORT_KEY = "hostPort";
+  private static final String PROTOCOL_KEY = "protocol";
+  private static final String PARAMETER_KEY_KEY = "key";
+  private static final String PARAMETER_VALUE_KEY = "value";
 
   private MyriadConfiguration cfg;
+  Random random = new Random();
 
   @Inject
   public TaskUtils(MyriadConfiguration cfg) {
@@ -208,14 +217,84 @@ public class TaskUtils {
     return auxConf.getJvmMaxMemoryMB().get();
   }
 
+  public TaskUtils() {
+    super();
+  }
+
+  public Iterable<Protos.Volume> getVolumes(Iterable<Map<String, String>> volume) {
+    return Iterables.transform(volume, new Function<Map<String, String>, Protos.Volume>() {
+      @Nullable
+      @Override
+      public Protos.Volume apply(Map<String, String> map) {
+        Preconditions.checkArgument(map.containsKey(HOST_PATH_KEY) && map.containsKey(CONTAINER_PATH_KEY));
+        Protos.Volume.Mode mode = Protos.Volume.Mode.RO;
+        if (map.containsKey(RW_MODE) && map.get(RW_MODE).toLowerCase().equals("rw")) {
+          mode = Protos.Volume.Mode.RW;
+        }
+        return Protos.Volume.newBuilder()
+            .setContainerPath(map.get(CONTAINER_PATH_KEY))
+            .setHostPath(map.get(HOST_PATH_KEY))
+            .setMode(mode)
+            .build();
+      }
+    });
+  }
+
+  public Iterable<Protos.Parameter> getParameters(Iterable<Map<String, String>> params) {
+    Preconditions.checkNotNull(params);
+    return Iterables.transform(params, new Function<Map<String, String>, Protos.Parameter>() {
+      @Override
+      public Protos.Parameter apply(Map<String, String> parameter) {
+        Preconditions.checkNotNull(parameter, "Null parameter");
+        Preconditions.checkState(parameter.containsKey(PARAMETER_KEY_KEY), "Missing key");
+        Preconditions.checkState(parameter.containsKey(PARAMETER_VALUE_KEY), "Missing value");
+        return Protos.Parameter.newBuilder()
+            .setKey(parameter.get(PARAMETER_KEY_KEY))
+            .setValue(PARAMETER_VALUE_KEY)
+            .build();
+      }
+    });
+  }
+
+  private Protos.ContainerInfo.DockerInfo getDockerInfo(MyriadDockerConfiguration dockerConfiguration) {
+    Preconditions.checkArgument(dockerConfiguration.getNetwork().equals("HOST"), "Currently only host networking supported");
+    Protos.ContainerInfo.DockerInfo.Builder dockerBuilder = Protos.ContainerInfo.DockerInfo.newBuilder()
+        .setImage(dockerConfiguration.getImage())
+        .setForcePullImage(dockerConfiguration.getForcePullImage())
+        .setNetwork(Protos.ContainerInfo.DockerInfo.Network.valueOf(dockerConfiguration.getNetwork()))
+        .setPrivileged(dockerConfiguration.getPrivledged())
+        .addAllParameters(getParameters(dockerConfiguration.getParameters()));
+    return dockerBuilder.build();
+  }
+
+  /**
+   * Builds a ContainerInfo Object
+   *
+   * @return ContainerInfo
+   */
+  public Protos.ContainerInfo getContainerInfo() {
+    Preconditions.checkArgument(cfg.getContainerInfo().isPresent(), "ContainerConfiguration doesn't exist!");
+    MyriadContainerConfiguration containerConfiguration = cfg.getContainerInfo().get();
+    Protos.ContainerInfo.Builder containerBuilder = Protos.ContainerInfo.newBuilder()
+        .setType(Protos.ContainerInfo.Type.valueOf(containerConfiguration.getType()))
+        .addAllVolumes(getVolumes(containerConfiguration.getVolumes()));
+    if (containerConfiguration.getDockerInfo().isPresent()) {
+      MyriadDockerConfiguration dockerConfiguration = containerConfiguration.getDockerInfo().get();
+      containerBuilder.setDocker(getDockerInfo(dockerConfiguration));
+    }
+    return containerBuilder.build();
+  }
+
+
   /**
    * Helper function that returns all scalar resources of a given name in an offer up to a given value.  Attempts to
    * take resource from the prescribed role first and then from the default role.  The variable used indicated any
    * resources previously requested.   Assumes enough resources are present.
+   *
    * @param offer - An offer by Mesos, assumed to have enough resources.
    * @param name  - The name of the SCALAR resource, i.e. cpus or mem
    * @param value - The amount of SCALAR resources needed.
-   * @param used - The amount of SCALAR resources already removed from this offer.
+   * @param used  - The amount of SCALAR resources already removed from this offer.
    * @return An Iterable containing one or two scalar resources of a given name in an offer up to a given value.
    */
   public Iterable<Protos.Resource> getScalarResource(Protos.Offer offer, String name, Double value, Double used) {

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
index a04b94b..14e0a00 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
@@ -19,13 +19,17 @@ package org.apache.myriad.scheduler;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import org.apache.mesos.Protos;
 import org.apache.myriad.configuration.MyriadBadConfigurationException;
 import org.apache.myriad.configuration.MyriadConfiguration;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -35,12 +39,20 @@ import static org.junit.Assert.fail;
 public class TestTaskUtils {
 
   static MyriadConfiguration cfg;
+  static MyriadConfiguration cfgWithRole;
+  static MyriadConfiguration cfgWithDocker;
+  static double epsilon = .0001;
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
     cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"),
         MyriadConfiguration.class);
+    cfgWithRole = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default-with-framework-role.yml"),
+        MyriadConfiguration.class);
+    cfgWithDocker = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default-with-docker-info.yml"),
+            MyriadConfiguration.class);
+
   }
 
   @AfterClass
@@ -74,7 +86,6 @@ public class TestTaskUtils {
     Gson gson = new GsonBuilder().registerTypeAdapter(ServiceResourceProfile.class, new ServiceResourceProfile.CustomDeserializer())
         .create();
 
-
     ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0);
 
     String parentStr = gson.toJson(parentProfile);
@@ -98,4 +109,85 @@ public class TestTaskUtils {
 
     new Throwable().printStackTrace();
   }
+  private Protos.Offer createScalarOffer(String name, double roleVal, double defaultVal) {
+    Protos.Offer offer = Protos.Offer.newBuilder()
+        .setId(Protos.OfferID.newBuilder().setValue("offerId"))
+        .setSlaveId(Protos.SlaveID.newBuilder().setValue("slaveId"))
+        .setHostname("test.com")
+        .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("frameworkId"))
+        .addResources(
+            Protos.Resource.newBuilder()
+                .setScalar(Protos.Value.Scalar.newBuilder().setValue(roleVal))
+                .setType(Protos.Value.Type.SCALAR)
+                .setName(name)
+                .setRole("test")
+                .build())
+        .addResources(
+            Protos.Resource.newBuilder()
+                .setScalar(Protos.Value.Scalar.newBuilder().setValue(defaultVal))
+                .setType(Protos.Value.Type.SCALAR)
+                .setName("cpus")
+                .build())
+        .build();
+    return offer;
+  }
+
+  private void checkResourceList(Iterable<Protos.Resource> resources, String name, Double roleVal, Double defaultVal) {
+    int i = 0;
+    Range defaultValueRange = Ranges.closed(defaultVal - epsilon, defaultVal + epsilon);
+    Range roleValueRange = Ranges.closed(roleVal - epsilon, roleVal + epsilon);
+
+    for (Protos.Resource resource: resources) {
+      if (resource.hasRole() && resource.getRole().equals("test")) {
+        double v = resource.getScalar().getValue();
+        assertTrue("Test Role  has " + v + " " + name + " should have " + roleVal, roleValueRange.contains(v));
+        i++;
+      } else {
+        double v = resource.getScalar().getValue();
+        assertTrue("Default Role has " + v + " " + name + " should have " + defaultVal , defaultValueRange.contains(v));
+        i++;
+      }
+    }
+    assertTrue("There should be at most 2 resources", i <= 2);
+  }
+
+  @Test
+  public void testGetScalarResourcesWithRole() {
+    TaskUtils taskUtils = new TaskUtils(cfgWithRole);
+    checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 3.0, 2.0), "cpus", 1.0, 0.0), "cpus", 1.0, 0.0);
+    checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 1.0, 1.0), "cpus", 0.0, 1.0);
+    checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 1.5, 2.0), "cpus", 2.0, 1.0), "cpus", 0.5, 1.5);
+    checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 1.5, 2.0), "cpus", 1.5, 2.0), "cpus", 0.0, 1.5);
+  }
+
+  @Test
+  public void testGetScalarResources() {
+    TaskUtils taskUtils = new TaskUtils(cfg);
+    checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 1.0, 0.0), "cpus", 0.0, 1.0);
+    checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 1.0, 1.0), "cpus", 0.0, 1.0);
+    checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 1.0, 1.0), "cpus", 0.0, 1.0);
+    checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 0.5, 1.5), "cpus", 0.0, 0.5);
+  }
+
+  @Test
+  public void testContainerInfo() {
+    TaskUtils taskUtils = new TaskUtils(cfgWithDocker);
+    Protos.ContainerInfo containerInfo = taskUtils.getContainerInfo();
+    assertTrue("The container should have a docker", containerInfo.hasDocker());
+    assertTrue("There should be two volumes", containerInfo.getVolumesCount() == 2);
+    assertTrue("The first volume should be read only", containerInfo.getVolumes(0).getMode().equals(Protos.Volume.Mode.RO));
+    assertTrue("The first volume should be read write", containerInfo.getVolumes(1).getMode().equals(Protos.Volume.Mode.RW));
+  }
+
+  @Test public void testDockerInfo() {
+    TaskUtils taskUtils = new TaskUtils(cfgWithDocker);
+    Protos.ContainerInfo containerInfo = taskUtils.getContainerInfo();
+    assertTrue("The container should have a docker", containerInfo.hasDocker());
+    assertTrue("There should be two volumes", containerInfo.getVolumesList().size() == 2);
+    assertTrue("There should be a docker image", containerInfo.getDocker().hasImage());
+    assertTrue("The docker image should be mesos/myraid", containerInfo.getDocker().getImage().equals("mesos/myriad"));
+    assertTrue("Should be using host networking", containerInfo.getDocker().getNetwork().equals(Protos.ContainerInfo.DockerInfo.Network.HOST));
+    assertTrue("There should be two parameters", containerInfo.getDocker().getParametersList().size() == 2);
+    assertTrue("Privledged mode should be false", containerInfo.getDocker().getPrivileged() == false);
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
new file mode 100644
index 0000000..221607a
--- /dev/null
+++ b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+mesosMaster: 10.0.2.15:5050
+checkpoint: false
+frameworkFailoverTimeout: 43200000
+frameworkName: MyriadTest
+frameworkRole: test
+frameworkUser: hduser     # User the Node Manager runs as, required if nodeManagerURI set, otherwise defaults to the user
+                          # running the resource manager.
+frameworkSuperUser: root  # To be depricated, currently permissions need set by a superuser due to Mesos-1790.  Must be
+                          # root or have passwordless sudo. Required if nodeManagerURI set, ignored otherwise.
+nativeLibrary: /usr/local/lib/libmesos.so
+zkServers: localhost:2181
+zkTimeout: 20000
+restApiPort: 8192
+profiles:
+  small:
+    cpu: 1
+    mem: 1100
+  medium:
+    cpu: 2
+    mem: 2048
+  large:
+    cpu: 4
+    mem: 4096
+rebalancer: false
+nodemanager:
+  jvmMaxMemoryMB: 1024
+  cpus: 0.2
+  cgroups: false
+executor:
+  jvmMaxMemoryMB: 256
+  path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar
+  #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid.
+  nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz
+  #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar
+yarnEnvironment:
+  YARN_HOME: /usr/local/hadoop
+  #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set
+  #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary
+services:
+ jobhistory:
+   jvmMaxMemoryMB: 1024
+   cpus: 1
+   ports:
+     myriad.mapreduce.jobhistory.admin.address: 0
+     myriad.mapreduce.jobhistory.address: 0
+     myriad.mapreduce.jobhistory.webapp.address: 0
+   envSettings: -Dcluster.name.prefix=/mycluster
+   taskName: jobhistory
+ timelineserver:
+   jvmMaxMemoryMB: 1024
+   cpus: 1
+   envSettings: -Dcluster.name.prefix=/mycluster2
+   taskName: timelineserver
+containerInfo:
+ type: DOCKER
+ dockerInfo:
+   image: mesos/myriad
+   parameters:
+     -
+       key: volume-driver
+       value: keywhiz
+     -
+       key: volume
+       value: all-my-secrets:/etc/secrets
+ volumes:
+   -
+     hostPath: /srv/data1/hadoop-yarn
+     containerPath: /data1/hadoop-yarn
+     mode: RO
+   -
+     hostPath: /srv/data2/hadoop-yarn
+     containerPath: /data2/hadoop-yarn
+     mode: RW
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/test/resources/myriad-config-test-default-with-framework-role.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default-with-framework-role.yml b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-framework-role.yml
new file mode 100644
index 0000000..ab7f137
--- /dev/null
+++ b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-framework-role.yml
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+mesosMaster: 10.0.2.15:5050
+checkpoint: false
+frameworkFailoverTimeout: 43200000
+frameworkName: MyriadTest
+frameworkRole: test
+frameworkUser: hduser     # User the Node Manager runs as, required if nodeManagerURI set, otherwise defaults to the user
+                          # running the resource manager.
+frameworkSuperUser: root  # To be depricated, currently permissions need set by a superuser due to Mesos-1790.  Must be
+                          # root or have passwordless sudo. Required if nodeManagerURI set, ignored otherwise.
+nativeLibrary: /usr/local/lib/libmesos.so
+zkServers: localhost:2181
+zkTimeout: 20000
+restApiPort: 8192
+profiles:
+  small:
+    cpu: 1
+    mem: 1100
+  medium:
+    cpu: 2
+    mem: 2048
+  large:
+    cpu: 4
+    mem: 4096
+rebalancer: false
+nodemanager:
+  jvmMaxMemoryMB: 1024
+  cpus: 0.2
+  cgroups: false
+executor:
+  jvmMaxMemoryMB: 256
+  path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar
+  #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid.
+  nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz
+  #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar
+yarnEnvironment:
+  YARN_HOME: /usr/local/hadoop
+  #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set
+  #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary
+services:
+   jobhistory:
+     jvmMaxMemoryMB: 1024
+     cpus: 1
+     ports:
+       myriad.mapreduce.jobhistory.admin.address: 0
+       myriad.mapreduce.jobhistory.address: 0
+       myriad.mapreduce.jobhistory.webapp.address: 0
+     envSettings: -Dcluster.name.prefix=/mycluster
+     taskName: jobhistory
+   timelineserver:
+     jvmMaxMemoryMB: 1024
+     cpus: 1
+     envSettings: -Dcluster.name.prefix=/mycluster2
+     taskName: timelineserver
\ No newline at end of file


Mime
View raw message