myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dar...@apache.org
Subject incubator-myriad git commit: Docker Gradle Integration
Date Thu, 31 Mar 2016 20:13:11 GMT
Repository: incubator-myriad
Updated Branches:
  refs/heads/master 4bce035ec -> e1db4d8b8


Docker Gradle Integration

Docker RM now is built via Gradle.

Pull Request:
  Closes #55

Author:
  Brandon Gulla brandon@brandongulla.com


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/e1db4d8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/e1db4d8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/e1db4d8b

Branch: refs/heads/master
Commit: e1db4d8b8cdb4fd1994be9097ab6e439e8f28cc0
Parents: 4bce035
Author: Brandon Gulla <brandon@brandongulla.com>
Authored: Wed Nov 18 12:51:49 2015 -0500
Committer: darinj <darinj.work@gmail.com>
Committed: Wed Mar 30 23:51:13 2016 -0400

----------------------------------------------------------------------
 build.gradle                      | 15 ++++++++++
 docker/Dockerfile                 |  2 +-
 docker/README.md                  | 20 +++++--------
 docker/ResourceManager.dockerfile | 55 ----------------------------------
 docker/build.gradle               | 50 +++++++++++++++++++++++++++++++
 docker/myriad-bin/create-user.sh  |  2 +-
 settings.gradle                   |  1 +
 7 files changed, 75 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 7e57403..918f84c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -21,6 +21,21 @@ allprojects {
     apply plugin: 'eclipse'
 }
 
+buildscript {
+    repositories {
+        jcenter()
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath 'com.bmuschko:gradle-docker-plugin:2.6.1'
+    }
+}
+
+repositories {
+    mavenCentral()
+}
+
 idea {
     project {
         languageLevel = '1.7'

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 18d4084..52ad7f3 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -43,7 +43,7 @@ ADD myriad-bin/install-yarn.sh /install-yarn.sh
 RUN sh /install-yarn.sh
 
 # Copy over myriad libraries
-ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
+ADD ./libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
 
 
 # Initialize hadoop confs with env vars

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/README.md
----------------------------------------------------------------------
diff --git a/docker/README.md b/docker/README.md
index 8f00d64..7ce77ac 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -1,12 +1,14 @@
 # Using Myriad with Docker #
+Docker is the easiest way to from 0 to Yarn on Mesos within minutes. 
 
 ## Building the Resource Manager Docker
-
-`./build-myriad.sh` will run the gradle scripts from the root myriad folder and compile all
necessary libraries.
+Run the following command:
+```./gradlew buildRMDocker```
+This will build the ResourceManager from src and save the image as *mesos/myriad*.
 
 #Configuration Guide#
 
-In order for the ResourceManager to operate correctly, you will need to provide 2 configuration
files:
+In order for the ResourceManager to operate correctly, you will need to provide 2 configuration
files. These files will need to mounted from a directory into */myriad-conf* within the docker
container.
 
 * [myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml)
 * modified [yarn-site.xml](https://github.com/mesos/myriad/blob/phase1/docs/myriad-dev.md)
@@ -17,19 +19,11 @@ In order for the ResourceManager to operate correctly, you will need to
provide
 ```bash
 docker run --net=host --name='myriad-resourcemanager' -t \
   -v /path/to/configs:/myriad-conf \
-  -e HADOOP_NAMENODE="10.100.3.237:9000" \
-  mesos/myriad-resourcemanager
+  mesos/myriad
   ```
 
 #Environment Variables#
-
-* *HADOOP_NAMENODE* : *Required*
-* *ALTERNATE_HADOOP_URL* : Optional - Allows user to override the hadoop distribution used
by Myriad.
-
-
-If you already had a working Vagrant instance, you will need to run `vagrant reload` in order
to allow zookeeper and hdfs port-forwarding.
-
-Still having problems connecting to Zookeeper? Be sure that your zkServer and MesosMaster
values are correct in the [myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml)
file.
+* *ALTERNATE_HADOOP_URL* : Optional - Allows user to override the hadoop distribution used
by Myriad. This will download the *.tar.gz file to be used as the hadoop distribution of choice
for Myriad. 
 
 ---
 <sub>

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/ResourceManager.dockerfile
----------------------------------------------------------------------
diff --git a/docker/ResourceManager.dockerfile b/docker/ResourceManager.dockerfile
deleted file mode 100644
index c4448d7..0000000
--- a/docker/ResourceManager.dockerfile
+++ /dev/null
@@ -1,55 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Myriad - YARN Resource Manager
-#
-# VERSION 0.1.0
-
-FROM debian
-MAINTAINER Apache Myriad dev@myriad.incubator.apache.org
-
-ENV HADOOP_USER="yarn"
-ENV HADOOP_VER="2.7.0"
-
-# Setup mesosphere repositories
-RUN apt-get -y update
-RUN apt-get install -y openjdk-7-jre-headless wget lsb-release
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF
-RUN DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') CODENAME=$(lsb_release -cs) &&
echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | tee /etc/apt/sources.list.d/mesosphere.list
-RUN apt-get -y update
-
-# Create the yarn user
-ADD myriad-bin/create-user.sh /create-user.sh
-RUN sh /create-user.sh
-
-# Install Mesos
-RUN apt-get install -y mesos curl tar
-
-# Run local YARN install
-ADD myriad-bin/install-yarn.sh /install-yarn.sh
-RUN sh /install-yarn.sh
-
-# Copy over myriad libraries
-ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
-
-# Initialize hadoop confs with env vars
-ADD myriad-bin/run-myriad.sh /run-myriad.sh
-RUN mkdir /myriad-conf/
-
-# Run the YARN resource manager
-USER yarn
-CMD sh /run-myriad.sh 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/build.gradle
----------------------------------------------------------------------
diff --git a/docker/build.gradle b/docker/build.gradle
new file mode 100644
index 0000000..4f35494
--- /dev/null
+++ b/docker/build.gradle
@@ -0,0 +1,50 @@
+task wrapper(type: Wrapper) {
+    gradleVersion = '2.9'
+}
+
+buildscript {
+    repositories {
+        jcenter()
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath 'com.bmuschko:gradle-docker-plugin:2.6.1'
+    }
+}
+
+repositories {
+    mavenCentral()
+}
+
+apply plugin: 'com.bmuschko.docker-remote-api'
+
+import com.bmuschko.gradle.docker.tasks.image.Dockerfile
+import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+
+task copySchedulerJars(type:Copy){
+    def subdir = new File(project.buildDir, "libs")
+    if( !subdir.exists() ) { subdir.mkdirs() }
+    from '../myriad-scheduler/build/libs'
+    into 'libs'
+}
+
+task copyMyriadJars(dependsOn: ":myriad-scheduler:build") {
+    dependsOn(":myriad-executor:build")
+    dependsOn(copySchedulerJars)
+}
+
+task buildRMDocker(type: DockerBuildImage) {
+    dependsOn(copyMyriadJars)
+    if (System.env.DOCKER_HOST) {
+        url = "$System.env.DOCKER_HOST".replace("tcp","https")
+        if (System.env.DOCKER_CERT_PATH) {
+            certPath = new File(System.env.DOCKER_CERT_PATH)
+        }
+    }
+    else {
+        url = 'unix:///var/run/docker.sock'
+    }  
+    inputDir = file('.')
+    tag = 'mesos/myriad'
+}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/myriad-bin/create-user.sh
----------------------------------------------------------------------
diff --git a/docker/myriad-bin/create-user.sh b/docker/myriad-bin/create-user.sh
index 9d6b2c2..c6c1bf6 100755
--- a/docker/myriad-bin/create-user.sh
+++ b/docker/myriad-bin/create-user.sh
@@ -27,7 +27,7 @@ export GROUP_UID=${GROUP_GID:='112'}
 # Add hduser user
 echo "Creating $HADOOP_USER user.."
 groupadd $HADOOP_GROUP -g ${GROUP_UID}
-useradd $HADOOP_USER -g $HADOOP_GROUP -u ${USER_UID} -s /bin/bash
+useradd $HADOOP_USER -g $HADOOP_GROUP -u ${USER_UID} -s /bin/bash -d /home/${HADOOP_USER}
 mkdir /home/${HADOOP_USER}
 chown -R $HADOOP_USER:$HADOOP_GROUP /home/${HADOOP_USER}
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/settings.gradle
----------------------------------------------------------------------
diff --git a/settings.gradle b/settings.gradle
index 9af746c..8569bf5 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -29,3 +29,4 @@ rootProject.name = 'myriad'
 include "myriad-commons"
 include "myriad-scheduler"
 include "myriad-executor"
+include "docker"


Mime
View raw message