myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smare...@apache.org
Subject incubator-myriad git commit: Updated Vagrantfile to work with the supported Mesos and Hadoop versi…
Date Thu, 12 Nov 2015 01:11:14 GMT
Repository: incubator-myriad
Updated Branches:
  refs/heads/master 9f0fa15bf -> fed956b86


Updated Vagrantfile to work with the supported Mesos and Hadoop versi…

…ons.

Broke provisioning into separate scripts in the ./vagrant directory.
Updated vagrant documentation
Added .npmrc that eliminates node symbolic links so build will work under Vagrant.

This closes: #43
Review: https://github.com/apache/incubator-myriad/pull/43


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/fed956b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/fed956b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/fed956b8

Branch: refs/heads/master
Commit: fed956b869bccb5b807221b5ed41e08d57926fe6
Parents: 9f0fa15
Author: klucar <klucar@gmail.com>
Authored: Wed Nov 11 16:58:44 2015 -0800
Committer: smarella <smarella@maprtech.com>
Committed: Wed Nov 11 16:58:44 2015 -0800

----------------------------------------------------------------------
 Vagrantfile                                     | 123 ++++++++++---------
 docs/vagrant.md                                 |  51 ++------
 .../src/main/resources/webapp/.npmrc            |   1 +
 setup-yarn-1.sh                                 |  18 ---
 setup-yarn-2.sh                                 |  64 ----------
 shutdown.sh                                     |  10 --
 vagrant/format_namenode.sh                      |  23 ++++
 vagrant/install_default_jdk.sh                  |  32 +++++
 vagrant/install_docker.sh                       |  25 ++++
 vagrant/install_hadoop.sh                       | 115 +++++++++++++++++
 vagrant/install_mesos.sh                        |  43 +++++++
 vagrant/shutdown.sh                             |  30 +++++
 vagrant/start_datanode.sh                       |  24 ++++
 vagrant/start_historyserver.sh                  |  23 ++++
 vagrant/start_mesos_master.sh                   |  34 +++++
 vagrant/start_mesos_slave.sh                    |  38 ++++++
 vagrant/start_namenode.sh                       |  27 ++++
 17 files changed, 488 insertions(+), 193 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/Vagrantfile
----------------------------------------------------------------------
diff --git a/Vagrantfile b/Vagrantfile
index 88039f8..7102fb9 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -1,80 +1,81 @@
 # -*- mode: ruby -*-
 # # vi: set ft=ruby :
 
-VAGRANTFILE_API_VERSION = "2"
-
-$provision_script = <<SCRIPT
-
-PREFIX="PROVISIONER:"
-
-set -e
-
-echo "${PREFIX} Installing pre-reqs..."
-
-# For installing Java 8
-add-apt-repository ppa:webupd8team/java
-
-# For Mesos
-apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF
-DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
-CODENAME=$(lsb_release -cs)
-echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
 
-apt-get -y update
-echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-apt-get -y install oracle-java8-installer
-apt-get -y install oracle-java8-set-default
-apt-get -y install libcurl3
-apt-get -y install zookeeperd
-apt-get -y install aria2
+VAGRANTFILE_API_VERSION = "2"
 
 MESOS_VERSION="0.21.1"
-echo "${PREFIX}Installing mesos version: ${MESOS_VERSION}..."
-apt-get -y install mesos
-echo "Done"
-
-ln -s /usr/lib/jvm/java-8-oracle/jre/lib/amd64/server/libjvm.so /usr/lib/libjvm.so
-
-echo "${PREFIX}Starting mesos master"
-start mesos-master
-
-echo "${PREFIX}Starting mesos slave"
-start mesos-slave
-
-echo "${PREFIX}Successfully provisioned machine for Myriad development"
-
-SCRIPT
+HADOOP_VERSION="2.7.0"
+#PRIVATE_IP="10.141.141.20"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box = "ubuntu/trusty64"
   config.vm.box_url = "https://vagrantcloud.com/ubuntu/boxes/trusty64"
+  #config.vm.hostname = "master"
+  #config.vm.network :private_network, ip: "#{PRIVATE_IP}"
 
-  # Consistent IP for demos and tutorials (10.141.141.10 is mesosphere playa address)
-  config.vm.network :private_network, ip: "10.141.141.20"
+  # Configure VM resources
+  config.vm.provider :virtualbox do |vb|
+    vb.name = "myriad-dev"
+    vb.customize ["modifyvm", :id, "--memory", "4096"]
+    vb.customize ["modifyvm", :id, "--cpus", "2"]
+    #vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+    #vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
+  end
+
+  if Vagrant.has_plugin?("vagrant-cachier")
+    # Configure cached packages to be shared between instances of the same base box.
+    config.cache.scope = :box
+  end
 
   # Forward mesos master and slave ports
-  config.vm.network "forwarded_port", guest: 5005, host: 5005
-  config.vm.network "forwarded_port", guest: 5050, host: 5050
-  config.vm.network "forwarded_port", guest: 5051, host: 5051
+  #config.vm.network "forwarded_port", guest: 5005, host: 5005
+  #config.vm.network "forwarded_port", guest: 5050, host: 5050
+  #config.vm.network "forwarded_port", guest: 5051, host: 5051
 
   # Forward myriad web and admin ports
-  config.vm.network "forwarded_port", guest: 8080, host: 8080
-  config.vm.network "forwarded_port", guest: 8081, host: 8081
+  #config.vm.network "forwarded_port", guest: 8080, host: 8080
+  #config.vm.network "forwarded_port", guest: 8081, host: 8081
 
   # Forward YARN/Hadoop ports
-  config.vm.network "forwarded_port", guest: 50070, host: 50070
-  config.vm.network "forwarded_port", guest: 50075, host: 50075
-  config.vm.network "forwarded_port", guest: 8088, host: 8088
-  config.vm.network "forwarded_port", guest: 8042, host: 8042
-  config.vm.network "forwarded_port", guest: 19888, host: 19888
-  config.vm.network "forwarded_port", guest: 8192, host: 8192
-  config.vm.network "forwarded_port", guest: 2181, host: 2181
+  #config.vm.network "forwarded_port", guest: 50070, host: 50070
+  #config.vm.network "forwarded_port", guest: 50075, host: 50075
+  #config.vm.network "forwarded_port", guest: 8088, host: 8088
+  #config.vm.network "forwarded_port", guest: 8042, host: 8042
+  #config.vm.network "forwarded_port", guest: 19888, host: 19888
+  #config.vm.network "forwarded_port", guest: 8192, host: 8192
+  #config.vm.network "forwarded_port", guest: 2181, host: 2181
+
+  # install software
+  config.vm.provision "shell", path: "vagrant/install_default_jdk.sh"
+  config.vm.provision "shell", path: "vagrant/install_mesos.sh", args: ["#{MESOS_VERSION}"]
+  config.vm.provision "shell", path: "vagrant/install_docker.sh"
+  config.vm.provision "shell", path: "vagrant/install_hadoop.sh", args: ["#{HADOOP_VERSION}"]
+
+  # start things up
+  config.vm.provision "shell", path: "vagrant/start_mesos_master.sh"
+  config.vm.provision "shell", path: "vagrant/start_mesos_slave.sh"
+  config.vm.provision "shell", path: "vagrant/format_namenode.sh"
+  config.vm.provision "shell", path: "vagrant/start_namenode.sh"
+  config.vm.provision "shell", path: "vagrant/start_datanode.sh"
+  config.vm.provision "shell", path: "vagrant/start_historyserver.sh"
 
-  config.vm.provision "shell", inline: $provision_script
-
-  # Configure VM resources
-  config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--memory", "4096"]
-    vb.customize ["modifyvm", :id, "--cpus", "2"]
-  end
 end

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/docs/vagrant.md
----------------------------------------------------------------------
diff --git a/docs/vagrant.md b/docs/vagrant.md
index 2762ef9..0c71ea7 100644
--- a/docs/vagrant.md
+++ b/docs/vagrant.md
@@ -15,7 +15,14 @@ To start the cluster run following:
 vagrant up
 ```
 
-At this point the VM will have a single node mesos cluster running.
+At this point the VM will have a single node Mesos cluster running and a single node
+HDFS cluster running. You can modify the Mesos and Hadoop versions by editing the
+corresponding variables in the `Vagrantfile`.
+
+Check that Mesos and Hadoop are running with a browser:
+
+[Hadoop/HDFS namenode](http://10.141.141.20:50070)
+[Mesos Master](http://10.141.141.20:5050/)
 
 To ssh in the cluster, run following:
 
@@ -23,51 +30,15 @@ To ssh in the cluster, run following:
 vagrant ssh
 ```
 
-The password for vagrant user is **vagrant'**
-
-## Setting Up YARN/Hadoop ##
-
-To setup YARN/Hadoop inside VM, run following YARN setup shell files:
-
-1 Run the first YARN setup shell command from the vagrant directory to create a user hduser
in group hadoop. Be sure to remember the password that you provide for this user.
-```
-cd /vagrant
-./setup-yarn-1.sh
-```
-2.  Run the second YARN setup shell command as sudo.
+The password for vagrant user is **vagrant** if prompted
+You can switch to the hadoop user when needed to launch hadoop processes
 
 ```
 sudo su - hduser
-cd /vagrant
-./setup-yarn-2.sh
-```
-
-The following processes should be running:
-
-```
-9844 Jps
-6709 NameNode
-6393 JobHistoryServer
-6874 DataNode
 ```
 
-**NOTE:** The process IDS are different.
-
-## Installing Myriad ##
-
-Installing Myriad involves downloading the Myriad binaries from GitHub. The Myriad location
is currently at [https://github.com/mesos/myriad](https://github.com/mesos/myriad).
-
-To install Myriad:
-
-1. Create a new myriad directory.
-2. Clone the Myriad GitHub.
-3. Pull the contents to the new directory.
-
-
 ## Configuring Myriad ##
 
-To configure Myriad:
-
 ### Step 1: Build Myriad ###
 
 To build Myriad Scheduler inside VM, run the gradlew build:
@@ -214,7 +185,7 @@ To shut down, from the vagrant ssh console, run the following:
 
 ```
 yarn-daemon.sh stop resourcemanager
-./shutdown.sh
+sh ./vagrant/shutdown.sh
 exit
 exit
 vagrant halt

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/myriad-scheduler/src/main/resources/webapp/.npmrc
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/resources/webapp/.npmrc b/myriad-scheduler/src/main/resources/webapp/.npmrc
new file mode 100644
index 0000000..d86ebdf
--- /dev/null
+++ b/myriad-scheduler/src/main/resources/webapp/.npmrc
@@ -0,0 +1 @@
+bin-links=false

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/setup-yarn-1.sh
----------------------------------------------------------------------
diff --git a/setup-yarn-1.sh b/setup-yarn-1.sh
deleted file mode 100755
index 314d5cc..0000000
--- a/setup-yarn-1.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-set -e
-
-HADOOP_VER=2.7.0
-
-cd ~/
-
-sudo apt-get update
-
-sudo apt-get install openssh-server
-
-# Add hduser user
-sudo addgroup hadoop
-sudo adduser --ingroup hadoop hduser
-sudo adduser hduser sudo
-
-# Setup password-less auth
-sudo -u hduser ssh-keygen -t rsa -P ''
-sudo sh -c 'cat /home/hduser/.ssh/id_rsa.pub >> /home/hduser/.ssh/authorized_keys'

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/setup-yarn-2.sh
----------------------------------------------------------------------
diff --git a/setup-yarn-2.sh b/setup-yarn-2.sh
deleted file mode 100755
index 074f531..0000000
--- a/setup-yarn-2.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-set -e
-
-HADOOP_VER=2.7.0
-
-# Download Hadoop
-cd ~
-if [ ! -f hadoop-${HADOOP_VER}.tar.gz ]; then
-	wget http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz
-fi
-
-sudo tar vxzf hadoop-${HADOOP_VER}.tar.gz -C /usr/local
-cd /usr/local
-sudo mv hadoop-${HADOOP_VER} hadoop
-sudo chown -R hduser:hadoop hadoop
-
-# Init bashrc with hadoop env variables
-sudo sh -c 'echo export JAVA_HOME=/usr/lib/jvm/java-8-oracle >> /home/hduser/.bashrc'
-sudo sh -c 'echo export HADOOP_INSTALL=/usr/local/hadoop >> /home/hduser/.bashrc'
-sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/bin >> /home/hduser/.bashrc'
-sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/sbin >> /home/hduser/.bashrc'
-sudo sh -c 'echo export HADOOP_MAPRED_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc'
-sudo sh -c 'echo export HADOOP_COMMON_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc'
-sudo sh -c 'echo export HADOOP_HDFS_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc'
-sudo sh -c 'echo export YARN_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc'
-sudo sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_INSTALL\}/lib/native >>
/home/hduser/.bashrc'
-sudo sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_INSTALL/lib\" >>
/home/hduser/.bashrc'
-
-# Modify JAVA_HOME in hadoop-env
-cd /usr/local/hadoop/etc/hadoop
-sudo -u hduser sed -i.bak s=\${JAVA_HOME}=/usr/lib/jvm/java-8-oracle/=g hadoop-env.sh
-pwd
-
-/usr/local/hadoop/bin/hadoop version
-
-# Update configuration
-sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>fs\.default\.name\</name>\<value>hdfs://localhost:9000\</value>\</property>=g'
core-site.xml
-sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>yarn\.nodemanager\.aux-services</name>\<value>mapreduce_shuffle</value>\</property>\<property>\<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>\<value>org\.apache\.hadoop\.mapred\.ShuffleHandler</value>\</property>=g'
yarn-site.xml
-
-sudo -u hduser cp mapred-site.xml.template mapred-site.xml
-sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>mapreduce\.framework\.name</name>\<value>yarn</value>\</property>=g'
mapred-site.xml
-
-cd ~
-sudo -u hduser sh -c 'mkdir -p mydata/hdfs/namenode'
-sudo -u hduser sh -c 'mkdir -p mydata/hdfs/datanode'
-sudo chown -R hduser:hadoop /home/hduser/mydata
-
-cd /usr/local/hadoop/etc/hadoop
-sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>dfs\.replication</name>\<value>1\</value>\</property>\<property>\<name>dfs\.namenode\.name\.dir</name>\<value>file:/home/hduser/mydata/hdfs/namenode</value>\</property>\<property>\<name>dfs\.datanode\.data\.dir</name>\<value>file:/home/hduser/mydata/hdfs/datanode</value>\</property>=g'
hdfs-site.xml
-
-
-# Format NameNode
-sudo -u hduser sh -c '/usr/local/hadoop/bin/hdfs namenode -format'
-
-# Start NameNode
-sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemon.sh start namenode'
-
-# Start DataNode
-sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemons.sh start datanode'
-
-# Start HistoryServer
-sudo -u hduser sh -c '/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver'
-
-# Check status
-sudo -u hduser -u hduser jps

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/shutdown.sh
----------------------------------------------------------------------
diff --git a/shutdown.sh b/shutdown.sh
deleted file mode 100755
index b0bfab5..0000000
--- a/shutdown.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-set -e
-
-# Stop HistoryServer
-sudo -u hduser sh -c '/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh stop historyserver'
-
-# Stop DataNode
-sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemons.sh stop datanode'
-
-# Stop NameNode
-sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemon.sh stop namenode'

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/format_namenode.sh
----------------------------------------------------------------------
diff --git a/vagrant/format_namenode.sh b/vagrant/format_namenode.sh
new file mode 100644
index 0000000..e944a41
--- /dev/null
+++ b/vagrant/format_namenode.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -e
+
+# Format NameNode
+sudo -u hduser sh -c 'yes Y | /usr/local/hadoop/bin/hdfs namenode -format'

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/install_default_jdk.sh
----------------------------------------------------------------------
diff --git a/vagrant/install_default_jdk.sh b/vagrant/install_default_jdk.sh
new file mode 100644
index 0000000..0a85c3c
--- /dev/null
+++ b/vagrant/install_default_jdk.sh
@@ -0,0 +1,32 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+PREFIX="JAVA Default JDK Provisioner:"
+set -e
+
+# For installing Java 8
+apt-get -y update
+apt-get -y install default-jdk
+
+if $(test -e /usr/lib/libjvm.so); then
+  rm /usr/lib/libjvm.so
+fi
+
+ln -s /usr/lib/jvm/default-java/jre/lib/amd64/server/libjvm.so /usr/lib/libjvm.so

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/install_docker.sh
----------------------------------------------------------------------
diff --git a/vagrant/install_docker.sh b/vagrant/install_docker.sh
new file mode 100644
index 0000000..a0a214f
--- /dev/null
+++ b/vagrant/install_docker.sh
@@ -0,0 +1,25 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -e
+
+#Install docker
+echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list
+apt-get update
+apt-get -y install docker.io

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/install_hadoop.sh
----------------------------------------------------------------------
diff --git a/vagrant/install_hadoop.sh b/vagrant/install_hadoop.sh
new file mode 100644
index 0000000..fe5c2a0
--- /dev/null
+++ b/vagrant/install_hadoop.sh
@@ -0,0 +1,115 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# $1 is HADOOP_VERSION
+set -e
+
+apt-get update
+
+apt-get install -y openssh-server
+apt-get install -y tar
+apt-get install -y gzip
+
+# Add hduser user and hadoop group
+
+if [ `/bin/egrep  -i "^hadoop:" /etc/group` ]; then
+   echo "Group hadoop already exists"
+else
+  echo "Adding hadoop group"
+  addgroup hadoop
+fi
+
+
+if [ `/bin/egrep  -i "^hduser:" /etc/passwd` ]; then
+  echo "User hduser already exists"
+else
+  echo "creating hduser in group hadoop"
+  adduser --ingroup hadoop --disabled-password --gecos "" --home /home/hduser hduser
+  adduser hduser sudo
+fi
+
+# Setup password-less auth
+sudo -u hduser sh -c "mkdir -p /home/hduser/.ssh"
+sudo -u hduser sh -c "chmod 700 /home/hduser/.ssh"
+sudo -u hduser sh -c "yes | ssh-keygen -t rsa -N '' -f /home/hduser/.ssh/id_rsa"
+sudo -u hduser sh -c 'cat /home/hduser/.ssh/id_rsa.pub >> /home/hduser/.ssh/authorized_keys'
+#sudo -u hduser sh -c "ssh-keyscan -H $1 >> /home/hduser/.ssh/known_hosts"
+sudo -u hduser sh -c "ssh-keyscan -H localhost >> /home/hduser/.ssh/known_hosts"
+#sudo -u hduser sh -c "ssh-keyscan -H $2 >> /home/hduser/.ssh/known_hosts"
+
+# Download Hadoop
+HADOOP_VER="$1"
+cd ~
+if [ ! -f /tmp/hadoop-${HADOOP_VER}.tar.gz ]; then
+	wget http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz
-O /tmp/hadoop-${HADOOP_VER}.tar.gz
+fi
+
+sudo tar ixzf /tmp/hadoop-${HADOOP_VER}.tar.gz -C /usr/local
+cd /usr/local
+rm -rf hadoop
+sudo mv -f hadoop-${HADOOP_VER} hadoop
+sudo chown -R hduser:hadoop hadoop
+
+# Init bashrc with hadoop env variables
+sudo sh -c 'echo export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 >> /home/hduser/.bashrc'
+sudo sh -c 'echo export HADOOP_INSTALL=/usr/local/hadoop >> /home/hduser/.bashrc'
+sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/bin >> /home/hduser/.bashrc'
+sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/sbin >> /home/hduser/.bashrc'
+sudo sh -c 'echo export HADOOP_MAPRED_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc'
+sudo sh -c 'echo export HADOOP_COMMON_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc'
+sudo sh -c 'echo export HADOOP_HDFS_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc'
+sudo sh -c 'echo export YARN_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc'
+sudo sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_INSTALL\}/lib/native >>
/home/hduser/.bashrc'
+sudo sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_INSTALL/lib\" >>
/home/hduser/.bashrc'
+# hit the vagrant user with the same thing
+sudo sh -c 'echo export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 >> /home/vagrant/.bashrc'
+sudo sh -c 'echo export HADOOP_INSTALL=/usr/local/hadoop >> /home/vagrant/.bashrc'
+sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/bin >> /home/vagrant/.bashrc'
+sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/sbin >> /home/vagrant/.bashrc'
+sudo sh -c 'echo export HADOOP_MAPRED_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc'
+sudo sh -c 'echo export HADOOP_COMMON_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc'
+sudo sh -c 'echo export HADOOP_HDFS_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc'
+sudo sh -c 'echo export YARN_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc'
+sudo sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_INSTALL\}/lib/native >>
/home/vagrant/.bashrc'
+sudo sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_INSTALL/lib\" >>
/home/vagrant/.bashrc'
+
+
+# Modify JAVA_HOME in hadoop-env
+cd /usr/local/hadoop/etc/hadoop
+sudo -u hduser sed -i.bak s=\${JAVA_HOME}=//usr/lib/jvm/java-7-openjdk-amd64/=g hadoop-env.sh
+pwd
+
+/usr/local/hadoop/bin/hadoop version
+
+# Update configuration
+sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>fs\.default\.name\</name>\<value>hdfs://localhost:9000\</value>\</property>=g'
core-site.xml
+#sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>fs\.default\.name\</name>\<value>hdfs://'"$1"':9000\</value>\</property>=g'
core-site.xml
+sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>yarn\.nodemanager\.aux-services</name>\<value>mapreduce_shuffle</value>\</property>\<property>\<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>\<value>org\.apache\.hadoop\.mapred\.ShuffleHandler</value>\</property>=g'
yarn-site.xml
+
+sudo -u hduser cp mapred-site.xml.template mapred-site.xml
+sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>mapreduce\.framework\.name</name>\<value>yarn</value>\</property>=g'
mapred-site.xml
+
+cd ~
+sudo -u hduser sh -c 'mkdir -p ~hduser/mydata/hdfs/namenode'
+sudo -u hduser sh -c 'mkdir -p ~hduser/mydata/hdfs/datanode'
+sudo chown -R hduser:hadoop ~hduser/mydata
+
+cd /usr/local/hadoop/etc/hadoop
+sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>dfs\.replication</name>\<value>1\</value>\</property>\<property>\<name>dfs\.namenode\.name\.dir</name>\<value>file:/home/hduser/mydata/hdfs/namenode</value>\</property>\<property>\<name>dfs\.datanode\.data\.dir</name>\<value>file:/home/hduser/mydata/hdfs/datanode</value>\</property>\<property>\<name>dfs\.namenode\.datanode\.registration\.ip-hostname-check</name>\<value>false</value>\</property>=g'
hdfs-site.xml

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/install_mesos.sh
----------------------------------------------------------------------
diff --git a/vagrant/install_mesos.sh b/vagrant/install_mesos.sh
new file mode 100644
index 0000000..28292bf
--- /dev/null
+++ b/vagrant/install_mesos.sh
@@ -0,0 +1,43 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# $1 = mesos version number to install (e.g. 0.21.1)
+
+PREFIX="Mesos Provisioner: "
+set -e
+
+echo "${PREFIX} Installing pre-reqs..."
+# For Mesos
+apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF
+DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+CODENAME=$(lsb_release -cs)
+echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list
+apt-get -y update
+
+apt-get -y install libcurl3
+apt-get -y install zookeeperd
+apt-get -y install aria2
+apt-get -y install ssh
+apt-get -y install rsync
+
+
+export MESOS_VERSION=$1
+echo "${PREFIX}Installing mesos version: ${MESOS_VERSION}..."
+apt-get -y install mesos

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/shutdown.sh
----------------------------------------------------------------------
diff --git a/vagrant/shutdown.sh b/vagrant/shutdown.sh
new file mode 100644
index 0000000..2012b84
--- /dev/null
+++ b/vagrant/shutdown.sh
@@ -0,0 +1,30 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+# Stop HistoryServer
+sudo -u hduser sh -c '/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh stop historyserver'
+
+# Stop DataNode
+sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemons.sh stop datanode'
+
+# Stop NameNode
+sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemon.sh stop namenode'

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_datanode.sh
----------------------------------------------------------------------
diff --git a/vagrant/start_datanode.sh b/vagrant/start_datanode.sh
new file mode 100644
index 0000000..e391187
--- /dev/null
+++ b/vagrant/start_datanode.sh
@@ -0,0 +1,24 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+# Start DataNode
+sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemons.sh start datanode'

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_historyserver.sh
----------------------------------------------------------------------
diff --git a/vagrant/start_historyserver.sh b/vagrant/start_historyserver.sh
new file mode 100644
index 0000000..88adfca
--- /dev/null
+++ b/vagrant/start_historyserver.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+sudo -u hduser sh -c '/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver'

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_mesos_master.sh
----------------------------------------------------------------------
diff --git a/vagrant/start_mesos_master.sh b/vagrant/start_mesos_master.sh
new file mode 100644
index 0000000..ebe9e66
--- /dev/null
+++ b/vagrant/start_mesos_master.sh
@@ -0,0 +1,34 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo "zk://127.0.0.1:2181/mesos" > /etc/mesos/zk
+#echo "$@ master" >> /etc/hosts
+#echo "192.168.50.102 slave" >> /etc/hosts
+echo 127.0.0.1 | sudo tee /etc/mesos-master/ip
+echo localhost | sudo tee /etc/mesos-master/hostname
+
+echo "export HADOOP_HOME=/usr/local/hadoop" >> /root/.bashrc
+echo "export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /root/.bashrc
+echo "export PATH=$PATH:$HADOOP_HOME/bin:$JAVA_HOME/bin" >> /root/.bashrc
+
+# keep mesos slave from starting here
+echo manual | sudo tee /etc/init/mesos-slave.override
+
+sudo start mesos-master

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_mesos_slave.sh
----------------------------------------------------------------------
diff --git a/vagrant/start_mesos_slave.sh b/vagrant/start_mesos_slave.sh
new file mode 100644
index 0000000..eeca9e1
--- /dev/null
+++ b/vagrant/start_mesos_slave.sh
@@ -0,0 +1,38 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+echo "127.0.0.1" > /etc/mesos-slave/ip
+echo "cgroups/cpu,cgroups/mem" > /etc/mesos-slave/isolation
+echo "mesos" > /etc/mesos-slave/containerizers
+echo "/usr/local/hadoop" > /etc/mesos-slave/hadoop_home
+
+echo "export HADOOP_HOME=/usr/local/hadoop" >> /root/.bashrc
+echo "export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /root/.bashrc
+
+echo "zk://127.0.0.1:2181/mesos" | sudo tee /etc/mesos/zk
+echo "localhost" | sudo tee /etc/mesos-slave/hostname
+echo "cpus:2;mem:2048" | sudo tee /etc/mesos-slave/resources
+echo manual | sudo tee /etc/init/mesos-master.override
+echo manual | sudo tee /etc/init/zookeeper.override
+
+
+start mesos-slave

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_namenode.sh
----------------------------------------------------------------------
diff --git a/vagrant/start_namenode.sh b/vagrant/start_namenode.sh
new file mode 100644
index 0000000..d82b260
--- /dev/null
+++ b/vagrant/start_namenode.sh
@@ -0,0 +1,27 @@
+#!/bin/bash -v
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+# Start NameNode
+sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemon.sh start namenode'
+
+# Open up the HDFS permissions
+sudo -u hduser sh -c "/usr/local/hadoop/bin/hadoop fs -chmod 777 /"


Mime
View raw message