mesos-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject svn commit: r1132176 - in /incubator/mesos/trunk/ec2: ./ deploy.centos64/ deploy.centos64/root/ deploy.centos64/root/ephemeral-hdfs/ deploy.centos64/root/ephemeral-hdfs/conf/ deploy.centos64/root/mesos-ec2/ deploy.centos64/root/mesos-ec2/hadoop-framewo...
Date Sun, 05 Jun 2011 08:54:52 GMT
Author: benh
Date: Sun Jun  5 08:54:50 2011
New Revision: 1132176

URL: http://svn.apache.org/viewvc?rev=1132176&view=rev
Log:
Added support for CentOS AMIs with Hypertable and both ephemeral and
persistent HDFS instances.

Added:
    incubator/mesos/trunk/ec2/deploy.centos64/
    incubator/mesos/trunk/ec2/deploy.centos64/root/
    incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/
    incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/
    incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/core-site.xml
    incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh
    incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hdfs-site.xml
    incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/mapred-site.xml
    incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/masters
    incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/slaves
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/approve-master-key   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/cluster-url
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/copy-dir   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/core-site.xml
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/haproxy+apache/
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/haproxy+apache/haproxy.config.template
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/Capfile
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/hypertable.cfg
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/masters
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/mesos-daemon   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/redeploy-mesos   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-slave
      - copied, changed from r1132175, incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-torque   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/slaves
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/ssh-no-keychecking   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-hypertable   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-hypertable   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-mesos   (with props)
    incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/zoo
    incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/
    incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/
    incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/core-site.xml
    incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hadoop-env.sh
    incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hdfs-site.xml
    incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/mapred-site.xml
    incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/masters
    incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/slaves
Modified:
    incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave
    incubator/mesos/trunk/ec2/mesos_ec2.py

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/core-site.xml?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/core-site.xml (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/core-site.xml Sun Jun  5 08:54:50 2011
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>/mnt/ephemeral-hdfs</value>
+  </property>
+
+  <property>
+    <name>fs.default.name</name>
+    <value>hdfs://{{active_master}}:9000</value>
+  </property>
+
+  <property>
+    <name>io.file.buffer.size</name>
+    <value>65536</value>
+  </property>
+
+</configuration>

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh Sun Jun  5 08:54:50 2011
@@ -0,0 +1,59 @@
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME.  All others are
+# optional.  When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use.  Required.
+export JAVA_HOME=/usr/java/default
+
+# Extra Java CLASSPATH elements.  Optional.
+# export HADOOP_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+export HADOOP_HEAPSIZE=1500
+
+# Extra Java runtime options.  Empty by default.
+# export HADOOP_OPTS=-server
+export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"
+
+# Command specific options appended to HADOOP_OPTS when specified
+export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_NAMENODE_OPTS"
+export HADOOP_SECONDARYNAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_SECONDARYNAMENODE_OPTS"
+export HADOOP_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_DATANODE_OPTS"
+export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPTS"
+export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS"
+# export HADOOP_TASKTRACKER_OPTS=
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+# export HADOOP_CLIENT_OPTS
+
+# Extra ssh options.  Empty by default.
+# export HADOOP_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR"
+export HADOOP_SSH_OPTS="-o ConnectTimeout=5"
+
+# Where log files are stored.  $HADOOP_HOME/logs by default.
+# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs
+export HADOOP_LOG_DIR=/mnt/ephemeral-hdfs/logs
+
+# File naming remote slave hosts.  $HADOOP_HOME/conf/slaves by default.
+# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves
+
+# host:path where hadoop code should be rsync'd from.  Unset by default.
+# export HADOOP_MASTER=master:/home/$USER/src/hadoop
+
+# Seconds to sleep between slave commands.  Unset by default.  This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HADOOP_SLAVE_SLEEP=0.1
+
+# The directory where pid files are stored. /tmp by default.
+export HADOOP_PID_DIR=/var/hadoop/ephemeral-hdfs/pids
+
+# A string representing this instance of hadoop. $USER by default.
+# export HADOOP_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes.  See 'man nice'.
+# export HADOOP_NICENESS=10
+
+ulimit -n 16000

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hdfs-site.xml?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hdfs-site.xml (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hdfs-site.xml Sun Jun  5 08:54:50 2011
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+  <property>
+    <name>dfs.replication</name>
+    <value>3</value>
+  </property>
+
+  <property>
+    <name>dfs.block.size</name>
+    <value>134217728</value>
+  </property>
+
+  <property>
+    <name>dfs.data.dir</name>
+    <value>{{hdfs_data_dirs}}</value>
+  </property>
+
+  <property>
+    <name>dfs.namenode.handler.count</name>
+    <value>25</value>
+  </property>
+
+  <property>
+    <name>dfs.datanode.handler.count</name>
+    <value>8</value>
+  </property>
+
+</configuration>

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/mapred-site.xml?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/mapred-site.xml (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/mapred-site.xml Sun Jun  5 08:54:50 2011
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+  <property>
+    <name>mapred.job.tracker</name>
+    <value>{{active_master}}:9001</value>
+  </property>
+
+  <property>
+    <name>mapred.tasktracker.map.tasks.maximum</name>
+    <value>4</value>
+    <description>The maximum number of map tasks that will be run
+    simultaneously by a task tracker.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.tasktracker.reduce.tasks.maximum</name>
+    <value>2</value>
+    <description>The maximum number of reduce tasks that will be run
+    simultaneously by a task tracker.
+    </description>
+  </property>
+
+</configuration>

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/masters
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/masters?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/masters (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/masters Sun Jun  5 08:54:50 2011
@@ -0,0 +1 @@
+{{active_master}}

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/slaves
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/slaves?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/slaves (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/slaves Sun Jun  5 08:54:50 2011
@@ -0,0 +1 @@
+{{slave_list}}

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/approve-master-key
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/approve-master-key?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/approve-master-key (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/approve-master-key Sun Jun  5 08:54:50 2011
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5"
+
+ssh $SSH_OPTS {{active_master}} echo -n

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/approve-master-key
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/cluster-url
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/cluster-url?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/cluster-url (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/cluster-url Sun Jun  5 08:54:50 2011
@@ -0,0 +1 @@
+{{cluster_url}}

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/copy-dir
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/copy-dir?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/copy-dir (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/copy-dir Sun Jun  5 08:54:50 2011
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+if [[ "$#" != "1" ]] ; then
+  echo "Usage: copy-dir <dir>"
+  exit 1
+fi
+
+DIR=`readlink -f "$1"`
+DIR=`echo "$DIR"|sed 's@/$@@'`
+DEST=`dirname "$DIR"`
+
+SLAVES=`cat /root/mesos-ec2/slaves`
+
+SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5"
+
+echo "RSYNC'ing $DIR to slaves..."
+for slave in $SLAVES; do
+    echo $slave
+    rsync -e "ssh $SSH_OPTS" -az "$DIR" "$slave:$DEST" & sleep 0.5
+done
+wait

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/copy-dir
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/core-site.xml?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/core-site.xml (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/core-site.xml Sun Jun  5 08:54:50 2011
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>/mnt/hadoop-framework</value>
+  </property>
+
+  <property>
+    <name>fs.default.name</name>
+    <value>hdfs://{{active_master}}:9000</value>
+  </property>
+
+  <property>
+    <name>io.file.buffer.size</name>
+    <value>65536</value>
+  </property>
+
+</configuration>

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh Sun Jun  5 08:54:50 2011
@@ -0,0 +1,59 @@
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME.  All others are
+# optional.  When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use.  Required.
+export JAVA_HOME=/usr/java/default
+
+# Extra Java CLASSPATH elements.  Optional.
+# export HADOOP_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+export HADOOP_HEAPSIZE=1500
+
+# Extra Java runtime options.  Empty by default.
+# export HADOOP_OPTS=-server
+export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"
+
+# Command specific options appended to HADOOP_OPTS when specified
+export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_NAMENODE_OPTS"
+export HADOOP_SECONDARYNAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_SECONDARYNAMENODE_OPTS"
+export HADOOP_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_DATANODE_OPTS"
+export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPTS"
+export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS"
+# export HADOOP_TASKTRACKER_OPTS=
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+# export HADOOP_CLIENT_OPTS
+
+# Extra ssh options.  Empty by default.
+# export HADOOP_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR"
+export HADOOP_SSH_OPTS="-o ConnectTimeout=5"
+
+# Where log files are stored.  $HADOOP_HOME/logs by default.
+# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs
+export HADOOP_LOG_DIR=/mnt/hadoop-logs
+
+# File naming remote slave hosts.  $HADOOP_HOME/conf/slaves by default.
+# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves
+
+# host:path where hadoop code should be rsync'd from.  Unset by default.
+# export HADOOP_MASTER=master:/home/$USER/src/hadoop
+
+# Seconds to sleep between slave commands.  Unset by default.  This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HADOOP_SLAVE_SLEEP=0.1
+
+# The directory where pid files are stored. /tmp by default.
+# export HADOOP_PID_DIR=/var/hadoop/pids
+
+# A string representing this instance of hadoop. $USER by default.
+# export HADOOP_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes.  See 'man nice'.
+# export HADOOP_NICENESS=10
+
+ulimit -n 10000

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml Sun Jun  5 08:54:50 2011
@@ -0,0 +1,86 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+  <property>
+    <name>mapred.job.tracker</name>
+    <value>{{active_master}}:9001</value>
+  </property>
+
+  <property>
+    <name>mapred.local.dir</name>
+    <value>{{mapred_local_dirs}}</value>
+  </property>
+
+  <property>
+    <name>mapred.jobtracker.taskScheduler</name>
+    <value>org.apache.hadoop.mapred.MesosScheduler</value>
+  </property>
+
+  <property>
+    <name>mapred.mesos.master</name>
+    <value>{{cluster_url}}</value>
+  </property>
+
+  <property>
+    <name>io.file.buffer.size</name>
+    <value>65536</value>
+  </property>
+
+  <property>
+    <name>mapred.job.tracker.handler.count</name>
+    <value>20</value>
+  </property>
+
+  <property>
+    <name>tasktracker.http.threads</name>
+    <value>50</value>
+  </property>
+
+  <property>
+    <name>mapred.child.java.opts</name>
+    <value>-Xmx1512m</value>
+  </property>
+
+  <property>
+    <name>mapred.job.reuse.jvm.num.tasks</name>
+    <value>-1</value>
+  </property>
+
+  <property>
+    <name>io.sort.factor</name>
+    <value>15</value>
+  </property>
+
+  <property>
+    <name>io.sort.mb</name>
+    <value>100</value>
+  </property>
+
+  <property>
+    <name>mapred.mesos.localitywait</name>
+    <value>5000</value>
+  </property>
+
+  <property>
+    <name>mapred.tasktracker.map.tasks.maximum</name>
+    <value>6</value>
+  </property>
+
+  <property>
+    <name>mapred.tasktracker.reduce.tasks.maximum</name>
+    <value>6</value>
+  </property>
+
+  <property>
+    <name>io.sort.mb</name>
+    <value>200</value>
+    <description>The total amount of buffer memory to use while sorting
+    files, in megabytes.  By default, gives each merge stream 1MB, which
+    should minimize seeks.</description>
+  </property>
+
+</configuration>

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/haproxy+apache/haproxy.config.template
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/haproxy%2Bapache/haproxy.config.template?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/haproxy+apache/haproxy.config.template (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/haproxy+apache/haproxy.config.template Sun Jun  5 08:54:50 2011
@@ -0,0 +1,8 @@
+listen webfarm {{active_master}}:80
+       timeout server 7500
+       timeout client 7500
+       timeout connect 7500
+       mode http
+       balance roundrobin
+       option httpchk HEAD /index.html HTTP/1.0
+       stats uri /stats

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/Capfile
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/Capfile?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/Capfile (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/Capfile Sun Jun  5 08:54:50 2011
@@ -0,0 +1,417 @@
+set :source_machine, "{{active_master}}"
+set :install_dir,  "/opt/hypertable"
+set :hypertable_version, "0.9.4.2"
+set :default_dfs, "hadoop"
+set :default_config, "/root/mesos-ec2/hypertable/hypertable.cfg"
+set :default_additional_args, ""
+set :hbase_home, "/opt/hbase/current"
+set :default_client_multiplier, 1
+set :default_test_driver, "hypertable"
+set :default_test_args, ""
+
+role :source, "{{active_master}}"
+role :master, "{{active_master}}"
+role :hyperspace, "{{active_master}}"
+open("/root/mesos-ec2/slaves").each do |slave|
+  role :slave, slave
+end
+role :thriftbroker
+role :spare
+role :localhost, "{{active_master}}"
+role :test_client
+role :test_dispatcher
+
+######################### END OF USER CONFIGURATION ############################
+
+def install_machines
+  (roles[:master].servers | \
+   roles[:hyperspace].servers | \
+   roles[:slave].servers | \
+   roles[:thriftbroker].servers | \
+   roles[:spare].servers | \
+   roles[:test_client].servers | \
+   roles[:test_dispatcher].servers) - roles[:source].servers
+end
+
+role(:install) { install_machines }
+
+set(:dfs) do
+  "#{default_dfs}"
+end unless exists?(:dfs)
+
+set(:config) do
+  "#{default_config}"
+end unless exists?(:config)
+
+set(:additional_args) do
+  "#{default_additional_args}"
+end unless exists?(:additional_args)
+
+set(:test_driver) do
+  "#{default_test_driver}"
+end unless exists?(:test_driver)
+
+set(:test_args) do
+  "#{default_test_args}"
+end unless exists?(:test_args)
+
+set(:client_multiplier) do
+  "#{default_client_multiplier}".to_i
+end unless exists?(:client_multiplier)
+
+set :config_file, "#{config}".split('/')[-1]
+set :config_option, \
+    "--config=#{install_dir}/#{hypertable_version}/conf/#{config_file}"
+
+ desc <<-DESC
+    Copies config file to installation on localhost.
+    This task runs on localhost and copies the config file specified \
+    by the variable 'config' (default=#{config}) \
+    to the installation directory specified by the variable 'install_dir' \
+    (default=#{install_dir})
+ DESC
+task :copy_config, :roles => :localhost do
+  run("rsync #{config} #{install_dir}/#{hypertable_version}/conf/")
+end
+
+ desc <<-DESC
+    rsyncs installation directory to cluster.  For each machine in the \
+    cluster, his commannd  rsyncs the installation from the source \
+    installation machine specified by the variable 'source_machine' \
+    (default=#{source_machine})
+ DESC
+task :rsync, :roles => :install do
+  run <<-CMD
+     rsync -av --exclude=log --exclude=run --exclude=demo --exclude=fs --exclude=hyperspace/ #{source_machine}:#{install_dir}/#{hypertable_version} #{install_dir} &&
+     rsync -av --exclude=log --exclude=run --exclude=demo --exclude=fs --exclude=hyperspace/ #{source_machine}:#{install_dir}/#{hypertable_version}/conf/ #{install_dir}/#{hypertable_version}/conf
+  CMD
+end
+
+ desc <<-DESC
+    sets up the symbolic link 'current' in the installation area \
+    to point to the directory of the current version
+    (default=#{hypertable_version})
+ DESC
+task :set_current, :roles => [:install, :source] do
+  run <<-CMD
+   cd #{install_dir} &&
+   rm -f current &&
+   ln -s #{hypertable_version} current
+  CMD
+end
+
+ desc <<-DESC
+    Distributes installation.  This task copiles the config file and \
+    then rsyncs the installation to each machine in the cluster
+ DESC
+task :dist do
+  transaction do
+    copy_config
+    rsync
+  end
+end
+
+ desc <<-DESC
+    Distributes and fhsizes the installation, then copies
+    config and rsyncs
+ DESC
+task :fhsize do
+  transaction do
+    rsync
+    fhsize_install
+    copy_config
+    rsync
+  end
+end
+
+
+ desc <<-DESC
+    fhsize's the installations
+ DESC
+task :fhsize_install, :roles => [:install, :source] do
+  run <<-CMD
+     #{install_dir}/#{hypertable_version}/bin/fhsize.sh
+  CMD
+end
+
+desc "Verify that upgrade is OK."
+task :qualify_upgrade, :roles => :source do
+  run <<-CMD
+     #{install_dir}/#{hypertable_version}/bin/upgrade-ok.sh #{install_dir}/current #{hypertable_version}
+  CMD
+end
+
+ desc <<-DESC
+    Upgrades installation.  Stops servers, copies config, rsyncs
+    the installation, then copies hyperspace and the rangeserver
+    state in the run/ directory to new installation
+ DESC
+task :upgrade do
+  transaction do
+    qualify_upgrade
+    stop
+    copy_config
+    rsync
+    upgrade_hyperspace
+    upgrade_rangeservers
+    set_current
+  end
+end
+
+ desc <<-DESC
+    Upgrades (copies) the Hyperspace database from the current
+    installation to the new installation specified by the
+    hypertable_version (#{hypertable_version})
+ DESC
+task :upgrade_hyperspace, :roles => :hyperspace do
+  run <<-CMD
+    cp -dpR #{install_dir}/current/hyperspace \
+       #{install_dir}/#{hypertable_version}
+  CMD
+end
+
+ desc <<-DESC
+    Upgrades (copies) the RangeServers by copying the contents
+    of the run directory from the current installation to
+    installation specified by the hypertable_version
+    (#{hypertable_version})
+ DESC
+task :upgrade_rangeservers, :roles => :slave do
+  run <<-CMD
+    cp -dpR #{install_dir}/current/run \
+       #{install_dir}/#{hypertable_version}
+  CMD
+end
+
+desc "Starts all processes."
+task :start do
+  transaction do
+    start_hyperspace
+    start_master
+    start_slaves
+  end
+end
+
+desc "Starts hyperspace processes."
+task :start_hyperspace, :roles => :hyperspace do
+  run <<-CMD
+   #{install_dir}/current/bin/start-hyperspace.sh \
+      #{config_option}
+  CMD
+end
+
+desc "Starts master processes."
+task :start_master, :roles => :master do
+  run <<-CMD
+   #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \
+      #{config_option} &&
+   #{install_dir}/current/bin/start-master.sh #{config_option} &&
+   #{install_dir}/current/bin/start-monitoring.sh
+  CMD
+end
+
+desc "Starts slave processes."
+task :start_slaves, :roles => :slave do
+  run <<-CMD
+   #{install_dir}/current/bin/random-wait.sh 5 &&
+   #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \
+      #{config_option} &&
+   #{install_dir}/current/bin/start-rangeserver.sh \
+      #{config_option} &&
+   #{install_dir}/current/bin/start-thriftbroker.sh \
+      #{config_option}
+  CMD
+end
+
+desc "Starts ThriftBroker processes."
+task :start_thriftbrokers, :roles => :thriftbroker do
+  run <<-CMD
+   #{install_dir}/current/bin/random-wait.sh 5 &&
+   #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \
+      #{config_option} &&
+   #{install_dir}/current/bin/start-thriftbroker.sh \
+      #{config_option}
+  CMD
+end
+
+
+desc "Starts DFS brokers."
+task :start_dfsbrokers, :roles => [:master, :slave] do
+  run "#{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \
+      #{config_option}"
+end
+
+desc "Stops all servers."
+task :stop do
+  transaction do
+    stop_slaves
+    stop_master
+    stop_hyperspace
+  end
+end
+
+desc "Stops slave processes."
+task :stop_slaves, :roles => :slave do
+  run <<-CMD
+  #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master #{additional_args}
+  CMD
+end
+
+desc "Stops master processes."
+task :stop_master, :roles => :master do
+  run <<-CMD
+  #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-rangeserver #{additional_args} &&
+  #{install_dir}/current/bin/stop-monitoring.sh
+  CMD
+end
+
+desc "Stops hyperspace processes."
+task :stop_hyperspace, :roles => :hyperspace do
+  run <<-CMD 
+  #{install_dir}/current/bin/stop-hyperspace.sh
+  CMD
+end
+
+desc "Stops ThriftBroker processes."
+task :stop_thriftbrokers, :roles => :thriftbroker do
+  run <<-CMD 
+  #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master --no-rangeserver
+  CMD
+end
+
+desc "Cleans hyperspace & rangeservers, removing all tables."
+task :cleandb do
+  transaction do
+    clean_master
+    clean_hyperspace
+    clean_slaves
+  end  
+end
+
+desc "Cleans master state but not hyperspace."
+task :clean_master, :roles => :master do
+  run <<-CMD
+   #{install_dir}/current/bin/start-dfsbroker.sh #{dfs} \
+      #{config_option} && \
+   #{install_dir}/current/bin/clean-database.sh #{config_option} ;
+  CMD
+end
+
+desc "Cleans hyperspace."
+task :clean_hyperspace, :roles => :hyperspace do
+  run <<-CMD
+   #{install_dir}/current/bin/clean-hyperspace.sh
+  CMD
+end
+
+desc "Cleans rangeservers and master state but not hyperspace."
+task :clean_slaves, :roles => :slave do
+  run <<-CMD
+   #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master &&
+   rm -rf #{install_dir}/current/run/*
+  CMD
+end
+
+desc "Reports status for all processes."
+task :status do
+  transaction do
+    dfs_status
+    master_status
+    hyperspace_status
+    rangeserver_status
+  end
+end
+
+desc "Get status for dfs processes."
+task :dfs_status, :roles => [:master, :slave] do
+  run <<-CMD
+   #{install_dir}/current/bin/ht serverup dfsbroker
+  CMD
+end
+
+desc "Get status for Hypertable.Master process."
+task :master_status, :roles => [:master] do
+  run <<-CMD
+   #{install_dir}/current/bin/ht serverup master
+  CMD
+end
+
+desc "Get status for Hyperspace.Master process."
+task :hyperspace_status, :roles => [:hyperspace] do
+  run <<-CMD
+   #{install_dir}/current/bin/ht serverup hyperspace
+  CMD
+end
+
+desc "Get status for rangeserver processes."
+task :rangeserver_status, :roles => [:slave] do
+  run <<-CMD
+   #{install_dir}/current/bin/ht serverup rangeserver
+  CMD
+end
+
+set :default_dumpfile, "/tmp/rsdump.txt"
+
+set(:dumpfile) do
+  "#{default_dumpfile}"
+end unless exists?(:dumpfile)
+
+desc "Run dump command on each rangeserver"
+task :rangeserver_dump, :roles => [:slave] do
+  run <<-CMD
+   echo "dump NOKEYS '#{dumpfile}';" | #{install_dir}/current/bin/ht ht_rsclient --batch #{config_option}
+  CMD
+end
+
+
+if "#{test_driver}" == "hypertable"
+  set :thrift_broker_command, "#{install_dir}/current/bin/start-thriftbroker.sh #{config_option}"
+  set :start_test_client_command, "#{install_dir}/current/bin/start-test-client.sh --count #{client_multiplier} #{roles[:test_dispatcher].servers[0]}"
+  set :run_test_dispatcher_command, "#{install_dir}/current/bin/jrun --pidfile #{install_dir}/#{hypertable_version}/run/Hypertable.TestDispatcher.pid org.hypertable.examples.PerformanceTest.Dispatcher --driver=#{test_driver} --clients=#{roles[:test_client].servers.length*client_multiplier} #{test_args}"
+  set :stop_test_args, ""
+elsif "#{test_driver}" == "hbase"
+  set :thrift_broker_command, "echo -n"
+  set :start_test_client_command, "#{install_dir}/current/bin/start-test-client.sh --jrun-opts \"--add-to-classpath #{hbase_home}/conf\" --count #{client_multiplier} #{roles[:test_dispatcher].servers[0]}"
+  set :run_test_dispatcher_command, "#{install_dir}/current/bin/jrun --pidfile #{install_dir}/#{hypertable_version}/run/Hypertable.TestDispatcher.pid --add-to-classpath #{hbase_home}/conf org.hypertable.examples.PerformanceTest.Dispatcher --driver=#{test_driver} --clients=#{roles[:test_client].servers.length*client_multiplier} #{test_args}"
+  set :stop_test_args, "--no-thriftbroker --no-dfsbroker"
+else
+  set :thrift_broker_command, "echo Invalid test driver - #{test_driver}"
+  set :start_test_client_command, "echo Invalid test driver - #{test_driver}"
+  set :run_test_dispatcher_command, "echo Invalid test driver - #{test_driver}"
+  set :stop_test_args, "--no-thriftbroker --no-dfsbroker"
+end
+
+desc "Starts test clients."
+task :start_test_clients, :roles => :test_client do
+  run <<-CMD
+   #{install_dir}/current/bin/random-wait.sh 5 &&
+   #{thrift_broker_command} &&
+   #{start_test_client_command}
+  CMD
+end
+
+desc "Run test dispatcher."
+task :run_test_dispatcher, :roles => :test_dispatcher do
+  run <<-CMD
+   #{thrift_broker_command} &&
+   #{run_test_dispatcher_command}
+  CMD
+end
+
+desc "Stops test."
+task :stop_test, :roles => [:test_client, :test_dispatcher] do
+  run <<-CMD
+  #{install_dir}/current/bin/stop-servers.sh --no-hyperspace --no-master --no-rangeserver #{stop_test_args}
+  CMD
+end
+
+desc "Run test"
+task :run_test do
+  transaction do
+    stop_test
+    start_test_clients
+    run_test_dispatcher
+  end
+end
+
+

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/hypertable.cfg
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/hypertable.cfg?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/hypertable.cfg (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hypertable/hypertable.cfg Sun Jun  5 08:54:50 2011
@@ -0,0 +1,43 @@
+#
+# hypertable.cfg
+#
+
+# HDFS Broker
+HdfsBroker.Port=38030
+HdfsBroker.fs.default.name=hdfs://{{active_master}}:9010
+HdfsBroker.Workers=20
+
+# Ceph Broker
+CephBroker.Port=38030
+CephBroker.Workers=20
+CephBroker.MonAddr=10.0.1.245:6789
+
+# Local Broker
+DfsBroker.Local.Port=38030
+DfsBroker.Local.Root=fs/local
+
+# DFS Broker - for clients
+DfsBroker.Host=localhost
+DfsBroker.Port=38030
+
+# Hyperspace
+Hyperspace.Replica.Host={{active_master}}
+Hyperspace.Replica.Port=38040
+Hyperspace.Replica.Dir=hyperspace
+Hyperspace.Replica.Workers=20
+
+# Hypertable.Master
+Hypertable.Master.Host={{active_master}}
+Hypertable.Master.Port=38050
+Hypertable.Master.Workers=20
+
+
+# Hypertable.RangeServer
+Hypertable.RangeServer.Port=38060
+
+Hyperspace.KeepAlive.Interval=30000
+Hyperspace.Lease.Interval=1000000
+Hyperspace.GracePeriod=200000
+
+# ThriftBroker
+ThriftBroker.Port=38080

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/masters
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/masters?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/masters (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/masters Sun Jun  5 08:54:50 2011
@@ -0,0 +1 @@
+{{master_list}}

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/mesos-daemon
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/mesos-daemon?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/mesos-daemon (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/mesos-daemon Sun Jun  5 08:54:50 2011
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Set up MESOS_HOME in order to find projd
+export MESOS_HOME=/root/mesos
+
+# Set MESOS_PUBLIC_DNS so slaves can be linked in master web UI
+export MESOS_PUBLIC_DNS=`wget -q -O - http://instance-data.ec2.internal/latest/meta-data/public-hostname`
+
+# Set PATH to include Scala
+export PATH=$PATH:/root/scala-2.8.0.final/bin
+
+# Set HADOOP_HOME variable to allow slaves to get executors from HDFS
+export HADOOP_HOME=/root/hadoop-0.20.2
+
+ulimit -n 8192
+
+PROGRAM=$1
+shift
+
+EXTRA_OPTS=""
+if [ "$PROGRAM" == "mesos-slave" ]; then
+  # Compute CPU resources (if not specified).
+  if [[ "$*" != *--cpus* ]]; then
+    CPUS=`grep processor /proc/cpuinfo | wc -l`
+    EXTRA_OPTS="$EXTRA_OPTS --cpus=$CPUS"
+  fi
+
+  # Compute memory resources (if not specified).
+  if [[ "$*" != *--mem* ]]; then
+    MEM_KB=`cat /proc/meminfo | grep MemTotal | awk '{print $2}'`
+    MEM=$[(MEM_KB - 1024 * 1024) / 1024]
+    EXTRA_OPTS="$EXTRA_OPTS --mem=$MEM"
+  fi
+fi
+
+cd $MESOS_HOME/bin
+nohup ./$PROGRAM --log_dir=/mnt/mesos-logs --work_dir=/mnt/mesos-work $EXTRA_OPTS $@ </dev/null >/mnt/mesos-logs/$PROGRAM.out 2>&1 &

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/mesos-daemon
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/redeploy-mesos
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/redeploy-mesos?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/redeploy-mesos (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/redeploy-mesos Sun Jun  5 08:54:50 2011
@@ -0,0 +1,24 @@
+#!/bin/bash
+cd /root/mesos-ec2
+
+MASTERS=`cat masters`
+NUM_MASTERS=`cat masters | wc -l`
+SLAVES=`cat slaves`
+
+SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5"
+
+if [[ $NUM_MASTERS -gt 1 ]]; then
+  echo "RSYNC'ing /root/mesos to masters..."
+  for master in $MASTERS; do
+    echo $master
+    rsync -e "ssh $SSH_OPTS" -az --exclude '*.d' --exclude '*.o' --exclude '*.cpp' --exclude '*.hpp' --exclude '*.pyc' --exclude 'mesos/frameworks/hadoop-0.20.0/logs/*' --exclude 'mesos/work' --exclude 'mesos/logs' --exclude 'mesos/test_output' /root/mesos $master:/root & sleep 0.3
+  done
+  wait
+fi
+
+echo "RSYNC'ing /root/mesos to slaves..."
+for slave in $SLAVES; do
+  echo $slave
+  rsync -e "ssh $SSH_OPTS" -az --exclude '*.d' --exclude '*.o' --exclude '*.cpp' --exclude '*.hpp' --exclude '*.pyc' --exclude 'mesos/frameworks/hadoop-0.20.0/logs/*' --exclude 'mesos/work' --exclude 'mesos/logs' --exclude 'mesos/test_output' /root/mesos $slave:/root & sleep 0.3
+done
+wait

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/redeploy-mesos
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup Sun Jun  5 08:54:50 2011
@@ -0,0 +1,262 @@
+#!/bin/bash
+
+# Make sure we are in the mesos-ec2 directory
+cd /root/mesos-ec2
+
+# Set hostname based on EC2 private DNS name, so that it is set correctly
+# even if the instance is restarted with a different private DNS name
+PRIVATE_DNS=`wget -q -O - http://instance-data.ec2.internal/latest/meta-data/local-hostname`
+hostname $PRIVATE_DNS
+echo $PRIVATE_DNS > /etc/hostname
+export HOSTNAME=$PRIVATE_DNS  # Fix the bash built-in hostname variable too
+
+echo "Setting up Mesos master on `hostname`..."
+
+# Read command-line arguments
+OS_NAME=$1
+DOWNLOAD_METHOD=$2
+BRANCH=$3
+
+MASTERS_FILE="masters"
+MASTERS=`cat $MASTERS_FILE`
+NUM_MASTERS=`cat $MASTERS_FILE | wc -l`
+SLAVES=`cat slaves`
+ZOOS=`cat zoo`
+
+if [[ $ZOOS = *NONE* ]]; then
+  NUM_ZOOS=0
+  ZOOS=""
+else
+  NUM_ZOOS=`cat zoo | wc -l`
+fi
+
+# Scripts that get used for/while running Mesos.
+SCRIPTS="approve-master-key
+         copy-dir
+         mesos-daemon
+         redeploy-mesos
+         setup-slave              
+         ssh-no-keychecking
+         start-hypertable
+         start-mesos
+         stop-hypertable
+         stop-mesos"
+
+EPHEMERAL_HDFS=/root/ephemeral-hdfs
+PERSISTENT_HDFS=/root/persistent-hdfs
+
+#TODO(*): update config scripts to have conditionals for handling different
+#         platforms
+JAVA_HOME=/usr/java/default
+
+SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5"
+
+if [[ `tty` == "not a tty" ]] ; then
+    echo "Expecting a tty or pty! (use the ssh -t option)."
+    exit 1
+fi
+
+echo "Setting executable permissions on scripts..."
+for s in $SCRIPTS; do chmod u+x $s; done
+
+echo "Running setup-slave on master to mount filesystems, etc..."
+./setup-slave
+
+echo "SSH'ing to master machine(s) to approve key(s)..."
+for master in $MASTERS; do
+  echo $master
+  ssh $SSH_OPTS $master echo -n &
+  sleep 0.3
+done
+ssh $SSH_OPTS localhost echo -n &
+ssh $SSH_OPTS `hostname` echo -n &
+wait
+
+if [[ $NUM_ZOOS != 0 ]] ; then
+  echo "SSH'ing to ZooKeeper server(s) to approve keys..."
+  zid=1
+  for zoo in $ZOO; do
+    echo $zoo
+    ssh $SSH_OPTS $zoo echo -n \; mkdir -p /tmp/zookeeper \; echo $zid \> /tmp/zookeeper/myid &
+    zid=$(($zid+1))
+    sleep 0.3
+  done
+fi
+
+echo "SSH'ing to slaves to approve keys..."
+for slave in $SLAVES; do
+  echo $slave
+  ssh $SSH_OPTS $slave echo -n &
+  sleep 0.3
+done
+
+echo "Waiting for ssh commands to finish..."
+wait
+
+if [[ $NUM_MASTERS -gt 1 ]] ; then
+  echo "RSYNC'ing /root/mesos-ec2 to other master servers..."
+  for master in `cat $MASTERS_FILE | sed '1d'`; do
+      echo $master
+      rsync -e "ssh $SSH_OPTS" -az /root/mesos-ec2 $master:/root & sleep 0.3
+  done
+  wait
+fi
+
+if [[ $NUM_ZOOS != 0 ]] ; then
+  echo "RSYNC'ing /root/mesos-ec2 to ZooKeeper servers..."
+  for zoo in $ZOOS; do
+      echo $zoo
+      rsync -e "ssh $SSH_OPTS" -az /root/mesos-ec2 $zoo:/root & sleep 0.3
+  done
+  wait
+fi
+
+echo "RSYNC'ing /root/mesos-ec2 to slaves..."
+for slave in $SLAVES; do
+  echo $slave
+  rsync -e "ssh $SSH_OPTS" -az /root/mesos-ec2 $slave:/root &
+  scp $SSH_OPTS ~/.ssh/id_rsa $slave:.ssh &
+  sleep 0.3
+done
+wait
+
+echo "Running slave setup script on slave and zookeeper nodes..."
+for node in $SLAVES $ZOO; do
+  echo $node
+  ssh -t $SSH_OPTS root@$node "mesos-ec2/setup-slave" & sleep 0.3
+done
+wait
+
+if [[ $NUM_MASTERS -gt 1 ]] ; then
+  echo "Running slave setup script on other masters..."
+  for master in `cat $MASTERS_FILE | sed '1d'`; do
+    echo $master
+    rsync -e "ssh $SSH_OPTS" mesos-ec2/setup-slave & sleep 0.3
+  done
+  wait
+  echo "RSYNC'ing HDFS config files to other masters..."
+  for master in `cat $MASTERS_FILE | sed '1d'`; do
+    echo $master
+    rsync -e "ssh $SSH_OPTS" -az $EPHEMERAL_HDFS/conf $master:$EPHEMERAL_HDFS &
+    rsync -e "ssh $SSH_OPTS" -az $PERSISTENT_HDFS/conf $master:$PERSISTENT_HDFS &
+    sleep 0.3
+  done
+  wait
+fi
+
+echo "RSYNC'ing HDFS config files to slaves..."
+for slave in $SLAVES; do
+  echo $slave
+  rsync -e "ssh $SSH_OPTS" -az $EPHEMERAL_HDFS/conf $slave:$EPHEMERAL_HDFS &
+  rsync -e "ssh $SSH_OPTS" -az $PERSISTENT_HDFS/conf $slave:$PERSISTENT_HDFS &
+  sleep 0.3
+done
+wait
+
+DOWNLOADED=0
+
+if [[ "$DOWNLOAD_METHOD" == "git" ]] ; then
+  # change git's ssh command so it does not ask to accept a keys
+  export GIT_SSH=/root/mesos-ec2/ssh-no-keychecking
+  REPOSITORY=git://github.com/mesos/mesos.git
+  echo "Checking out Mesos from $REPOSITORY"
+  pushd /root > /dev/null 2>&1
+  rm -rf mesos mesos.tgz
+  # Set git SSH command to a script that uses -o StrictHostKeyChecking=no
+  git clone $REPOSITORY mesos
+  pushd mesos 2>&1
+  git checkout -b $BRANCH --track origin/$BRANCH
+  popd > /dev/null 2>&1
+  popd > /dev/null 2>&1
+  DOWNLOADED=1
+fi
+
+# Build Mesos if we downloaded it
+if [[ "$DOWNLOADED" == "1" ]] ; then
+  echo "Building Mesos..."
+  pushd /root/mesos > /dev/null 2>&1
+  ./configure.template.ubuntu-lucid-64
+  make clean
+  make
+  popd > /dev/null 2>&1
+  if [ -d /root/spark ] ; then
+    echo "Building Spark..."
+    pushd /root/spark > /dev/null 2>&1
+    MESOS_HOME=/root/mesos make all native
+    popd > /dev/null 2>&1
+  fi
+  echo "Building Hadoop framework..."
+  pushd /root/mesos/frameworks/hadoop-0.20.2 > /dev/null 2>&1
+  ant
+  ant examples
+  popd > /dev/null 2>&1
+fi
+
+echo "Setting up Hadoop framework config files..."
+cp hadoop-framework-conf/* /root/mesos/frameworks/hadoop-0.20.2/conf
+
+echo "Setting up haproxy+apache framework config files..."
+cp haproxy+apache/* /root/mesos/frameworks/haproxy+apache
+
+echo "Setting up Spark config files..."
+# TODO: This currently overwrites whatever the user wrote there; on
+# the other hand, we also don't want to leave an old file created by
+# us because it would have the wrong hostname for HDFS etc
+mkdir -p /root/spark/conf
+echo "-Dspark.dfs=hdfs://$HOSTNAME:9000" \
+     > /root/spark/conf/java-opts
+
+echo "Redeploying /root/mesos..."
+./redeploy-mesos
+
+echo "Setting up NFS..."
+if [ ! -e /nfs ] ; then
+  mkdir -p /mnt/nfs
+  rm -fr /nfs
+  ln -s /mnt/nfs /nfs
+fi
+if ! grep -e '^/nfs ' /etc/exports; then
+  echo "/nfs    10.0.0.0/8(ro,async,no_subtree_check)" >> /etc/exports
+fi
+/sbin/service portmap start
+/sbin/service nfs start
+# Unexport and re-export everything in /etc/exports because, if we are
+# restarting a stopped EC2 instance, we might have had an entry for /nfs in
+# /etc/exports before we created /mnt/nfs.
+exportfs -ua
+exportfs -a
+
+echo "Mounting NFS on slaves..."
+for slave in $SLAVES; do
+  echo $slave
+  ssh -t $SSH_OPTS root@$slave "mkdir -p /nfs; service portmap start; service nfs start; mount $HOSTNAME:/nfs /nfs" & sleep 0.3
+done
+wait
+
+echo "Formatting ephemeral HDFS namenode..."
+$EPHEMERAL_HDFS/bin/hadoop namenode -format
+
+echo "Starting ephemeral HDFS..."
+$EPHEMERAL_HDFS/bin/start-dfs.sh
+
+if [[ ! -e /vol/persistent-hdfs/dfs/name ]] ; then
+  echo "Formatting persistent HDFS namenode..."
+  $PERSISTENT_HDFS/bin/hadoop namenode -format
+fi
+
+echo "Starting persistent HDFS..."
+$PERSISTENT_HDFS/bin/start-dfs.sh
+
+sleep 1
+
+if [[ $NUM_ZOOS != 0 ]]; then
+  echo "Starting ZooKeeper quorum..."
+  for zoo in $ZOOS; do
+    ssh $SSH_OPTS $zoo "/root/mesos/third_party/zookeeper-*/bin/zkServer.sh start </dev/null >/dev/null" & sleep 0.1
+  done
+  wait
+  sleep 2
+fi
+
+echo "Starting Mesos cluster..."
+./start-mesos

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup
------------------------------------------------------------------------------
    svn:executable = *

Copied: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-slave (from r1132175, incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave)
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-slave?p2=incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-slave&p1=incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave&r1=1132175&r2=1132176&rev=1132176&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave (original)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-slave Sun Jun  5 08:54:50 2011
@@ -21,13 +21,14 @@ XFS_MOUNT_OPTS="defaults,noatime,nodirat
 # (for example /mnt, /mnt2, and so on)
 function create_hadoop_dirs {
   location=$1
-  mkdir -p $location/hdfs/dfs $location/hadoop/tmp
+  mkdir -p $location/ephemeral-hdfs/data $location/hadoop/tmp
   mkdir -p $location/hadoop/mrlocal $location/hadoop/mrlocal2
 }
 
 # Set up Hadoop and Mesos directories in /mnt
 create_hadoop_dirs /mnt
-mkdir -p /mnt/hdfs-logs
+mkdir -p /mnt/ephemeral-hdfs/logs
+mkdir -p /mnt/persistent-hdfs/logs
 mkdir -p /mnt/hadoop-logs
 mkdir -p /mnt/mesos-logs
 mkdir -p /mnt/mesos-work
@@ -74,3 +75,6 @@ fi
 # Remove ~/.ssh/known_hosts because it gets polluted as you start/stop many
 # clusters (new machines tend to come up under old hostnames)
 rm -f /root/.ssh/known_hosts
+
+# SSH to the master to approve its key (needed for Capistrano to work)
+./approve-master-key

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-torque
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-torque?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-torque (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-torque Sun Jun  5 08:54:50 2011
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+cd /root/mesos-ec2
+
+MASTERS=`cat master`
+SLAVES_FILE="slaves"
+SLAVES=`cat $SLAVES_FILE`
+
+SCHEDULER_ITERATION=5
+
+#These seem to be broken, i.e. missing directories after install
+#ssh $MASTERS "apt-get install -y torque-server"
+#ssh $MASTERS "apt-get install -y torque-scheduler"
+#ssh $MASTERS "apt-get install -y torque-client"
+
+#install torque: download/unzip torque
+function installtorque {
+	pushd ~
+	echo "downloading and installing torque on master"
+	#wget http://www.clusterresources.com/downloads/torque/torque-2.4.7.tar.gz
+	rm -rf torque-2.4.7.tar.gz
+	wget http://mesos.berkeley.edu/torque-2.4.7.tar.gz
+	tar xzf torque-2.4.7.tar.gz
+	pushd torque-2.4.7
+	./configure --prefix=/usr
+	make -j8
+	make install
+	popd;popd
+}
+
+function setuptorque {
+	pushd ~/torque-2.4.7
+	echo "running ldconfig on master"
+	ldconfig
+        #./torque.setup root # Note: sets some defaults for batch queue
+	qterm
+        yes|./torque.setup root localhost # Note: sets some defaults for batch queue
+
+	#WARNING: allow root to qsub for debug purposes only, may be dangerous
+	qmgr -c 'set server acl_roots+=root@*' #allow root to submit jobs
+	qmgr -c "set server scheduler_iteration=$SCHEDULER_ITERATION"
+	#qmgr -c 's s allow_node_submit=true' #other hosts can submit too
+
+	NUM_SLAVES=`cat ~/mesos-ec2/slaves|wc -l`
+	#the server be restarted after this
+	qmgr -c "set queue batch resources_available.nodect=$NUM_SLAVES"
+	#qmgr -c "set server resources_available.nodect=$NUM_SLAVES"
+	qterm
+        pbs_server
+
+	touch ~/.rhosts
+	echo `hostname` |cat >> ~/.rhosts
+	echo `hostname -f` |cat >> ~/.rhosts
+	echo localhost |cat >> ~/.rhosts
+
+	popd
+}
+
+
+function installslaves {
+	pushd ~/torque-2.4.7
+	echo "building packages for slave"
+	make packages
+	#install torque-mom on slave nodes
+	apt-get install -y dsh
+	
+        echo "copying slave install packages to nfs"
+	mkdir /nfs/torque
+	cp torque-package-mom-linux-x86_64.sh /nfs/torque/torque-package-mom-linux-x86_64.sh
+	cp torque-package-mom-linux-x86_64.sh /nfs/torque/torque-package-clients-linux-x86_64.sh
+
+	echo "installing torque mom and clients package on slaves"
+	for i in `cat $SLAVES_FILE`; do ssh $i /nfs/torque/torque-package-mom-linux-x86_64.sh --install; ldconfig; done
+	for i in `cat $SLAVES_FILE`; do ssh $i /nfs/torque/torque-package-clients-linux-x86_64.sh --install; ldconfig; done
+
+	echo "Running ldconfig on slaves"
+	dsh -f $SLAVES_FILE ldconfig
+	popd
+}
+
+function installmpi {
+        #setup mpich2 on all of the cluster nodes
+        ./setup-mpi
+
+        #setup prologue script
+        cp ./prologue.setup-mpi-master /var/spool/torque/mom_priv/prologue
+        cp ./epilogue.kill-mpi-ring /var/spool/torque/mom_priv/epilogue
+        
+	for i in `cat $SLAVES_FILE`; do scp ./prologue.setup-mpi-master $i:/var/spool/torque/mom_priv/prologue; done
+	for i in `cat $SLAVES_FILE`; do scp ./epilogue.kill-mpi-ring $i:/var/spool/torque/mom_priv/epilogue; done
+}
+
+function installmaui {
+	pushd ~
+	#http://www.clusterresources.com/download/maui/maui-3.3.tar.gz
+	rm -rf mesos-maui-3.3.tar
+	wget http://mesos.berkeley.edu/mesos-maui-3.3.tar
+	tar -xf mesos-maui-3.3.tar
+	pushd maui-3.3
+	./configure
+	make
+	make install
+	/usr/local/maui/sbin/maui
+}
+
+installtorque
+setuptorque
+installslaves
+installmpi
+installmaui

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup-torque
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/slaves
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/slaves?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/slaves (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/slaves Sun Jun  5 08:54:50 2011
@@ -0,0 +1 @@
+{{slave_list}}

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/ssh-no-keychecking
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/ssh-no-keychecking?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/ssh-no-keychecking (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/ssh-no-keychecking Sun Jun  5 08:54:50 2011
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# Utility script that exec's SSH without key checking so that we can check
+# out code from GitHub without prompting the user.
+
+exec ssh -o StrictHostKeyChecking=no $@

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/ssh-no-keychecking
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-hypertable
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-hypertable?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-hypertable (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-hypertable Sun Jun  5 08:54:50 2011
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd /root/mesos-ec2/hypertable
+cap dist && cap start

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-hypertable
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos Sun Jun  5 08:54:50 2011
@@ -0,0 +1,54 @@
+#!/bin/bash
+cd /root/mesos-ec2
+
+MASTERS=`cat masters`
+ACTIVE_MASTER=`cat masters | head -1`
+SLAVES=`cat slaves`
+ZOOS=`cat zoo`
+
+
+if [[ $ZOOS = *NONE* ]]; then
+  NUM_ZOOS=0
+else
+  NUM_ZOOS=`cat zoo | wc -l`
+fi
+
+SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5"
+
+cluster_url=`cat cluster-url`
+
+echo "Running with cluster URL: "$cluster_url
+
+if [[ $NUM_ZOOS != 0 ]]; then
+  masterid=1
+  for master in $MASTERS; do
+    echo "Starting master $masterid on $master"
+    ssh $SSH_OPTS $master "/root/mesos-ec2/mesos-daemon mesos-master -p 5050 -u $cluster_url $@ </dev/null >/dev/null" & sleep 0.3
+    masterid=$(($masterid+1))
+  done
+  wait
+else
+  echo "Starting master on $ACTIVE_MASTER"
+  ssh $SSH_OPTS $ACTIVE_MASTER "/root/mesos-ec2/mesos-daemon mesos-master -p 5050 $@ </dev/null >/dev/null"
+fi
+
+sleep 2
+
+for slave in $SLAVES; do
+  echo "Starting slave on $slave"
+  ssh $SSH_OPTS $slave "/root/mesos-ec2/mesos-daemon mesos-slave -u ${cluster_url} </dev/null >/dev/null" &
+  sleep 0.3
+done
+wait
+
+if [[ $NUM_ZOOS != 0 ]]; then
+  echo "ZooKeeper is running at"
+  for zoo in $ZOOS; do
+    echo "      $zoo:2181"
+  done
+fi
+
+echo "Everything's started! You can view the master Web UI at"
+for master in $MASTERS; do
+  echo "      http://$master:8080"
+done

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-hypertable
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-hypertable?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-hypertable (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-hypertable Sun Jun  5 08:54:50 2011
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd /root/mesos-ec2/hypertable
+cap stop

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-hypertable
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-mesos
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-mesos?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-mesos (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-mesos Sun Jun  5 08:54:50 2011
@@ -0,0 +1,21 @@
+#!/bin/bash
+cd /root/mesos-ec2
+
+MASTERS=`cat masters`
+SLAVES=`cat slaves`
+
+SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5"
+
+for slave in $SLAVES; do
+  echo "Stopping slave on $slave"
+  ssh $SSH_OPTS $slave pkill mesos-slave &
+  sleep 0.1
+done
+wait
+
+for master in $MASTERS; do
+  echo "Stopping master on $master"
+  ssh $SSH_OPTS $master pkill mesos-master &
+  sleep 0.1
+done
+wait

Propchange: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/stop-mesos
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/zoo
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/zoo?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/zoo (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/zoo Sun Jun  5 08:54:50 2011
@@ -0,0 +1 @@
+{{zoo_list}}

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/core-site.xml?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/core-site.xml (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/core-site.xml Sun Jun  5 08:54:50 2011
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>/vol/persistent-hdfs</value>
+  </property>
+
+  <property>
+    <name>fs.default.name</name>
+    <value>hdfs://{{active_master}}:9010</value>
+  </property>
+
+  <property>
+    <name>io.file.buffer.size</name>
+    <value>65536</value>
+  </property>
+
+</configuration>

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hadoop-env.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hadoop-env.sh?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hadoop-env.sh (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hadoop-env.sh Sun Jun  5 08:54:50 2011
@@ -0,0 +1,59 @@
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME.  All others are
+# optional.  When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use.  Required.
+export JAVA_HOME=/usr/java/default
+
+# Extra Java CLASSPATH elements.  Optional.
+# export HADOOP_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+export HADOOP_HEAPSIZE=1000
+
+# Extra Java runtime options.  Empty by default.
+# export HADOOP_OPTS=-server
+export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"
+
+# Command specific options appended to HADOOP_OPTS when specified
+export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_NAMENODE_OPTS"
+export HADOOP_SECONDARYNAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_SECONDARYNAMENODE_OPTS"
+export HADOOP_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_DATANODE_OPTS"
+export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPTS"
+export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS"
+# export HADOOP_TASKTRACKER_OPTS=
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+# export HADOOP_CLIENT_OPTS
+
+# Extra ssh options.  Empty by default.
+# export HADOOP_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR"
+export HADOOP_SSH_OPTS="-o ConnectTimeout=5"
+
+# Where log files are stored.  $HADOOP_HOME/logs by default.
+# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs
+export HADOOP_LOG_DIR=/mnt/persistent-hdfs/logs
+
+# File naming remote slave hosts.  $HADOOP_HOME/conf/slaves by default.
+# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves
+
+# host:path where hadoop code should be rsync'd from.  Unset by default.
+# export HADOOP_MASTER=master:/home/$USER/src/hadoop
+
+# Seconds to sleep between slave commands.  Unset by default.  This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HADOOP_SLAVE_SLEEP=0.1
+
+# The directory where pid files are stored. /tmp by default.
+export HADOOP_PID_DIR=/var/hadoop/persistent-hdfs/pids
+
+# A string representing this instance of hadoop. $USER by default.
+# export HADOOP_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes.  See 'man nice'.
+# export HADOOP_NICENESS=10
+
+ulimit -n 16000

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hdfs-site.xml?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hdfs-site.xml (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/hdfs-site.xml Sun Jun  5 08:54:50 2011
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+  <property>
+    <name>dfs.replication</name>
+    <value>2</value>
+  </property>
+
+  <property>
+    <name>dfs.block.size</name>
+    <value>134217728</value>
+  </property>
+
+  <property>
+    <name>dfs.secondary.http.address</name>
+    <value>0.0.0.0:60090</value>
+    <description>
+      The secondary namenode http server address and port.
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+  
+  <property>
+    <name>dfs.datanode.address</name>
+    <value>0.0.0.0:60010</value>
+    <description>
+      The address where the datanode server will listen to.
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+  
+  <property>
+    <name>dfs.datanode.http.address</name>
+    <value>0.0.0.0:60075</value>
+    <description>
+      The datanode http server address and port.
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+  
+  <property>
+    <name>dfs.datanode.ipc.address</name>
+    <value>0.0.0.0:60020</value>
+    <description>
+      The datanode ipc server address and port.
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+  
+  <property>
+    <name>dfs.http.address</name>
+    <value>0.0.0.0:60070</value>
+    <description>
+      The address and the base port where the dfs namenode web ui will listen on.
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.namenode.handler.count</name>
+    <value>25</value>
+  </property>
+
+  <property>
+    <name>dfs.datanode.handler.count</name>
+    <value>8</value>
+  </property>
+
+</configuration>

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/mapred-site.xml?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/mapred-site.xml (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/mapred-site.xml Sun Jun  5 08:54:50 2011
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+  <property>
+    <name>mapred.job.tracker</name>
+    <value>{{active_master}}:9001</value>
+  </property>
+
+  <property>
+    <name>mapred.tasktracker.map.tasks.maximum</name>
+    <value>4</value>
+    <description>The maximum number of map tasks that will be run
+    simultaneously by a task tracker.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.tasktracker.reduce.tasks.maximum</name>
+    <value>2</value>
+    <description>The maximum number of reduce tasks that will be run
+    simultaneously by a task tracker.
+    </description>
+  </property>
+
+</configuration>

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/masters
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/masters?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/masters (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/masters Sun Jun  5 08:54:50 2011
@@ -0,0 +1 @@
+{{active_master}}

Added: incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/slaves
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/slaves?rev=1132176&view=auto
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/slaves (added)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/persistent-hdfs/conf/slaves Sun Jun  5 08:54:50 2011
@@ -0,0 +1 @@
+{{slave_list}}

Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave?rev=1132176&r1=1132175&r2=1132176&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave Sun Jun  5 08:54:50 2011
@@ -74,3 +74,5 @@ fi
 # Remove ~/.ssh/known_hosts because it gets polluted as you start/stop many
 # clusters (new machines tend to come up under old hostnames)
 rm -f /root/.ssh/known_hosts
+
+./approve-master-key

Modified: incubator/mesos/trunk/ec2/mesos_ec2.py
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/mesos_ec2.py?rev=1132176&r1=1132175&r2=1132176&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/mesos_ec2.py (original)
+++ incubator/mesos/trunk/ec2/mesos_ec2.py Sun Jun  5 08:54:50 2011
@@ -134,6 +134,8 @@ def launch_cluster(conn, opts, cluster_n
     master_group.authorize('tcp', 8080, 8081, '0.0.0.0/0')
     master_group.authorize('tcp', 50030, 50030, '0.0.0.0/0')
     master_group.authorize('tcp', 50070, 50070, '0.0.0.0/0')
+    master_group.authorize('tcp', 60070, 60070, '0.0.0.0/0')
+    master_group.authorize('tcp', 38090, 38090, '0.0.0.0/0')
   if slave_group.rules == []: # Group was just now created
     slave_group.authorize(src_group=master_group)
     slave_group.authorize(src_group=slave_group)
@@ -142,6 +144,8 @@ def launch_cluster(conn, opts, cluster_n
     slave_group.authorize('tcp', 8080, 8081, '0.0.0.0/0')
     slave_group.authorize('tcp', 50060, 50060, '0.0.0.0/0')
     slave_group.authorize('tcp', 50075, 50075, '0.0.0.0/0')
+    slave_group.authorize('tcp', 60060, 60060, '0.0.0.0/0')
+    slave_group.authorize('tcp', 60075, 60075, '0.0.0.0/0')
   if zoo_group.rules == []: # Group was just now created
     zoo_group.authorize(src_group=master_group)
     zoo_group.authorize(src_group=slave_group)
@@ -276,8 +280,8 @@ def wait_for_cluster(conn, master_res, s
   wait_for_instances(conn, slave_res)
   if zoo_res != None:
     wait_for_instances(conn, zoo_res)
-  print "Waiting 40 more seconds..."
-  time.sleep(40)
+  print "Waiting 60 more seconds..."
+  time.sleep(60)
 
 
 # Get number of local disks available for a given EC2 instance type.
@@ -299,11 +303,11 @@ def deploy_files(conn, root_dir, opts, m
   active_master = master_res.instances[0].public_dns_name
 
   num_disks = get_num_disks(opts.instance_type)
-  hdfs_data_dirs = "/mnt/hdfs/dfs/data"
+  hdfs_data_dirs = "/mnt/ephemeral-hdfs/data"
   mapred_local_dirs = "/mnt/hadoop/mrlocal"
   if num_disks > 1:
     for i in range(2, num_disks + 1):
-      hdfs_data_dirs += ",/mnt%d/hdfs/dfs/data" % i
+      hdfs_data_dirs += ",/mnt%d/ephemeral-hdfs/data" % i
       mapred_local_dirs += ",/mnt%d/hadoop/mrlocal" % i
 
   if zoo_res != None:



Mime
View raw message