hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r696615 - in /hadoop/core/trunk: ./ src/contrib/ec2/bin/ src/contrib/ec2/bin/image/
Date Thu, 18 Sep 2008 09:45:00 GMT
Author: tomwhite
Date: Thu Sep 18 02:44:59 2008
New Revision: 696615

URL: http://svn.apache.org/viewvc?rev=696615&view=rev
Log:
HADOOP-4117. Improve configurability of Hadoop EC2 instances.

Added:
    hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh
    hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data
Removed:
    hadoop/core/trunk/src/contrib/ec2/bin/image/hadoop-init
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image
    hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template
    hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote
    hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master
    hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Sep 18 02:44:59 2008
@@ -341,6 +341,9 @@
     HADOOP-4075. test-patch.sh now spits out ant commands that it runs.
     (Ramya R via nigel)
 
+    HADOOP-4117. Improve configurability of Hadoop EC2 instances.
+    (tomwhite)
+
   OPTIMIZATIONS
 
     HADOOP-3556. Removed lock contention in MD5Hash by changing the 

Modified: hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/create-hadoop-image Thu Sep 18 02:44:59 2008
@@ -63,7 +63,7 @@
 # Copy setup scripts
 scp $SSH_OPTS "$bin"/hadoop-ec2-env.sh "root@$HOSTNAME:/mnt"
 scp $SSH_OPTS "$bin"/image/create-hadoop-image-remote "root@$HOSTNAME:/mnt"
-scp $SSH_OPTS "$bin"/image/hadoop-init "root@$HOSTNAME:"
+scp $SSH_OPTS "$bin"/image/ec2-run-user-data "root@$HOSTNAME:/etc/init.d"
 
 # Copy private key and certificate (for bundling image)
 scp $SSH_OPTS $EC2_KEYDIR/pk*.pem "root@$HOSTNAME:/mnt"

Modified: hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template Thu Sep 18 02:44:59 2008
@@ -51,10 +51,8 @@
 # Enable public access to JobTracker and TaskTracker web interfaces
 ENABLE_WEB_PORTS=true
 
-# Boot parameters
-MAX_MAP_TASKS=2
-MAX_REDUCE_TASKS=2
-COMPRESS=true
+# The script to run on instance boot.
+USER_DATA_FILE=hadoop-ec2-init-remote.sh
 
 # The EC2 instance type: m1.small, m1.large, m1.xlarge
 INSTANCE_TYPE="m1.small"

Added: hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh?rev=696615&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh (added)
+++ hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-init-remote.sh Thu Sep 18 02:44:59 2008
@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+
+################################################################################
+# Script that is run on each EC2 instance on boot. It is passed in the EC2 user
+# data, so should not exceed 16K in size.
+################################################################################
+
+################################################################################
+# Initialize variables
+################################################################################
+
+# Slaves are started after the master, and are told its address by sending a
+# modified copy of this file which sets the MASTER_HOST variable. 
+# A node  knows if it is the master or not by inspecting the security group
+# name. If it is the master then it retrieves its address using instance data.
+MASTER_HOST=%MASTER_HOST% # Interpolated before being sent to EC2 node
+SECURITY_GROUPS=`wget -q -O - http://169.254.169.254/latest/meta-data/security-groups`
+IS_MASTER=`echo $SECURITY_GROUPS | awk '{ a = match ($0, "-master$"); if (a) print "true";
else print "false"; }'`
+if [ "$IS_MASTER" == "true" ]; then
+ MASTER_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/local-hostname`
+fi
+
+HADOOP_HOME=`ls -d /usr/local/hadoop-*`
+
+################################################################################
+# Hadoop configuration
+# Modify this section to customize your Hadoop cluster.
+################################################################################
+
+cat > $HADOOP_HOME/conf/hadoop-site.xml <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+<property>
+  <name>hadoop.tmp.dir</name>
+  <value>/mnt/hadoop</value>
+</property>
+
+<property>
+  <name>fs.default.name</name>
+  <value>hdfs://$MASTER_HOST:50001</value>
+</property>
+
+<property>
+  <name>mapred.job.tracker</name>
+  <value>hdfs://$MASTER_HOST:50002</value>
+</property>
+
+<property>
+  <name>tasktracker.http.threads</name>
+  <value>80</value>
+</property>
+
+<property>
+  <name>mapred.tasktracker.map.tasks.maximum</name>
+  <value>3</value>
+</property>
+
+<property>
+  <name>mapred.tasktracker.reduce.tasks.maximum</name>
+  <value>3</value>
+</property>
+
+<property>
+  <name>mapred.output.compress</name>
+  <value>true</value>
+</property>
+
+<property>
+  <name>mapred.output.compression.type</name>
+  <value>BLOCK</value>
+</property>
+
+<property>
+  <name>dfs.client.block.write.retries</name>
+  <value>3</value>
+</property>
+
+</configuration>
+EOF
+
+# Configure Hadoop for Ganglia
+# overwrite hadoop-metrics.properties
+cat > $HADOOP_HOME/conf/hadoop-metrics.properties <<EOF
+
+# Ganglia
+# we push to the master gmond so hostnames show up properly
+dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+dfs.period=10
+dfs.servers=$MASTER_HOST:8649
+
+mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+mapred.period=10
+mapred.servers=$MASTER_HOST:8649
+
+jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+jvm.period=10
+jvm.servers=$MASTER_HOST:8649
+EOF
+
+################################################################################
+# Start services
+################################################################################
+
+[ ! -f /etc/hosts ] &&  echo "127.0.0.1 localhost" > /etc/hosts
+
+mkdir -p /mnt/hadoop/logs
+
+# not set on boot
+export USER="root"
+
+if [ "$IS_MASTER" == "true" ]; then
+  # MASTER
+  # Prep Ganglia
+  sed -i -e "s|\( *mcast_join *=.*\)|#\1|" \
+         -e "s|\( *bind *=.*\)|#\1|" \
+         -e "s|\( *mute *=.*\)|  mute = yes|" \
+         -e "s|\( *location *=.*\)|  location = \"master-node\"|" \
+         /etc/gmond.conf
+  mkdir -p /mnt/ganglia/rrds
+  chown -R ganglia:ganglia /mnt/ganglia/rrds
+  rm -rf /var/lib/ganglia; cd /var/lib; ln -s /mnt/ganglia ganglia; cd
+  service gmond start
+  service gmetad start
+  apachectl start
+
+  # Hadoop
+  # only format on first boot
+  [ ! -e /mnt/hadoop/dfs ] && "$HADOOP_HOME"/bin/hadoop namenode -format
+
+  "$HADOOP_HOME"/bin/hadoop-daemon.sh start namenode
+  "$HADOOP_HOME"/bin/hadoop-daemon.sh start jobtracker
+else
+  # SLAVE
+  # Prep Ganglia
+  sed -i -e "s|\( *mcast_join *=.*\)|#\1|" \
+         -e "s|\( *bind *=.*\)|#\1|" \
+         -e "s|\(udp_send_channel {\)|\1\n  host=$MASTER_HOST|" \
+         /etc/gmond.conf
+  service gmond start
+
+  # Hadoop
+  "$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode
+  "$HADOOP_HOME"/bin/hadoop-daemon.sh start tasktracker
+fi
+
+# Run this script on next boot
+rm -f /var/ec2/ec2-run-user-data.*

Modified: hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/image/create-hadoop-image-remote Thu Sep 18 02:44:59
2008
@@ -53,8 +53,9 @@
        -e 's|# export HADOOP_OPTS=.*|export HADOOP_OPTS=-server|' \
       /usr/local/hadoop-$HADOOP_VERSION/conf/hadoop-env.sh
 
-# Do configuration on instance startup
-echo "/root/hadoop-init" >> /etc/rc.d/rc.local
+# Run user data as script on instance startup
+chmod +x /etc/init.d/ec2-run-user-data
+echo "/etc/init.d/ec2-run-user-data" >> /etc/rc.d/rc.local
 
 # Setup root user bash environment
 echo "export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}" >> /root/.bash_profile

Added: hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data?rev=696615&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data (added)
+++ hadoop/core/trunk/src/contrib/ec2/bin/image/ec2-run-user-data Thu Sep 18 02:44:59 2008
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# ec2-run-user-data - Run instance user-data if it looks like a script.
+#
+# Only retrieves and runs the user-data script once per instance.  If
+# you want the user-data script to run again (e.g., on the next boot)
+# then add this command in the user-data script:
+#   rm -f /var/ec2/ec2-run-user-data.*
+#
+# History:
+#   2008-05-16 Eric Hammond <ehammond@thinksome.com>
+#   - Initial version including code from Kim Scheibel, Jorge Oliveira
+#   2008-08-06 Tom White
+#   - Updated to use mktemp on fedora
+#
+
+prog=$(basename $0)
+logger="logger -t $prog"
+curl="curl --retry 3 --silent --show-error --fail"
+instance_data_url=http://169.254.169.254/2008-02-01
+
+# Wait until networking is up on the EC2 instance.
+perl -MIO::Socket::INET -e '
+ until(new IO::Socket::INET("169.254.169.254:80")){print"Waiting for network...\n";sleep
1}
+' | $logger
+
+# Exit if we have already run on this instance (e.g., previous boot).
+ami_id=$($curl $instance_data_url/meta-data/ami-id)
+been_run_file=/var/ec2/$prog.$ami_id
+mkdir -p $(dirname $been_run_file)
+if [ -f $been_run_file ]; then
+  $logger < $been_run_file
+  exit
+fi
+
+# Retrieve the instance user-data and run it if it looks like a script
+user_data_file=`mktemp -t ec2-user-data.XXXXXXXXXX`
+chmod 700 $user_data_file
+$logger "Retrieving user-data"
+$curl -o $user_data_file $instance_data_url/user-data 2>&1 | $logger
+if [ ! -s $user_data_file ]; then
+  $logger "No user-data available"
+elif head -1 $user_data_file | egrep -v '^#!'; then
+  $logger "Skipping user-data as it does not begin with #!"
+else
+  $logger "Running user-data"
+  echo "user-data has already been run on this instance" > $been_run_file
+  $user_data_file 2>&1 | logger -t "user-data"
+  $logger "user-data exit code: $?"
+fi
+rm -f $user_data_file

Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master Thu Sep 18 02:44:59 2008
@@ -81,7 +81,7 @@
 # Start a master
 echo "Starting master with AMI $AMI_IMAGE"
 USER_DATA="MASTER_HOST=master,MAX_MAP_TASKS=$MAX_MAP_TASKS,MAX_REDUCE_TASKS=$MAX_REDUCE_TASKS,COMPRESS=$COMPRESS"
-INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -d "$USER_DATA"
-t $INSTANCE_TYPE | grep INSTANCE | awk '{print $2}'`
+INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -f "$bin"/$USER_DATA_FILE
-t $INSTANCE_TYPE | grep INSTANCE | awk '{print $2}'`
 echo "Waiting for instance $INSTANCE to start"
 while true; do
   printf "."

Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves?rev=696615&r1=696614&r2=696615&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves Thu Sep 18 02:44:59 2008
@@ -44,9 +44,11 @@
 MASTER_HOST=`cat $MASTER_PRIVATE_IP_PATH`
 MASTER_ZONE=`cat $MASTER_ZONE_PATH`
 
+# Substituting master hostname
+sed -e "s|%MASTER_HOST%|$MASTER_HOST|" "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.slave
+
 # Start slaves
 echo "Adding $1 node(s) to cluster group $CLUSTER with AMI $AMI_IMAGE"
-USER_DATA="MASTER_HOST=$MASTER_HOST,MAX_MAP_TASKS=$MAX_MAP_TASKS,MAX_REDUCE_TASKS=$MAX_REDUCE_TASKS,COMPRESS=$COMPRESS"
-ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME" -d "$USER_DATA"
-t "$INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
-
+ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME" -f "$bin"/$USER_DATA_FILE.slave
-t "$INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
 
+rm "$bin"/$USER_DATA_FILE.slave



Mime
View raw message