hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r696626 - in /hadoop/core/trunk: ./ src/contrib/ec2/bin/
Date Thu, 18 Sep 2008 10:55:33 GMT
Author: tomwhite
Date: Thu Sep 18 03:55:32 2008
New Revision: 696626

URL: http://svn.apache.org/viewvc?rev=696626&view=rev
Log:
HADOOP-2411. Add support for larger CPU EC2 instance types. Contributed by Chris K Wensel.

Removed:
    hadoop/core/trunk/src/contrib/ec2/bin/destroy-hadoop-cluster
    hadoop/core/trunk/src/contrib/ec2/bin/login-hadoop-cluster
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2
    hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template
    hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster
    hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master
    hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves
    hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Sep 18 03:55:32 2008
@@ -344,6 +344,9 @@
     HADOOP-4117. Improve configurability of Hadoop EC2 instances.
     (tomwhite)
 
+    HADOOP-2411. Add support for larger CPU EC2 instance types.
+    (Chris K Wensel via tomwhite)
+
   OPTIMIZATIONS
 
     HADOOP-3556. Removed lock contention in MD5Hash by changing the 

Modified: hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2 (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2 Thu Sep 18 03:55:32 2008
@@ -26,7 +26,7 @@
   echo "  launch-cluster <group> <num slaves>  launch a cluster of Hadoop EC2
instances - launch-master then launch-slaves"
   echo "  launch-master  <group>               launch or find a cluster master"
   echo "  launch-slaves  <group> <num slaves>  launch the cluster slaves"
-  echo "  terminate-cluster                    terminate all Hadoop EC2 instances"
+  echo "  terminate-cluster  <group>           terminate all Hadoop EC2 instances"
   echo "  login  <group|instance id>           login to the master node of the Hadoop
EC2 cluster"
   echo "  screen <group|instance id>           start or attach 'screen' on the master
node of the Hadoop EC2 cluster"
   echo "  proxy  <group|instance id>           start a socks proxy on localhost:6666
(use w/foxyproxy)"

Modified: hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template Thu Sep 18 03:55:32 2008
@@ -40,13 +40,13 @@
 SSH_OPTS=`echo -i "$PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o ServerAliveInterval=30`
 
 # The version of Hadoop to use.
-HADOOP_VERSION=0.17.0
+HADOOP_VERSION=0.19.0
 
 # The Amazon S3 bucket where the Hadoop AMI is stored.
 # The default value is for public images, so can be left if you are using running a public
image.
 # Change this value only if you are creating your own (private) AMI
 # so you can store it in a bucket you own.
-S3_BUCKET=hadoop-ec2-images
+S3_BUCKET=hadoop-images
 
 # Enable public access to JobTracker and TaskTracker web interfaces
 ENABLE_WEB_PORTS=true
@@ -58,6 +58,8 @@
 INSTANCE_TYPE="m1.small"
 #INSTANCE_TYPE="m1.large"
 #INSTANCE_TYPE="m1.xlarge"
+#INSTANCE_TYPE="c1.medium"
+#INSTANCE_TYPE="c1.xlarge"
 
 # The EC2 group master name. CLUSTER is set by calling scripts
 CLUSTER_MASTER=$CLUSTER-master
@@ -72,16 +74,28 @@
 #
 
 # The version number of the installed JDK.
-JAVA_VERSION=1.6.0_05
+JAVA_VERSION=1.6.0_07
 
 # SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
 # The download URL for the Sun JDK. Visit http://java.sun.com/javase/downloads/index.jsp
and get the URL for the "Linux self-extracting file".
-if [ "$INSTANCE_TYPE" == "m1.small" ]; then
+if [ "$INSTANCE_TYPE" == "m1.small" -o "$INSTANCE_TYPE" == "c1.medium" ]; then
   ARCH='i386'
-  BASE_AMI_IMAGE="ami-f51aff9c"  # ec2-public-images/fedora-8-i386-base-v1.06.manifest.xml
-  JAVA_BINARY_URL=
+  BASE_AMI_IMAGE="ami-2b5fba42"  # ec2-public-images/fedora-8-i386-base-v1.07.manifest.xml
+  JAVA_BINARY_URL=''
 else
   ARCH='x86_64'
-  BASE_AMI_IMAGE="ami-f21aff9b"  # ec2-public-images/fedora-8-x86_64-base-v1.06.manifest.xml
-  JAVA_BINARY_URL=
+  BASE_AMI_IMAGE="ami-2a5fba43"  # ec2-public-images/fedora-8-x86_64-base-v1.07.manifest.xml
+  JAVA_BINARY_URL=''
+fi
+
+if [ "$INSTANCE_TYPE" == "c1.medium" ]; then
+  AMI_KERNEL=aki-9b00e5f2 # ec2-public-images/vmlinuz-2.6.18-xenU-ec2-v1.0.i386.aki.manifest.xml
+fi
+
+if [ "$INSTANCE_TYPE" == "c1.xlarge" ]; then
+  AMI_KERNEL=aki-9800e5f1 # ec2-public-images/vmlinuz-2.6.18-xenU-ec2-v1.0.x86_64.aki.manifest.xml
+fi
+
+if [ "$AMI_KERNEL" != "" ]; then
+  KERNEL_ARG="--kernel ${AMI_KERNEL}"
 fi

Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster Thu Sep 18 03:55:32 2008
@@ -28,6 +28,7 @@
 
 if [ -z $2 ]; then
   echo "Must specify the number of slaves to start."
+  exit -1
 fi
 
 if ! "$bin"/launch-hadoop-master $1 ; then

Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master Thu Sep 18 03:55:32 2008
@@ -81,7 +81,7 @@
 # Start a master
 echo "Starting master with AMI $AMI_IMAGE"
 USER_DATA="MASTER_HOST=master,MAX_MAP_TASKS=$MAX_MAP_TASKS,MAX_REDUCE_TASKS=$MAX_REDUCE_TASKS,COMPRESS=$COMPRESS"
-INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -f "$bin"/$USER_DATA_FILE
-t $INSTANCE_TYPE | grep INSTANCE | awk '{print $2}'`
+INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -f "$bin"/$USER_DATA_FILE
-t $INSTANCE_TYPE $KERNEL_ARG | grep INSTANCE | awk '{print $2}'`
 echo "Waiting for instance $INSTANCE to start"
 while true; do
   printf "."

Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves Thu Sep 18 03:55:32 2008
@@ -24,6 +24,7 @@
 
 if [ -z $2 ]; then
   echo "Must specify the number of slaves to start."
+  exit -1
 fi
 
 CLUSTER=$1
@@ -49,6 +50,6 @@
 
 # Start slaves
 echo "Adding $1 node(s) to cluster group $CLUSTER with AMI $AMI_IMAGE"
-ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME" -f "$bin"/$USER_DATA_FILE.slave
-t "$INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
+ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME" -f "$bin"/$USER_DATA_FILE.slave
-t "$INSTANCE_TYPE" -z "$MASTER_ZONE" $KERNEL_ARG | grep INSTANCE | awk '{print $2}'
 
 rm "$bin"/$USER_DATA_FILE.slave

Modified: hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters Thu Sep 18 03:55:32 2008
@@ -23,7 +23,7 @@
 . "$bin"/hadoop-ec2-env.sh
 
 # Finding Hadoop clusters
-CLUSTERS=`ec2-describe-instances | awk '"RESERVATION" == $1 {print $4}' | grep -v -e "-master$"
| sort | uniq`
+CLUSTERS=`ec2-describe-instances | awk '"RESERVATION" == $1 && $4 ~ /-master$/, "INSTANCE"
== $1' | tr '\n' '\t' | grep running | cut -f4 | rev | cut -d'-' -f2- | rev`
 
 [ -z "$CLUSTERS" ] && echo "No running clusters." && exit 0
 



Mime
View raw message