hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r884448 - in /hadoop/hbase/trunk: ./ src/contrib/ec2/bin/ src/contrib/ec2/bin/image/
Date Thu, 26 Nov 2009 08:00:03 GMT
Author: apurtell
Date: Thu Nov 26 08:00:02 2009
New Revision: 884448

URL: http://svn.apache.org/viewvc?rev=884448&view=rev
Log:
HBASE-2012 [EC2] LZO support

Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/src/contrib/ec2/bin/create-hbase-image
    hadoop/hbase/trunk/src/contrib/ec2/bin/hbase-ec2-env.sh
    hadoop/hbase/trunk/src/contrib/ec2/bin/image/create-hbase-image-remote
    hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-master
    hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-slaves
    hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-zookeeper

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=884448&r1=884447&r2=884448&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Thu Nov 26 08:00:02 2009
@@ -206,6 +206,7 @@
    HBASE-1958  Remove "# TODO: PUT BACK !!! "${HADOOP_HOME}"/bin/hadoop
                dfsadmin -safemode wait"
    HBASE-2009  [EC2] Support mapreduce
+   HBASE-2012  [EC2] LZO support
 
   NEW FEATURES
    HBASE-1901  "General" partitioner for "hbase-48" bulk (behind the api, write

Modified: hadoop/hbase/trunk/src/contrib/ec2/bin/create-hbase-image
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/contrib/ec2/bin/create-hbase-image?rev=884448&r1=884447&r2=884448&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/contrib/ec2/bin/create-hbase-image (original)
+++ hadoop/hbase/trunk/src/contrib/ec2/bin/create-hbase-image Thu Nov 26 08:00:02 2009
@@ -18,23 +18,26 @@
 # Create a HBase AMI.
 # Inspired by Jonathan Siegel's EC2 script (http://blogsiegel.blogspot.com/2006/08/sandboxing-amazon-ec2.html)
 
-# allow override of INSTANCE_TYPE from the command line 
-[ ! -z $1 ] && INSTANCE_TYPE=$1
+# allow override of SLAVE_INSTANCE_TYPE from the command line 
+[ ! -z $1 ] && SLAVE_INSTANCE_TYPE=$1
 
 # Import variables
 bin=`dirname "$0"`
 bin=`cd "$bin"; pwd`
 . "$bin"/hbase-ec2-env.sh
 
-echo "INSTANCE_TYPE is $INSTANCE_TYPE."
-echo "ARCH is $ARCH."
+type=$SLAVE_INSTANCE_TYPE
+arch=$SLAVE_ARCH
 
-AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION
| grep $ARCH | grep available | awk '{print $2}'`
+echo "INSTANCE_TYPE is $type"
+echo "ARCH is $arch"
+
+AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION
| grep $arch | grep available | awk '{print $2}'`
 
 [ ! -z $AMI_IMAGE ] && echo "AMI already registered, use: ec2-deregister $AMI_IMAGE"
&& exit 1
 
 echo "Starting a AMI with ID $BASE_AMI_IMAGE."
-OUTPUT=`ec2-run-instances $BASE_AMI_IMAGE $TOOL_OPTS -k root -t $INSTANCE_TYPE`
+OUTPUT=`ec2-run-instances $BASE_AMI_IMAGE $TOOL_OPTS -k root -t $type`
 BOOTING_INSTANCE=`echo $OUTPUT | awk '{print $6}'`
 
 echo "Instance is $BOOTING_INSTANCE."
@@ -69,9 +72,9 @@
 scp $SSH_OPTS $EC2_CERT "root@$HOSTNAME:/mnt"
 
 # Connect to it
-ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$INSTANCE_TYPE /mnt/create-hbase-image-remote\""
+ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$type ARCH=$arch /mnt/create-hbase-image-remote\""
 
 # Register image
-ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$ARCH.manifest.xml
+ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$arch.manifest.xml
 
 echo "Terminate with: ec2-terminate-instances $BOOTING_INSTANCE"

Modified: hadoop/hbase/trunk/src/contrib/ec2/bin/hbase-ec2-env.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/contrib/ec2/bin/hbase-ec2-env.sh?rev=884448&r1=884447&r2=884448&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/contrib/ec2/bin/hbase-ec2-env.sh (original)
+++ hadoop/hbase/trunk/src/contrib/ec2/bin/hbase-ec2-env.sh Thu Nov 26 08:00:02 2009
@@ -45,10 +45,10 @@
 TOOL_OPTS=`echo -K "$EC2_PRIVATE_KEY" -C "$EC2_CERT"`
 
 # The version of HBase to use.
-HBASE_VERSION=0.20.1
+HBASE_VERSION=0.20.2
 
 # The version of Hadoop to use.
-HADOOP_VERSION=$HBASE_VERSION
+HADOOP_VERSION=0.20.1
 
 # The Amazon S3 bucket where the HBase AMI is stored.
 # The default value is for public images, so can be left if you are using running a public
image.
@@ -64,7 +64,10 @@
 USER_DATA_FILE=hbase-ec2-init-remote.sh
 
 # Use only c1.xlarge unless you know what you are doing
-INSTANCE_TYPE=${INSTANCE_TYPE:-c1.xlarge}
+MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge}
+
+# Use only c1.xlarge unless you know what you are doing
+SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-c1.xlarge}
 
 # Use only c1.medium unless you know what you are doing
 ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-c1.medium}
@@ -89,17 +92,20 @@
 JAVA_VERSION=1.6.0_16
 
 # SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
-# The download URL for the Sun JDK. Visit http://java.sun.com/javase/downloads/index.jsp
and get the URL for the "Linux self-extracting file".
-if [ "$INSTANCE_TYPE" = "m1.small" -o "$INSTANCE_TYPE" = "c1.medium" ]; then
-  ARCH='i386'
+if [ "$SLAVE_INSTANCE_TYPE" = "m1.small" -o "$SLAVE_INSTANCE_TYPE" = "c1.medium" ]; then
+  SLAVE_ARCH='i386'
   BASE_AMI_IMAGE="ami-48aa4921"  # ec2-public-images/fedora-8-i386-base-v1.10.manifest.xml
-  AMI_IMAGE="ami-c644a7af"
-  JAVA_BINARY_URL='http://iridiant.s3.amazonaws.com/jdk/jdk-6u16-linux-i586.bin'
 else
-  ARCH='x86_64'
+  SLAVE_ARCH='x86_64'
   BASE_AMI_IMAGE="ami-f61dfd9f"  # ec2-public-images/fedora-8-x86_64-base-v1.10.manifest.xml
-  AMI_IMAGE="ami-f244a79b"
-  JAVA_BINARY_URL='http://iridiant.s3.amazonaws.com/jdk/jdk-6u16-linux-x64.bin'
+fi
+
+if [ "$MASTER_INSTANCE_TYPE" = "m1.small" -o "$MASTER_INSTANCE_TYPE" = "c1.medium" ]; then
+  MASTER_ARCH='i386'
+  MASTER_AMI_IMAGE="ami-c644a7af"
+else
+  MASTER_ARCH='x86_64'
+  MASTER_AMI_IMAGE="ami-f244a79b"
 fi
 
 if [ "$ZOO_INSTANCE_TYPE" = "m1.small" -o "$ZOO_INSTANCE_TYPE" = "c1.medium" ]; then

Modified: hadoop/hbase/trunk/src/contrib/ec2/bin/image/create-hbase-image-remote
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/contrib/ec2/bin/image/create-hbase-image-remote?rev=884448&r1=884447&r2=884448&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/contrib/ec2/bin/image/create-hbase-image-remote (original)
+++ hadoop/hbase/trunk/src/contrib/ec2/bin/image/create-hbase-image-remote Thu Nov 26 08:00:02
2009
@@ -15,7 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 # Create a Hbase AMI. Runs on the EC2 instance.
 
 # Import variables
@@ -23,8 +22,13 @@
 bin=`cd "$bin"; pwd`
 . "$bin"/hbase-ec2-env.sh
 
-echo "Remote: INSTANCE_TYPE is $INSTANCE_TYPE."
-echo "Remote: ARCH is $ARCH."
+type=$INSTANCE_TYPE
+[ -z "$type" ] && type=$SLAVE_INSTANCE_TYPE
+arch=$ARCH
+[ -z "$arch" ] && arch=$SLAVE_ARCH
+
+echo "Remote: INSTANCE_TYPE is $type"
+echo "Remote: ARCH is $arch"
 
 # Remove environment script since it contains sensitive information
 rm -f "$bin"/hbase-ec2-env.sh
@@ -32,14 +36,14 @@
 # Install Java
 echo "Downloading and installing java binary."
 cd /usr/local
-wget -nv -O java.bin $JAVA_BINARY_URL
+wget -nv -O java.bin http://iridiant.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-${arch}.bin
 sh java.bin
 rm -f java.bin
 
 # Install tools
 echo "Installing rpms."
 yum -y update
-yum -y install rsync lynx screen ganglia-gmetad ganglia-gmond ganglia-web httpd php
+yum -y install rsync lynx screen ganglia-gmetad ganglia-gmond ganglia-web httpd php lzo-devel
 yum -y clean all
 
 # Install Hadoop
@@ -89,13 +93,20 @@
 # Ensure logging in to new hosts is seamless.
 echo '    StrictHostKeyChecking no' >> /etc/ssh/ssh_config
 
+# Install LZO for HBase 
+echo "Installing LZO codec support for HBase"
+cd /usr/local/hbase-${HBASE_VERSION}
+wget -nv http://iridiant.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz
+tar xzf /var/tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
+rm /var/tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
+
 # Bundle and upload image
 cd ~root
 # Don't need to delete .bash_history since it isn't written until exit.
 df -h
-ec2-bundle-vol -d /mnt -k /mnt/pk*.pem -c /mnt/cert*.pem -u $AWS_ACCOUNT_ID -s 3072 -p hbase-$HBASE_VERSION-$ARCH
-r $ARCH
+ec2-bundle-vol -d /mnt -k /mnt/pk*.pem -c /mnt/cert*.pem -u $AWS_ACCOUNT_ID -s 3072 -p hbase-$HBASE_VERSION-$arch
-r $arch
 
-ec2-upload-bundle -b $S3_BUCKET -m /mnt/hbase-$HBASE_VERSION-$ARCH.manifest.xml -a $AWS_ACCESS_KEY_ID
-s $AWS_SECRET_ACCESS_KEY
+ec2-upload-bundle -b $S3_BUCKET -m /mnt/hbase-$HBASE_VERSION-$arch.manifest.xml -a $AWS_ACCESS_KEY_ID
-s $AWS_SECRET_ACCESS_KEY
 
 # End
 echo Done

Modified: hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-master
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-master?rev=884448&r1=884447&r2=884448&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-master (original)
+++ hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-master Thu Nov 26 08:00:02 2009
@@ -29,6 +29,11 @@
 bin=`cd "$bin"; pwd`
 . "$bin"/hbase-ec2-env.sh
 
+type=$MASTER_INSTANCE_TYPE
+[ -z "$type" ] && type=$SLAVE_INSTANCE_TYPE
+arch=$MASTER_ARCH
+[ -z "$arch" ] && arch=$SLAVE_ARCH
+
 if [ -z $AWS_ACCOUNT_ID ]; then
   echo "Please set AWS_ACCOUNT_ID in $bin/hbase-ec2-env.sh."
   exit 1
@@ -47,15 +52,15 @@
 fi
 
 # Finding HBase image
-[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET
| grep $HBASE_VERSION | grep $ARCH | grep available | awk '{print $2}'`
+[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET
| grep $HBASE_VERSION | grep $arch | grep available | awk '{print $2}'`
 
 # Start a master
-echo "Starting master with AMI $AMI_IMAGE (arch $ARCH)"
+echo "Starting master with AMI $AMI_IMAGE (arch $arch)"
 # Substituting zookeeper quorum
 ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH`
 sed -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \
     "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master
-INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master
-t $INSTANCE_TYPE | grep INSTANCE | awk '{print $2}'`
+INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master
-t $type | grep INSTANCE | awk '{print $2}'`
 echo -n "Waiting for instance $INSTANCE to start"
 while true; do
   printf "."

Modified: hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-slaves
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-slaves?rev=884448&r1=884447&r2=884448&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-slaves (original)
+++ hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-slaves Thu Nov 26 08:00:02 2009
@@ -41,7 +41,7 @@
   exit 1
 fi
 
-[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET
| grep $HBASE_VERSION | grep $ARCH |grep available | awk '{print $2}'`
+[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET
| grep $HBASE_VERSION | grep $SLAVE_ARCH |grep available | awk '{print $2}'`
 
 MASTER_HOST=`cat $MASTER_PRIVATE_IP_PATH`
 MASTER_ZONE=`cat $MASTER_ZONE_PATH`
@@ -53,7 +53,7 @@
     "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.slave
 
 # Start slaves
-echo "Starting $NO_INSTANCES AMI(s) with ID $AMI_IMAGE (arch $ARCH) in group $CLUSTER in
zone $MASTER_ZONE"
-ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NO_INSTANCES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave
-t "$INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
+echo "Starting $NO_INSTANCES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER
in zone $MASTER_ZONE"
+ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NO_INSTANCES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave
-t "$SLAVE_INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
 
 rm "$bin"/$USER_DATA_FILE.slave

Modified: hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-zookeeper
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-zookeeper?rev=884448&r1=884447&r2=884448&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-zookeeper (original)
+++ hadoop/hbase/trunk/src/contrib/ec2/bin/launch-hbase-zookeeper Thu Nov 26 08:00:02 2009
@@ -36,9 +36,9 @@
 . "$bin"/hbase-ec2-env.sh
 
 type=$ZOO_INSTANCE_TYPE
-[ -z "$type" ] && type=$INSTANCE_TYPE
+[ -z "$type" ] && type=$SLAVE_INSTANCE_TYPE
 arch=$ZOO_ARCH
-[ -z "$arch" ] && arch=$ARCH
+[ -z "$arch" ] && arch=$SLAVE_ARCH
 
 # Finding HBase image
 [ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep
$S3_BUCKET | grep hbase | grep $HBASE_VERSION | grep $arch |grep available | awk '{print $2}'`



Mime
View raw message