hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r892398 - in /hadoop/hbase/branches/0.20/src/contrib/ec2/bin: hbase-ec2-env.sh hbase-ec2-init-remote.sh image/create-hbase-image-remote launch-hbase-master launch-hbase-slaves
Date Fri, 18 Dec 2009 23:27:46 GMT
Author: apurtell
Date: Fri Dec 18 23:27:45 2009
New Revision: 892398

URL: http://svn.apache.org/viewvc?rev=892398&view=rev
Log:
HBASE-2032 Support for installation of user packages

Modified:
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh?rev=892398&r1=892397&r2=892398&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh Fri Dec 18 23:27:45 2009
@@ -48,6 +48,12 @@
 # Enable public access web interfaces
 ENABLE_WEB_PORTS=false
 
+# Extra packages
+# Allows you to add a private Yum repo and pull packages from it as your
+# instances boot up. Format is <repo-descriptor-URL> <pkg1> ... <pkgN>
+# The repository descriptor will be fetched into /etc/yum/repos.d.
+EXTRA_PACKAGES="http://iridiant.s3.amazonaws.com/yum/iridiant.repo heritrix"
+
 # Use only c1.xlarge unless you know what you are doing
 MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge}
 

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh?rev=892398&r1=892397&r2=892398&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh Fri Dec 18 23:27:45
2009
@@ -1,12 +1,11 @@
 #!/usr/bin/env bash
 
-###############################################################################
 # Script that is run on each EC2 instance on boot. It is passed in the EC2 user
 # data, so should not exceed 16K in size.
-###############################################################################
 
-MASTER_HOST=%MASTER_HOST%
-ZOOKEEPER_QUORUM=%ZOOKEEPER_QUORUM%
+MASTER_HOST="%MASTER_HOST%"
+ZOOKEEPER_QUORUM="%ZOOKEEPER_QUORUM%"
+EXTRA_PACKAGES="%EXTRA_PACKAGES%"
 SECURITY_GROUPS=`wget -q -O - http://169.254.169.254/latest/meta-data/security-groups`
 IS_MASTER=`echo $SECURITY_GROUPS | awk '{ a = match ($0, "-master$"); if (a) print "true";
else print "false"; }'`
 if [ "$IS_MASTER" = "true" ]; then
@@ -17,9 +16,7 @@
 HBASE_HOME=`ls -d /usr/local/hbase-*`
 HBASE_VERSION=`echo $HBASE_HOME | cut -d '-' -f 2`
 
-###############################################################################
 # Hadoop configuration
-###############################################################################
 
 cat > $HADOOP_HOME/conf/core-site.xml <<EOF
 <?xml version="1.0"?>
@@ -76,7 +73,6 @@
 EOF
 
 # Configure Hadoop for Ganglia
-# overwrite hadoop-metrics.properties
 cat > $HADOOP_HOME/conf/hadoop-metrics.properties <<EOF
 dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
 dfs.period=10
@@ -89,9 +85,7 @@
 mapred.servers=$MASTER_HOST:8649
 EOF
 
-###############################################################################
 # HBase configuration
-###############################################################################
 
 cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
 <?xml version="1.0"?>
@@ -147,7 +141,6 @@
 EOF
 
 # Configure HBase for Ganglia
-# overwrite hadoop-metrics.properties
 cat > $HBASE_HOME/conf/hadoop-metrics.properties <<EOF
 dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
 dfs.period=10
@@ -160,26 +153,28 @@
 jvm.servers=$MASTER_HOST:8649
 EOF
 
-###############################################################################
-# Start services
-###############################################################################
-
 # up open file descriptor limits
 echo "root soft nofile 32768" >> /etc/security/limits.conf
 echo "root hard nofile 32768" >> /etc/security/limits.conf
 
-# up epoll limits
-# ok if this fails, only valid for kernels 2.6.27+
+# up epoll limits; ok if this fails, only valid for kernels 2.6.27+
 sysctl -w fs.epoll.max_user_instances=32768 > /dev/null 2>&1
 
-mkdir -p /mnt/hadoop/logs
-mkdir -p /mnt/hbase/logs
+mkdir -p /mnt/hadoop/logs /mnt/hbase/logs
 
 [ ! -f /etc/hosts ] &&  echo "127.0.0.1 localhost" > /etc/hosts
 
-# not set on boot
 export USER="root"
 
+if [ "$EXTRA_PACKAGES" != "" ] ; then
+  # format should be <repo-descriptor-URL> <package1> ... <packageN>
+    # this will only work with bash
+  pkg=( $EXTRA_PACKAGES )
+  wget -nv -O /etc/yum.repos.d/user.repo ${pkg[0]}
+  yum -y update yum
+  yum -y install ${pkg[@]:1}
+fi
+
 if [ "$IS_MASTER" = "true" ]; then
   # MASTER
   # Prep Ganglia
@@ -199,13 +194,9 @@
   [ ! -e /mnt/hadoop/dfs ] && "$HADOOP_HOME"/bin/hadoop namenode -format
 
   "$HADOOP_HOME"/bin/hadoop-daemon.sh start namenode
-
   "$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode
-
   "$HADOOP_HOME"/bin/hadoop-daemon.sh start jobtracker
-
   sleep 10
-
   "$HBASE_HOME"/bin/hbase-daemon.sh start master
 
 else
@@ -218,11 +209,8 @@
          -e "s|\(udp_send_channel {\)|\1\n  host=$MASTER_HOST|" \
          /etc/gmond.conf
   service gmond start
-
   "$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode
-
   "$HBASE_HOME"/bin/hbase-daemon.sh start regionserver
-
   "$HADOOP_HOME"/bin/hadoop-daemon.sh start tasktracker
 
 fi

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote?rev=892398&r1=892397&r2=892398&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote Fri Dec
18 23:27:45 2009
@@ -73,7 +73,7 @@
 # Configure HBase
 sed -i \
   -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|" \
-  -e 's|# export HBASE_OPTS=.*|export HBASE_OPTS="$HBASE_OPTS -server -XX:+UseConcMarkSweepGC
-XX:+DoEscapeAnalysis -XX:+AggressiveOpts"|' \
+  -e 's|# export HBASE_OPTS=.*|export HBASE_OPTS="$HBASE_OPTS -server -XX:+HeapDumpOnOutOfMemoryError
-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=90 -XX:NewSize=64m -XX:MaxNewSize=64m
-XX:+DoEscapeAnalysis -XX:+AggressiveOpts"|' \
   -e 's|# export HBASE_LOG_DIR=.*|export HBASE_LOG_DIR=/mnt/hbase/logs|' \
   -e 's|# export HBASE_SLAVE_SLEEP=.*|export HBASE_SLAVE_SLEEP=1|' \
   /usr/local/hbase-$HBASE_VERSION/conf/hbase-env.sh
@@ -83,7 +83,8 @@
 echo "/etc/init.d/ec2-run-user-data" >> /etc/rc.d/rc.local
 
 # Setup root user bash environment
-echo "export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}" >> /root/.bash_profile
+
+  echo "export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}" >> /root/.bash_profile
 echo "export HADOOP_HOME=/usr/local/hadoop-${HADOOP_VERSION}" >> /root/.bash_profile
 echo "export HBASE_HOME=/usr/local/hbase-${HBASE_VERSION}" >> /root/.bash_profile
 echo 'export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$HBASE_HOME/bin:$PATH' >> /root/.bash_profile
@@ -94,12 +95,12 @@
 # Ensure logging in to new hosts is seamless.
 echo '    StrictHostKeyChecking no' >> /etc/ssh/ssh_config
 
-# Install LZO for HBase 
-echo "Installing LZO codec support for HBase"
-cd /usr/local/hbase-${HBASE_VERSION}
-wget -nv http://iridiant.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz
-tar xzf lzo-linux-${HADOOP_VERSION}.tar.gz
-rm lzo-linux-${HADOOP_VERSION}.tar.gz
+# Install LZO
+echo "Installing LZO codec support"
+wget -nv -O /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz http://iridiant.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz
+cd /usr/local/hadoop-${HADOOP_VERSION} && tar xzf /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
+cd /usr/local/hbase-${HBASE_VERSION} && tar xzf /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
+rm -f /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
 
 # Bundle and upload image
 cd ~root

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master?rev=892398&r1=892397&r2=892398&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master Fri Dec 18 23:27:45
2009
@@ -59,6 +59,7 @@
 # Substituting zookeeper quorum
 ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH`
 sed -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \
+    -e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \
     "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master
 INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master
-t $type | grep INSTANCE | awk '{print $2}'`
 echo -n "Waiting for instance $INSTANCE to start"

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves?rev=892398&r1=892397&r2=892398&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves Fri Dec 18 23:27:45
2009
@@ -50,6 +50,7 @@
 # Substituting master hostname and zookeeper quorum
 sed -e "s|%MASTER_HOST%|$MASTER_HOST|" \
     -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \
+    -e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \
     "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.slave
 
 # Start slaves



Mime
View raw message