incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1326670 [3/47] - in /incubator/bigtop/trunk: ./ bigtop-deploy/puppet/manifests/ bigtop-deploy/puppet/modules/hadoop-hbase/manifests/ bigtop-deploy/puppet/modules/hadoop-hbase/templates/ bigtop-deploy/puppet/modules/hadoop-oozie/manifests/ ...
Date Mon, 16 Apr 2012 16:10:32 GMT
Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc Mon Apr 16 16:10:22 2012
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+HADOOP_NAME="datanode"
+DAEMON="hadoop-hdfs-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop/sbin/hadoop-daemon.sh"
+SVC_USER="hdfs"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/hadoop/conf"
+PIDFILE="/var/run/hadoop-hdfs/hadoop-$SVC_USER-$HADOOP_NAME.pid"
+LOCKFILE="/var/lock/subsys/hadoop-$HADOOP_NAME"
+
+CHKCONFIG="2345 85 15"
+INIT_DEFAULT_START="3 4 5"
+INIT_DEFAULT_STOP="0 1 2 6"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+  [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+  [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED
+  log_success_msg "Starting ${DESC}: "
+
+  TARGET_USER_NAME="HADOOP_`echo datanode | tr a-z A-Z`_USER"
+  TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
+
+  # The following needs to be removed once HDFS-1943 gets finally put to rest.
+  # The logic of this ugly hack is this: IFF we do NOT have jsvc installed it is
+  # guaranteed that we can NOT be running in a secure mode and thus we need to
+  # workaround HDFS-1943 (start as non-root). As soon as jsvc gets installed
+  # we are assuming a secure installation and starting a data node as root.
+  # This leaves 2 corner cases:
+  #    1. HADOOP_DATANODE_USER being set to root
+  #    2. jsvc is installed but Hadoop is configures to run in an unsecure mode
+  # Both will currently fail
+  if [ -n "$HADOOP_SECURE_DN_USER" ]; then
+    TARGET_USER=root
+  fi
+
+  export HADOOP_IDENT_STRING=$TARGET_USER
+  su -s /bin/bash $TARGET_USER -c "$EXEC_PATH --config '$CONF_DIR' start $DAEMON_FLAGS"
+
+  # Some processes are slow to start
+  sleep $SLEEP_TIME
+  checkstatusofproc
+  RETVAL=$?
+
+  [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE
+  return $RETVAL
+}
+__EOT__
+
+}

Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-namenode.svc
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-namenode.svc?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-namenode.svc (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-namenode.svc Mon Apr 16 16:10:22 2012
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+HADOOP_NAME="namenode"
+DAEMON="hadoop-hdfs-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop/sbin/hadoop-daemon.sh"
+SVC_USER="hdfs"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/hadoop/conf"
+PIDFILE="/var/run/hadoop-hdfs/hadoop-$SVC_USER-$HADOOP_NAME.pid"
+LOCKFILE="/var/lock/subsys/hadoop-$HADOOP_NAME"
+
+CHKCONFIG="2345 85 15"
+INIT_DEFAULT_START="3 4 5"
+INIT_DEFAULT_STOP="0 1 2 6"
+
+generate_extra_commands() {
+
+cat <<'__EOT__'
+    upgrade|rollback)
+      DAEMON_FLAGS="$DAEMON_FLAGS -${1}"
+      start
+      ;;
+    init)
+      check_for_root
+      checkstatusofproc
+      if [ "$?" = "$STATUS_RUNNING" ] ; then
+        echo "Error: $DESC is running. Stop it first."
+        exit 1
+      else
+        su -s /bin/bash $SVC_USER -c "hdfs namenode -format"
+      fi
+      ;;
+    *)
+      echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart|upgrade|rollback|init}"
+      exit 1
+__EOT__
+
+}

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-secondarynamenode.svc (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-secondarynamenode.svc?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-secondarynamenode.svc&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-hdfs-secondarynamenode.svc Mon Apr 16 16:10:22 2012
@@ -1,4 +1,3 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,7 +12,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+HADOOP_NAME="secondarynamenode"
+DAEMON="hadoop-hdfs-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop/sbin/hadoop-daemon.sh"
+SVC_USER="hdfs"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/hadoop/conf"
+PIDFILE="/var/run/hadoop-hdfs/hadoop-$SVC_USER-$HADOOP_NAME.pid"
+LOCKFILE="/var/lock/subsys/hadoop-$HADOOP_NAME"
 
-set -ex
-
-ant -f src/build.xml -Dhadoop.mirror=http://archive.cloudera.com/hive-deps package "$@"
+CHKCONFIG="2345 85 15"
+INIT_DEFAULT_START="3 4 5"
+INIT_DEFAULT_STOP="0 1 2 6"

Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-httpfs.svc
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-httpfs.svc?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-httpfs.svc (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-httpfs.svc Mon Apr 16 16:10:22 2012
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+HADOOP_NAME="httpfs"
+DAEMON="hadoop-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop-httpfs/sbin/httpfs.sh"
+SVC_USER="$HADOOP_NAME"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/$DAEMON/conf"
+PIDFILE="/var/run/$DAEMON/hadoop-$SVC_USER-$HADOOP_NAME.pid"
+LOCKFILE="/var/lock/subsys/$DAEMON"
+
+CHKCONFIG="345 90 10"
+INIT_DEFAULT_START="3 4 5"
+INIT_DEFAULT_STOP="0 1 2 6"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+  [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+  [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED
+  log_success_msg "Starting ${DESC}: "
+
+  export HTTPFS_USER="$SVC_USER"
+  export HTTPFS_CONFIG="$CONF_DIR"
+  export HTTPFS_LOG=${HTTPFS_LOG:-"/var/log/hadoop-httpfs/"}
+  export HTTPFS_TEMP="$(dirname $PIDFILE)"
+  export HTTPFS_SLEEP_TIME="$SLEEP_TIME"
+  export CATALINA_BASE=${CATALINA_BASE:-"/usr/lib/hadoop-httpfs"}
+  export CATALINA_PID="$PIDFILE"
+  export CATALINA_TMPDIR="$HTTPFS_TEMP"
+
+  su -s /bin/bash -c "${EXEC_PATH} start $DAEMON_FLAGS" $HTTPFS_USER
+
+  checkstatusofproc
+  RETVAL=$?
+
+  [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE
+  return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+  log_success_msg "Stopping ${DESC}: "
+  su -s /bin/bash $SVC_USER -c "${EXEC_PATH} stop $SLEEP_TIME -force"
+  RETVAL=$?
+
+  [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
+}
+__EOT__
+
+}

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-layout.sh (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-layout.sh?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-layout.sh&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-layout.sh Mon Apr 16 16:10:22 2012
@@ -1,4 +1,3 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,8 +12,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+HADOOP_COMMON_DIR="./"
+HADOOP_COMMON_LIB_JARS_DIR="lib"
+HADOOP_COMMON_LIB_NATIVE_DIR="lib/native"
+HDFS_DIR="./"
+HDFS_LIB_JARS_DIR="lib"
+YARN_DIR="./"
+YARN_LIB_JARS_DIR="lib"
+MAPRED_DIR="./"
+MAPRED_LIB_JARS_DIR="lib"
 
-set -ex
-
-mvn clean -DskipTests -Dhbase.version=${HBASE_VERSION} -Dhadoop.version=0.20.205.0 install assembly:assembly "$@"
-tar --strip-components=1 -xzf target/hbase*.tar.gz
+HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-"/usr/lib/hadoop/libexec"}
+HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop/conf"}
+HADOOP_COMMON_HOME=${HADOOP_COMMON_HOME:-"/usr/lib/hadoop"}
+HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME:-"/usr/lib/hadoop-hdfs"}
+HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-"/usr/lib/hadoop-mapreduce"}
+YARN_HOME=${YARN_HOME:-"/usr/lib/hadoop-yarn"}

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-mapreduce-historyserver.svc (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-mapreduce-historyserver.svc?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-mapreduce-historyserver.svc&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-mapreduce-historyserver.svc Mon Apr 16 16:10:22 2012
@@ -1,4 +1,3 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,7 +12,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+HADOOP_NAME="historyserver"
+DAEMON="hadoop-mapreduce-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop-mapreduce/sbin/mr-jobhistory-daemon.sh"
+SVC_USER="mapred"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/hadoop/conf"
+PIDFILE="/var/run/hadoop-mapreduce/yarn-$SVC_USER-$HADOOP_NAME.pid"
+LOCKFILE="/var/lock/subsys/hadoop-$HADOOP_NAME"
 
-set -ex
-
-ant -f src/build.xml -Dhadoop.mirror=http://archive.cloudera.com/hive-deps package "$@"
+CHKCONFIG="345 85 15"
+INIT_DEFAULT_START="3 4 5"
+INIT_DEFAULT_STOP="0 1 2 6"

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-nodemanager.svc (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-nodemanager.svc?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-nodemanager.svc&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-nodemanager.svc Mon Apr 16 16:10:22 2012
@@ -1,4 +1,3 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,7 +12,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+HADOOP_NAME="nodemanager"
+DAEMON="hadoop-yarn-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh"
+SVC_USER="yarn"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/hadoop/conf"
+PIDFILE="/var/run/hadoop-yarn/yarn-$SVC_USER-$HADOOP_NAME.pid"
+LOCKFILE="/var/lock/subsys/hadoop-$HADOOP_NAME"
 
-set -ex
-
-ant -f src/build.xml -Dhadoop.mirror=http://archive.cloudera.com/hive-deps package "$@"
+CHKCONFIG="345 85 15"
+INIT_DEFAULT_START="3 4 5"
+INIT_DEFAULT_STOP="0 1 2 6"

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-proxyserver.svc (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-proxyserver.svc?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-proxyserver.svc&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-proxyserver.svc Mon Apr 16 16:10:22 2012
@@ -1,4 +1,3 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,7 +12,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+HADOOP_NAME="proxyserver"
+DAEMON="hadoop-yarn-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh"
+SVC_USER="yarn"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/hadoop/conf"
+PIDFILE="/var/run/hadoop-yarn/yarn-$SVC_USER-$HADOOP_NAME.pid"
+LOCKFILE="/var/lock/subsys/hadoop-$HADOOP_NAME"
 
-set -ex
-
-ant -f src/build.xml -Dhadoop.mirror=http://archive.cloudera.com/hive-deps package "$@"
+CHKCONFIG="345 85 15"
+INIT_DEFAULT_START="3 4 5"
+INIT_DEFAULT_STOP="0 1 2 6"

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-resourcemanager.svc (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-resourcemanager.svc?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-resourcemanager.svc&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop-yarn-resourcemanager.svc Mon Apr 16 16:10:22 2012
@@ -1,4 +1,3 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,7 +12,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+HADOOP_NAME="resourcemanager"
+DAEMON="hadoop-yarn-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh"
+SVC_USER="yarn"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/hadoop/conf"
+PIDFILE="/var/run/hadoop-yarn/yarn-$SVC_USER-$HADOOP_NAME.pid"
+LOCKFILE="/var/lock/subsys/hadoop-$HADOOP_NAME"
 
-set -ex
-
-ant -f src/build.xml -Dhadoop.mirror=http://archive.cloudera.com/hive-deps package "$@"
+CHKCONFIG="345 85 15"
+INIT_DEFAULT_START="3 4 5"
+INIT_DEFAULT_STOP="0 1 2 6"

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default Mon Apr 16 16:10:22 2012
@@ -14,11 +14,12 @@
 # limitations under the License.
 export HADOOP_HOME_WARN_SUPPRESS=true
 export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_NAMENODE_USER=hdfs
-export HADOOP_SECONDARYNAMENODE_USER=hdfs
-export HADOOP_DATANODE_USER=hdfs
-export HADOOP_JOBTRACKER_USER=mapred
-export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hadoop
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
+export HADOOP_PREFIX=/usr/lib/hadoop
+
+export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec
+export HADOOP_CONF_DIR=/etc/hadoop/conf
+
+export HADOOP_COMMON_HOME=/usr/lib/hadoop
+export HADOOP_HDFS_HOME=/usr/lib/hadoop-hdfs
+export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
+export YARN_HOME=/usr/lib/hadoop-yarn

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hdfs.default (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hdfs.default?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hdfs.default&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hdfs.default Mon Apr 16 16:10:22 2012
@@ -12,13 +12,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-export HADOOP_HOME_WARN_SUPPRESS=true
-export HADOOP_HOME=/usr/lib/hadoop
+export HADOOP_PID_DIR=/var/run/hadoop-hdfs
+export HADOOP_LOG_DIR=/var/log/hadoop-hdfs
 export HADOOP_NAMENODE_USER=hdfs
 export HADOOP_SECONDARYNAMENODE_USER=hdfs
 export HADOOP_DATANODE_USER=hdfs
-export HADOOP_JOBTRACKER_USER=mapred
-export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hadoop
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
+export HADOOP_IDENT_STRING=hdfs
+
+# export HADOOP_SECURE_DN_USER=hdfs
+# export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop-hdfs
+# export HADOOP_SECURE_DN_LOG_DIR=/var/log/hadoop-hdfs

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/httpfs.default (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/httpfs.default?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/httpfs.default&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/httpfs.default Mon Apr 16 16:10:22 2012
@@ -12,13 +12,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-export HADOOP_HOME_WARN_SUPPRESS=true
-export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_NAMENODE_USER=hdfs
-export HADOOP_SECONDARYNAMENODE_USER=hdfs
-export HADOOP_DATANODE_USER=hdfs
-export HADOOP_JOBTRACKER_USER=mapred
-export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hadoop
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
+export HTTPFS_USER=httpfs
+export HTTPFS_CONFIG=/etc/hadoop-httpfs/conf
+export HTTPFS_LOG=/var/log/hadoop-httpfs/
+export HTTPFS_TEMP=/var/run/hadoop-httpfs/
+export HTTPFS_CATALINA_HOME=/usr/lib/bigtop-tomcat
+export CATALINA_PID=/var/run/hadoop-httpfs/hadoop-httpfs-httpfs.pid
+export CATALINA_BASE=/usr/lib/hadoop-httpfs
+export CATALINA_TMPDIR=/var/run/hadoop-httpfs/
+# HTTPFS_HTTP_PORT
+# HTTPFS_ADMIN_PORT

Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/init.d.tmpl
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/init.d.tmpl?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/init.d.tmpl (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/init.d.tmpl Mon Apr 16 16:10:22 2012
@@ -0,0 +1,273 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a poor man's templating engine for generating init.d scripts to
+# support all the Apache services that Bigtop distro has. An actual init.d
+# script gets generate via running this script under bash and giving it
+# a mandatory argument of a file containing the configuration for the service. 
+# The argument file should be a valid piece of bash code since it gets directly
+# source into this template. E.g.
+#    $ bash ./init.d.tmpl hadoop-hdfs-namenode.svc > hadoop-hdfs-namenode
+# 
+# You must declare the following in your .svc configuration file:
+#     DAEMON="name of the resulting init.d script"
+#     DESC="Free form human readable description of the service"
+#     EXEC_PATH="path to the upstream daemon management script"
+#     SVC_USER="user to run this service as"
+#     DAEMON_FLAGS="flags to be passed to the $EXEC_PATH"
+#     CONF_DIR="path to the configuration directory"
+#     PIDFILE="file holding a PID of the running daemon"
+#     LOCKFILE="file signifying the service lock"
+#
+#     CHKCONFIG="chkconfig(8) registration signature"
+#     INIT_DEFAULT_START="run levels to use"
+#     INIT_DEFAULT_STOP="run levels not to use ;-)"
+# 
+# You can, also, override parts of the generated init.d script by providing
+# function definitions for: generate_start, generate_stop and generate_extra_commands.
+# See the default implemenations below and feel free to customize. Also look
+# for exising .svc files in common to see how different services are tweaking
+# the defaults.
+#
+# Of course, if this whole templating thing grows too big we might need to
+# consider a real templating engine (I have played with m4, but it seems
+# qutie brittle when used for manipulating pieces of the shell code -- think
+# $0 the like).
+
+if [ $# -lt 1 ] ; then
+  echo "Usage: ${BASH_SOURCE-0} service.definition.svc"
+  exit 1
+fi
+
+generate_start() {
+
+cat <<'__EOT__'
+
+start() {
+  [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+  [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED
+  log_success_msg "Starting ${DESC}: "
+
+  su -s /bin/bash $SVC_USER -c "$EXEC_PATH --config '$CONF_DIR' start $DAEMON_FLAGS"
+
+  # Some processes are slow to start
+  sleep $SLEEP_TIME
+  checkstatusofproc
+  RETVAL=$?
+
+  [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE
+  return $RETVAL
+}
+
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+
+stop() {
+  log_success_msg "Stopping ${DESC}: "
+  start_daemon $EXEC_PATH --config "$CONF_DIR" stop $DAEMON_FLAGS
+  RETVAL=$?
+
+  [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
+}
+
+__EOT__
+
+}
+
+generate_extra_commands() {
+
+cat <<'__EOT__'
+    *)
+      echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart}"
+      exit 1
+__EOT__
+
+}
+
+###################################################################
+# NOTE how we are sourcing the argument here so that a user-defined
+# settings have a chance to override the default values for 
+# generate_start, generate_stop and generate_extra_commands. If you
+# ever want to make this template even more flexible -- define the
+# default values above this line
+. $1
+
+cat <<__EOT__
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Starts a $DESC
+#
+# chkconfig: $CHKCONFIG
+# description: $DESC
+#
+### BEGIN INIT INFO
+# Provides:          $DAEMON
+# Short-Description: $DESC
+# Default-Start:     $INIT_DEFAULT_START
+# Default-Stop:      $INIT_DEFAULT_STOP
+# Required-Start:    \$syslog \$remote_fs
+# Required-Stop:     \$syslog \$remote_fs
+# Should-Start:
+# Should-Stop:
+### END INIT INFO
+
+. /lib/lsb/init-functions
+. /etc/default/hadoop
+
+if [ -f /etc/default/$DAEMON ] ; then
+  . /etc/default/$DAEMON
+fi
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+  . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+RETVAL_SUCCESS=0
+
+STATUS_RUNNING=0
+STATUS_DEAD=1
+STATUS_DEAD_AND_LOCK=2
+STATUS_NOT_RUNNING=3
+STATUS_OTHER_ERROR=102
+
+
+ERROR_PROGRAM_NOT_INSTALLED=5
+ERROR_PROGRAM_NOT_CONFIGURED=6
+
+
+RETVAL=0
+SLEEP_TIME=5
+PROC_NAME="su"
+
+DAEMON="$DAEMON"
+DESC="$DESC"
+EXEC_PATH="$EXEC_PATH"
+SVC_USER="$SVC_USER"
+DAEMON_FLAGS="$DAEMON_FLAGS"
+CONF_DIR="$CONF_DIR"
+PIDFILE="$PIDFILE"
+LOCKFILE="$LOCKFILE"
+
+install -d -m 0755 -o $SVC_USER -g $SVC_USER $(dirname $PIDFILE) 1>/dev/null 2>&1 || :
+__EOT__
+
+generate_start
+generate_stop
+
+cat <<'__EOT__'
+restart() {
+  stop
+  start
+}
+
+checkstatusofproc(){
+  pidofproc -p $PIDFILE $PROC_NAME > /dev/null
+}
+
+checkstatus(){
+  checkstatusofproc
+  status=$?
+
+  case "$status" in
+    $STATUS_RUNNING)
+      log_success_msg "${DESC} is running"
+      ;;
+    $STATUS_DEAD)
+      log_failure_msg "${DESC} is dead and pid file exists"
+      ;;
+    $STATUS_DEAD_AND_LOCK)
+      log_failure_msg "${DESC} is dead and lock file exists"
+      ;;
+    $STATUS_NOT_RUNNING)
+      log_failure_msg "${DESC} is not running"
+      ;;
+    *)
+      log_failure_msg "${DESC} status is unknown"
+      ;;
+  esac
+  return $status
+}
+
+condrestart(){
+  [ -e $LOCKFILE ] && restart || :
+}
+
+check_for_root() {
+  if [ $(id -ur) -ne 0 ]; then
+    echo 'Error: root user required'
+    echo
+    exit 1
+  fi
+}
+
+service() {
+  case "$1" in
+    start)
+      check_for_root
+      start
+      ;;
+    stop)
+      check_for_root
+      stop
+      ;;
+    status)
+      checkstatus
+      RETVAL=$?
+      ;;
+    restart)
+      check_for_root
+      restart
+      ;;
+    condrestart|try-restart)
+      check_for_root
+      condrestart
+      ;;
+__EOT__
+
+generate_extra_commands
+
+cat <<'__EOT__'
+  esac
+}
+
+service "$1"
+
+exit $RETVAL
+__EOT__

Propchange: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/init.d.tmpl
------------------------------------------------------------------------------
    svn:executable = *

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/install_hadoop.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/install_hadoop.sh?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/install_hadoop.sh (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/install_hadoop.sh Mon Apr 16 16:10:22 2012
@@ -34,15 +34,22 @@ usage: $0 <options>
 OPTS=$(getopt \
   -n $0 \
   -o '' \
-  -l 'distro-dir:' \
   -l 'prefix:' \
+  -l 'distro-dir:' \
   -l 'build-dir:' \
   -l 'native-build-string:' \
   -l 'installed-lib-dir:' \
-  -l 'lib-dir:' \
+  -l 'hadoop-dir:' \
+  -l 'httpfs-dir:' \
+  -l 'hdfs-dir:' \
+  -l 'yarn-dir:' \
+  -l 'mapreduce-dir:' \
+  -l 'client-dir:' \
+  -l 'system-include-dir:' \
   -l 'system-lib-dir:' \
-  -l 'src-dir:' \
-  -l 'etc-dir:' \
+  -l 'system-libexec-dir:' \
+  -l 'hadoop-etc-dir:' \
+  -l 'httpfs-etc-dir:' \
   -l 'doc-dir:' \
   -l 'man-dir:' \
   -l 'example-dir:' \
@@ -62,12 +69,33 @@ while true ; do
         --distro-dir)
         DISTRO_DIR=$2 ; shift 2
         ;;
-        --lib-dir)
-        LIB_DIR=$2 ; shift 2
+        --httpfs-dir)
+        HTTPFS_DIR=$2 ; shift 2
+        ;;
+        --hadoop-dir)
+        HADOOP_DIR=$2 ; shift 2
+        ;;
+        --hdfs-dir)
+        HDFS_DIR=$2 ; shift 2
+        ;;
+        --yarn-dir)
+        YARN_DIR=$2 ; shift 2
+        ;;
+        --mapreduce-dir)
+        MAPREDUCE_DIR=$2 ; shift 2
+        ;;
+        --client-dir)
+        CLIENT_DIR=$2 ; shift 2
+        ;;
+        --system-include-dir)
+        SYSTEM_INCLUDE_DIR=$2 ; shift 2
         ;;
         --system-lib-dir)
         SYSTEM_LIB_DIR=$2 ; shift 2
         ;;
+        --system-libexec-dir)
+        SYSTEM_LIBEXEC_DIR=$2 ; shift 2
+        ;;
         --build-dir)
         BUILD_DIR=$2 ; shift 2
         ;;
@@ -77,8 +105,11 @@ while true ; do
         --doc-dir)
         DOC_DIR=$2 ; shift 2
         ;;
-        --etc-dir)
-        ETC_DIR=$2 ; shift 2
+        --hadoop-etc-dir)
+        HADOOP_ETC_DIR=$2 ; shift 2
+        ;;
+        --httpfs-etc-dir)
+        HTTPFS_ETC_DIR=$2 ; shift 2
         ;;
         --installed-lib-dir)
         INSTALLED_LIB_DIR=$2 ; shift 2
@@ -89,9 +120,6 @@ while true ; do
         --example-dir)
         EXAMPLE_DIR=$2 ; shift 2
         ;;
-        --src-dir)
-        SRC_DIR=$2 ; shift 2
-        ;;
         --)
         shift ; break
         ;;
@@ -110,200 +138,227 @@ for var in PREFIX BUILD_DIR; do
   fi
 done
 
-LIB_DIR=${LIB_DIR:-$PREFIX/usr/lib/hadoop}
+HADOOP_DIR=${HADOOP_DIR:-$PREFIX/usr/lib/hadoop}
+HDFS_DIR=${HDFS_DIR:-$PREFIX/usr/lib/hadoop-hdfs}
+YARN_DIR=${YARN_DIR:-$PREFIX/usr/lib/hadoop-yarn}
+MAPREDUCE_DIR=${MAPREDUCE_DIR:-$PREFIX/usr/lib/hadoop-mapreduce}
+CLIENT_DIR=${CLIENT_DIR:-$PREFIX/usr/lib/hadoop/client}
+HTTPFS_DIR=${HTTPFS_DIR:-$PREFIX/usr/lib/hadoop-httpfs}
 SYSTEM_LIB_DIR=${SYSTEM_LIB_DIR:-/usr/lib}
 BIN_DIR=${BIN_DIR:-$PREFIX/usr/bin}
 DOC_DIR=${DOC_DIR:-$PREFIX/usr/share/doc/hadoop}
 MAN_DIR=${MAN_DIR:-$PREFIX/usr/man}
+SYSTEM_INCLUDE_DIR=${SYSTEM_INCLUDE_DIR:-$PREFIX/usr/include}
+SYSTEM_LIBEXEC_DIR=${SYSTEM_LIBEXEC_DIR:-$PREFIX/usr/libexec}
 EXAMPLE_DIR=${EXAMPLE_DIR:-$DOC_DIR/examples}
-SRC_DIR=${SRC_DIR:-$PREFIX/usr/src/hadoop}
-ETC_DIR=${ETC_DIR:-$PREFIX/etc/hadoop}
+HADOOP_ETC_DIR=${HADOOP_ETC_DIR:-$PREFIX/etc/hadoop}
+HTTPFS_ETC_DIR=${HTTPFS_ETC_DIR:-$PREFIX/etc/hadoop-httpfs}
 
-INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/hadoop}
-BUILD_SRC_DIR=./src
+INSTALLED_HADOOP_DIR=${INSTALLED_HADOOP_DIR:-/usr/lib/hadoop}
+HADOOP_NATIVE_LIB_DIR=${HADOOP_DIR}/lib/native
 
-mkdir -p $LIB_DIR
-(cd $BUILD_DIR && tar -cf - .) | (cd $LIB_DIR && tar xf - )
+HADOOP_VERSION=0.23.1
 
-# Take out things we've installed elsewhere
-for x in docs lib/native c++ src conf contrib/fuse-dfs/fuse_dfs usr/bin/fuse_dfs contrib/fuse share sbin/task-controller 'lib/lib*so*' 'lib/lib*a' ; do
-  rm -rf $LIB_DIR/$x 
-done
+##Needed for some distros to find ldconfig
+export PATH="/sbin/:$PATH"
 
 # Make bin wrappers
 mkdir -p $BIN_DIR
 
-for bin_wrapper in hadoop ; do
-  wrapper=$BIN_DIR/$bin_wrapper
+for component in $HADOOP_DIR/bin/hadoop $HDFS_DIR/bin/hdfs $YARN_DIR/bin/yarn $MAPREDUCE_DIR/bin/mapred ; do
+  wrapper=$BIN_DIR/${component#*/bin/}
   cat > $wrapper <<EOF
 #!/bin/sh
 
 # Autodetect JAVA_HOME if not defined
 if [ -e /usr/libexec/bigtop-detect-javahome ]; then
-  . /usr/libexec/bigtop-detect-javahome
+. /usr/libexec/bigtop-detect-javahome
 elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
-  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+. /usr/lib/bigtop-utils/bigtop-detect-javahome
 fi
 
-. /etc/default/hadoop
-exec $INSTALLED_LIB_DIR/bin/$bin_wrapper "\$@"
+export HADOOP_LIBEXEC_DIR=/${SYSTEM_LIBEXEC_DIR#${PREFIX}}
+
+exec ${component#${PREFIX}} "\$@"
 EOF
   chmod 755 $wrapper
 done
 
-# Link examples to /usr/share
-mkdir -p $EXAMPLE_DIR
-for x in $LIB_DIR/*examples*jar ; do
-  INSTALL_LOC=`echo $x | sed -e "s,$LIB_DIR,$INSTALLED_LIB_DIR,"`
-  ln -sf $INSTALL_LOC $EXAMPLE_DIR/
+#libexec
+install -d -m 0755 ${SYSTEM_LIBEXEC_DIR}
+cp ${BUILD_DIR}/libexec/* ${SYSTEM_LIBEXEC_DIR}/
+cp ${DISTRO_DIR}/hadoop-layout.sh ${SYSTEM_LIBEXEC_DIR}/
+
+# hadoop jar
+install -d -m 0755 ${HADOOP_DIR}
+cp ${BUILD_DIR}/share/hadoop/common/*.jar ${HADOOP_DIR}/
+cp ${BUILD_DIR}/share/hadoop/common/lib/hadoop-auth*.jar ${HADOOP_DIR}/
+cp ${BUILD_DIR}/share/hadoop/mapreduce/lib/hadoop-annotations*.jar ${HADOOP_DIR}/
+install -d -m 0755 ${MAPREDUCE_DIR}
+cp ${BUILD_DIR}/share/hadoop/mapreduce/hadoop-mapreduce*.jar ${MAPREDUCE_DIR}
+cp ${BUILD_DIR}/share/hadoop/tools/lib/*.jar ${MAPREDUCE_DIR}
+install -d -m 0755 ${HDFS_DIR}
+cp ${BUILD_DIR}/share/hadoop/hdfs/*.jar ${HDFS_DIR}/
+install -d -m 0755 ${YARN_DIR}
+cp ${BUILD_DIR}/share/hadoop/mapreduce/hadoop-yarn*.jar ${YARN_DIR}/
+chmod 644 ${HADOOP_DIR}/*.jar ${MAPREDUCE_DIR}/*.jar ${HDFS_DIR}/*.jar ${YARN_DIR}/*.jar
+
+# lib jars
+install -d -m 0755 ${HADOOP_DIR}/lib
+cp ${BUILD_DIR}/share/hadoop/common/lib/*.jar ${HADOOP_DIR}/lib
+install -d -m 0755 ${MAPREDUCE_DIR}/lib
+cp ${BUILD_DIR}/share/hadoop/mapreduce/lib/*.jar ${MAPREDUCE_DIR}/lib
+install -d -m 0755 ${HDFS_DIR}/lib 
+cp ${BUILD_DIR}/share/hadoop/hdfs/lib/*.jar ${HDFS_DIR}/lib
+install -d -m 0755 ${YARN_DIR}/lib
+cp ${BUILD_DIR}/share/hadoop/mapreduce/lib/*.jar ${YARN_DIR}/lib
+chmod 644 ${HADOOP_DIR}/lib/*.jar ${MAPREDUCE_DIR}/lib/*.jar ${HDFS_DIR}/lib/*.jar ${YARN_DIR}/lib/*.jar
+
+# Install webapps
+cp -ra ${BUILD_DIR}/share/hadoop/hdfs/webapps ${HDFS_DIR}/
+
+# bin
+install -d -m 0755 ${HADOOP_DIR}/bin
+cp -a ${BUILD_DIR}/bin/{hadoop,rcc,fuse_dfs,fuse_dfs_wrapper.sh} ${HADOOP_DIR}/bin
+install -d -m 0755 ${HDFS_DIR}/bin
+cp -a ${BUILD_DIR}/bin/hdfs ${HDFS_DIR}/bin
+install -d -m 0755 ${YARN_DIR}/bin
+cp -a ${BUILD_DIR}/bin/{yarn,container-executor} ${YARN_DIR}/bin
+install -d -m 0755 ${MAPREDUCE_DIR}/bin
+cp -a ${BUILD_DIR}/bin/mapred ${MAPREDUCE_DIR}/bin
+# FIXME: MAPREDUCE-3980
+cp -a ${BUILD_DIR}/bin/mapred ${YARN_DIR}/bin
+
+# sbin
+install -d -m 0755 ${HADOOP_DIR}/sbin
+cp -a ${BUILD_DIR}/sbin/{hadoop-daemon,hadoop-daemons,slaves}.sh ${HADOOP_DIR}/sbin
+install -d -m 0755 ${HDFS_DIR}/sbin
+cp -a ${BUILD_DIR}/sbin/{distribute-exclude,refresh-namenodes}.sh ${HDFS_DIR}/sbin
+install -d -m 0755 ${YARN_DIR}/sbin
+cp -a ${BUILD_DIR}/sbin/{yarn-daemon,yarn-daemons}.sh ${YARN_DIR}/sbin
+install -d -m 0755 ${MAPREDUCE_DIR}/sbin
+cp -a ${BUILD_DIR}/sbin/mr-jobhistory-daemon.sh ${MAPREDUCE_DIR}/sbin
+
+# native libs
+install -d -m 0755 ${SYSTEM_LIB_DIR}
+install -d -m 0755 ${HADOOP_NATIVE_LIB_DIR}
+for library in libhdfs.so.0.0.0; do
+  cp ${BUILD_DIR}/lib/native/${library} ${SYSTEM_LIB_DIR}/
+  ldconfig -vlN ${SYSTEM_LIB_DIR}/${library}
+  ln -s ${library} ${SYSTEM_LIB_DIR}/${library/.so.*/}.so
 done
-# And copy the source
-mkdir -p $EXAMPLE_DIR/src
-cp -a $BUILD_SRC_DIR/examples/* $EXAMPLE_DIR/src
-
-# Install docs
-mkdir -p $DOC_DIR
-cp -r $BUILD_DIR/docs/* $DOC_DIR
-
-# Install source
-mkdir -p ${SRC_DIR}
-rm -f ${BUILD_SRC_DIR}/contrib/fuse-dfs/src/*.o 
-rm -f ${BUILD_SRC_DIR}/contrib/fuse-dfs/src/fuse_dfs
-rm -rf ${BUILD_SRC_DIR}/contrib/hod
-rm -f ${SRC_DIR}/contrib/fuse-dfs/fuse_dfs
-
-
-cp -a ${BUILD_SRC_DIR}/* ${SRC_DIR}/
-
-# Make the empty config
-install -d -m 0755 $ETC_DIR/conf.empty
-(cd ${BUILD_DIR}/conf && tar cf - .) | (cd $ETC_DIR/conf.empty && tar xf -)
-
-# Create symlinks to preserve old jar names
-# Also create symlinks of versioned jars to jars without version names, which other
-# packages can depend on
-(cd $LIB_DIR &&
-for j in hadoop-*.jar; do
-  if [[ $j =~ hadoop-(.*)-([^-]+).jar ]]; then
-    name=${BASH_REMATCH[1]}
-    ver=${BASH_REMATCH[2]}
-    ln -s hadoop-$name-$ver.jar hadoop-$name.jar
-  fi
-done)
 
-# Link the HADOOP_HOME conf, log and pid dir to installed locations
-rm -rf $LIB_DIR/conf
-ln -s ${ETC_DIR#$PREFIX}/conf $LIB_DIR/conf
-rm -rf $LIB_DIR/logs
-ln -s /var/log/hadoop $LIB_DIR/logs
-rm -rf $LIB_DIR/pids
-ln -s /var/run/hadoop $LIB_DIR/pids
+install -d -m 0755 ${SYSTEM_INCLUDE_DIR}
+cp ${BUILD_DIR}/include/hdfs.h ${SYSTEM_INCLUDE_DIR}/
 
-# Make the pseudo-distributed config
-for conf in conf.pseudo ; do
-  install -d -m 0755 $ETC_DIR/$conf
-  # Install the default configurations and set some permissions.
-  (cd ${BUILD_DIR}/conf && tar -cf - .) | (cd $ETC_DIR/$conf && tar -xf -)
-  chmod -R 0644 $ETC_DIR/$conf/*
-  # Overlay the -site files
-  (cd $DISTRO_DIR/$conf && tar -cf - .) | (cd $ETC_DIR/$conf && tar -xf -)
-  # When building straight out of svn we have to account for pesky .svn subdirs
-  rm -rf `find $ETC_DIR/$conf -name .svn -type d`
+cp ${BUILD_DIR}/lib/native/*.a ${HADOOP_NATIVE_LIB_DIR}/
+for library in `cd ${BUILD_DIR}/lib ; ls libsnappy.so.1.* 2>/dev/null` libhadoop.so.1.0.0; do
+  cp ${BUILD_DIR}/lib/native/${library} ${HADOOP_NATIVE_LIB_DIR}/
+  ldconfig -vlN ${HADOOP_NATIVE_LIB_DIR}/${library}
+  ln -s ${library} ${HADOOP_NATIVE_LIB_DIR}/${library/.so.*/}.so
 done
 
-# man pages
-mkdir -p $MAN_DIR/man1
-gzip -c < $DISTRO_DIR/hadoop.1 > $MAN_DIR/man1/hadoop.1.gz
-
-############################################################
-# ARCH DEPENDENT STUFF
-############################################################
-
-if [ ! -z "$NATIVE_BUILD_STRING" ]; then
-  # Fuse 
-  mkdir -p $LIB_DIR/bin
-  if [ -d ./src/contrib/fuse-dfs ]; then
-    gzip -c < $DISTRO_DIR/hadoop-fuse-dfs.1 > $MAN_DIR/man1/hadoop-fuse-dfs.1.gz
-
-    # Fuse 
-    mkdir -p $LIB_DIR/bin
-    mv  ${BUILD_DIR}/contrib/fuse-dfs/* $LIB_DIR/bin
-    rmdir ${BUILD_DIR}/contrib/fuse-dfs
-
-    fuse_wrapper=${BIN_DIR}/hadoop-fuse-dfs
-  cat > $fuse_wrapper << EOF
+# Install fuse wrapper
+fuse_wrapper=${BIN_DIR}/hadoop-fuse-dfs
+cat > $fuse_wrapper << EOF
 #!/bin/bash
 
 /sbin/modprobe fuse
 
 # Autodetect JAVA_HOME if not defined
 if [ -e /usr/libexec/bigtop-detect-javahome ]; then
-  . /usr/libexec/bigtop-detect-javahome
+. /usr/libexec/bigtop-detect-javahome
 elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
-  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+. /usr/lib/bigtop-utils/bigtop-detect-javahome
 fi
 
-. /etc/default/hadoop
+export HADOOP_HOME=\${HADOOP_HOME:-${HADOOP_DIR#${PREFIX}}}
 
-if [ -f /etc/default/hadoop-fuse ] 
-  then . /etc/default/hadoop-fuse
+if [ -f /etc/default/hadoop-fuse ]
+then . /etc/default/hadoop-fuse
 fi
 
-if [ -f \$HADOOP_HOME/bin/hadoop-config.sh ] 
-  then . \$HADOOP_HOME/bin/hadoop-config.sh
-fi
+export HADOOP_LIBEXEC_DIR=${SYSTEM_LIBEXEC_DIR#${PREFIX}}
 
 if [ "\${LD_LIBRARY_PATH}" = "" ]; then
   export LD_LIBRARY_PATH=/usr/lib
-  for f in \`find \${JAVA_HOME}/jre/lib -name client -prune -o -name libjvm.so -exec dirname {} \;\`; do
+  for f in \`find \${JAVA_HOME} -name client -prune -o -name libjvm.so -exec dirname {} \;\`; do
     export LD_LIBRARY_PATH=\$f:\${LD_LIBRARY_PATH}
   done
 fi
 
-for i in \${HADOOP_HOME}/*.jar \${HADOOP_HOME}/lib/*.jar
-  do CLASSPATH+=\$i:
+# Pulls all jars from hadoop client package
+for jar in \${HADOOP_HOME}/client/*.jar; do
+  CLASSPATH+="\$jar:"
 done
 
-export PATH=\$PATH:\${HADOOP_HOME}/bin/
-
-env CLASSPATH=\$CLASSPATH \${HADOOP_HOME}/bin/fuse_dfs \$@
+env CLASSPATH="\${CLASSPATH}" \${HADOOP_HOME}/bin/fuse_dfs \$@
 EOF
 
-    chmod 755 $fuse_wrapper
-  fi
+chmod 755 $fuse_wrapper
 
-  # Security related binaries
-  mkdir -p $LIB_DIR/sbin/${NATIVE_BUILD_STRING}
-  # FIXME: workaround for BIGTOP-139
-  cp -f $LIB_DIR/bin/task-controller $LIB_DIR/sbin/${NATIVE_BUILD_STRING}
-  cp -f $LIB_DIR/libexec/jsvc* $LIB_DIR/sbin/${NATIVE_BUILD_STRING}/jsvc
-
-  # Native compression libs
-  mkdir -p $LIB_DIR/lib/native/
-  cp -r ${BUILD_DIR}/lib/native/${NATIVE_BUILD_STRING} $LIB_DIR/lib/native/
-
-  # Pipes
-  mkdir -p $PREFIX/$SYSTEM_LIB_DIR $PREFIX/usr/include
-  cp ${BUILD_DIR}/c++/${NATIVE_BUILD_STRING}/lib/libhadooppipes.a \
-      ${BUILD_DIR}/c++/${NATIVE_BUILD_STRING}/lib/libhadooputils.a \
-      $PREFIX/$SYSTEM_LIB_DIR
-  cp -r ${BUILD_DIR}/c++/${NATIVE_BUILD_STRING}/include/hadoop $PREFIX/usr/include/
-
-  # libhdfs
-  cp ${BUILD_DIR}/c++/${NATIVE_BUILD_STRING}/lib/libhdfs.so.0.0.0 $PREFIX/$SYSTEM_LIB_DIR
-  ln -sf libhdfs.so.0.0.0 $PREFIX/$SYSTEM_LIB_DIR/libhdfs.so.0
-
-  # libhdfs-devel - hadoop doesn't realy install these things in nice places :(
-  mkdir -p $PREFIX/usr/share/doc/libhdfs-devel/examples
-
-  cp ${BUILD_SRC_DIR}/c++/libhdfs/hdfs.h $PREFIX/usr/include/
-  cp ${BUILD_SRC_DIR}/c++/libhdfs/hdfs_*.c $PREFIX/usr/share/doc/libhdfs-devel/examples
-
-  #    This is somewhat unintuitive, but the -devel package has this symlink (see Debian Library Packaging Guide)
-  ln -sf libhdfs.so.0.0.0 $PREFIX/$SYSTEM_LIB_DIR/libhdfs.so
-  sed -e "s|^libdir='.*'|libdir=\"$SYSTEM_LIB_DIR\"|" \
-      ./build/c++/${NATIVE_BUILD_STRING}/lib/libhdfs.la > $PREFIX/$SYSTEM_LIB_DIR/libhdfs.la
-fi
 
-# XXX Hack to get hadoop to get packaged
-find $PREFIX -name "*.debug" | xargs rm -fv
+# conf
+install -d -m 0755 $HADOOP_ETC_DIR/conf.empty
+
+cp ${BUILD_DIR}/etc/hadoop/* $HADOOP_ETC_DIR/conf.empty
+
+# docs
+install -d -m 0755 ${DOC_DIR}
+cp -r ${BUILD_DIR}/share/doc/* ${DOC_DIR}/
+
+# man pages
+mkdir -p $MAN_DIR/man1
+gzip -c < $DISTRO_DIR/hadoop.1 > $MAN_DIR/man1/hadoop.1.gz
+chmod 644 $MAN_DIR/man1/hadoop.1.gz
+
+# HTTPFS
+install -d -m 0755 ${HTTPFS_DIR}/sbin
+cp ${BUILD_DIR}/sbin/httpfs.sh ${HTTPFS_DIR}/sbin/
+cp -r ${BUILD_DIR}/share/hadoop/httpfs/tomcat/webapps ${HTTPFS_DIR}/
+cp -r ${BUILD_DIR}/share/hadoop/httpfs/tomcat/conf ${HTTPFS_DIR}/
+chmod 644 ${HTTPFS_DIR}/conf/*
+install -d -m 0755 $HTTPFS_ETC_DIR/conf.empty
+mv $HADOOP_ETC_DIR/conf.empty/httpfs* $HTTPFS_ETC_DIR/conf.empty
+
+# Make the pseudo-distributed config
+for conf in conf.pseudo ; do
+  install -d -m 0755 $HADOOP_ETC_DIR/$conf
+  # Overlay the -site files
+  (cd $DISTRO_DIR/$conf && tar -cf - .) | (cd $HADOOP_ETC_DIR/$conf && tar -xf -)
+  chmod -R 0644 $HADOOP_ETC_DIR/$conf/*
+  # When building straight out of svn we have to account for pesky .svn subdirs 
+  rm -rf `find $HADOOP_ETC_DIR/$conf -name .svn -type d` 
+done
+cp ${BUILD_DIR}/etc/hadoop/log4j.properties $HADOOP_ETC_DIR/conf.pseudo
+
+# FIXME: Provide a convenience link for configuration (HADOOP-7939)
+install -d -m 0755 ${HADOOP_DIR}/etc
+ln -s ${HADOOP_ETC_DIR##${PREFIX}}/conf ${HADOOP_DIR}/etc/hadoop
+
+# Create log, var and lib
+install -d -m 0755 $PREFIX/var/{log,run,lib}/hadoop-hdfs
+install -d -m 0755 $PREFIX/var/{log,run,lib}/hadoop-yarn
+install -d -m 0755 $PREFIX/var/{log,run,lib}/hadoop-mapreduce
+
+# Remove all source and test jars and create version-less symlinks to offer integration point with other projects
+for DIR in ${HADOOP_DIR} ${HDFS_DIR} ${YARN_DIR} ${MAPREDUCE_DIR} ${HTTPFS_DIR} ; do
+  (cd $DIR &&
+   rm -fv *-test.jar *-sources.jar
+   rm -fv lib/hadoop-*.jar
+   for j in hadoop-*.jar; do
+     if [[ $j =~ hadoop-(.*)-${HADOOP_VERSION}.jar ]]; then
+       name=${BASH_REMATCH[1]}
+       ln -s $j hadoop-$name.jar
+     fi
+   done)
+done
+
+# Now create a client installation area full of symlinks
+install -d -m 0755 ${CLIENT_DIR}
+for file in `cat ${BUILD_DIR}/hadoop-client.list` ; do
+  for dir in ${HADOOP_DIR}/{lib,} ${HDFS_DIR}/{lib,} ${YARN_DIR}/{lib,} ${MAPREDUCE_DIR}/{lib,} ; do
+    [ -e $dir/$file ] && ln -fs ${dir#$PREFIX}/$file ${CLIENT_DIR}/$file && continue 2
+  done
+  exit 1
+done

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapreduce.conf (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapreduce.conf?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapreduce.conf&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapreduce.conf Mon Apr 16 16:10:22 2012
@@ -13,5 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-mapred - nofile 32768
-mapred - nproc  65536
+mapred    - nofile 32768
+mapred    - nproc  65536

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapreduce.default (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapreduce.default?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapreduce.default&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapreduce.default Mon Apr 16 16:10:22 2012
@@ -1,4 +1,3 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-
-mvn clean -DskipTests -Dhbase.version=${HBASE_VERSION} -Dhadoop.version=0.20.205.0 install assembly:assembly "$@"
-tar --strip-components=1 -xzf target/hbase*.tar.gz
+# FIXME: MAPREDUCE-3900
+export YARN_IDENT_STRING=mapred
+export YARN_PID_DIR=/var/run/hadoop-mapreduce
+export YARN_LOG_DIR=/var/log/hadoop-mapreduce
+export HADOOP_LOG_DIR=/var/log/hadoop-mapreduce

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/yarn.conf (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/yarn.conf?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/yarn.conf&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/yarn.conf Mon Apr 16 16:10:22 2012
@@ -13,5 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-mapred - nofile 32768
-mapred - nproc  65536
+yarn   - nofile 32768
+yarn   - nproc  65536

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/yarn.default (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/yarn.default?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/yarn.default&p1=incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/yarn.default Mon Apr 16 16:10:22 2012
@@ -1,4 +1,3 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,8 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-set -ex
-
-mvn clean install -Dmahout.skip.distribution=false -DskipTests "$@"
-
+export YARN_IDENT_STRING=yarn
+export YARN_PID_DIR=/var/run/hadoop-yarn
+export YARN_LOG_DIR=/var/log/hadoop-yarn
+export YARN_CONF_DIR=/etc/hadoop/conf

Added: incubator/bigtop/trunk/bigtop-packages/src/common/hbase/HBASE-5212.patch
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hbase/HBASE-5212.patch?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hbase/HBASE-5212.patch (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hbase/HBASE-5212.patch Mon Apr 16 16:10:22 2012
@@ -0,0 +1,73 @@
+--- pom.xml	2012-01-16 07:03:35.000000000 -0800
++++ pom.xml	2012-02-17 09:46:32.506195389 -0800
+@@ -1788,6 +1788,13 @@
+           <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+           <version>${hadoop.version}</version>
+           <optional>true</optional>
++          <scope>test</scope>
++        </dependency>
++        <dependency>
++          <groupId>org.apache.hadoop</groupId>
++          <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
++          <version>${hadoop.version}</version>
++          <optional>true</optional>
+           <type>test-jar</type>
+           <scope>test</scope>
+         </dependency>
+--- src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java	2012-01-16 07:03:40.000000000 -0800
++++ src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java	2012-02-17 09:46:32.466194915 -0800
+@@ -423,7 +423,7 @@
+     if (os != null) {
+       try {
+         m = os.getWrappedStream().getClass().
+-          getMethod("getNumCurrentReplicas", new Class<?> []{});
++          getDeclaredMethod("getNumCurrentReplicas", new Class<?> []{});
+         m.setAccessible(true);
+       } catch (NoSuchMethodException e) {
+         // Thrown if getNumCurrentReplicas() function isn't available
+@@ -438,7 +438,7 @@
+       LOG.info("Using getNumCurrentReplicas--HDFS-826");
+     } else {
+       LOG.info("getNumCurrentReplicas--HDFS-826 not available; hdfs_out=" +
+-        os + ", exception=" + exception.getMessage());
++        os, exception);
+     }
+     return m;
+   }
+--- src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java	2012-01-16 07:03:36.000000000 -0800
++++ src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java	2012-02-17 09:46:32.446194449 -0800
+@@ -1164,6 +1164,8 @@
+     LOG.info("Mini mapreduce cluster started");
+     c.set("mapred.job.tracker",
+         mrCluster.createJobConf().get("mapred.job.tracker"));
++    /* this for mrv2 support */
++    conf.set("mapreduce.framework.name", "yarn");
+   }
+ 
+   /**
+--- src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java	2012-01-16 07:03:37.000000000 -0800
++++ src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java	2012-02-17 09:46:32.436195465 -0800
+@@ -335,13 +335,18 @@
+     // We start 3 servers and then stop 2 to avoid a directory naming conflict
+     //  when we stop/start a namenode later, as mentioned in HBASE-5163
+     List<DataNode> existingNodes = dfsCluster.getDataNodes();
+-    dfsCluster
+-        .startDataNodes(TEST_UTIL.getConfiguration(), 3, true, null, null);
+-    for (DataNode dn: existingNodes){
+-      dfsCluster.stopDataNode( dn.dnRegistration.getName() );
++    int numDataNodes = 3;
++    dfsCluster.startDataNodes(TEST_UTIL.getConfiguration(), numDataNodes, true,
++        null, null);
++    List<DataNode> allNodes = dfsCluster.getDataNodes();
++    for (int i = allNodes.size()-1; i >= 0; i--) {
++      if (existingNodes.contains(allNodes.get(i))) {
++        dfsCluster.stopDataNode( i );
++      }
+     }
+ 
+-    assertTrue(
++    assertTrue("DataNodes " + dfsCluster.getDataNodes().size() +
++        " default replication " + fs.getDefaultReplication(),
+       dfsCluster.getDataNodes().size() >= fs.getDefaultReplication() + 1);
+ 
+     writeData(table, 2);

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hbase/do-component-build Mon Apr 16 16:10:22 2012
@@ -16,5 +16,5 @@
 
 set -ex
 
-mvn clean -DskipTests -Dhbase.version=${HBASE_VERSION} -Dhadoop.version=0.20.205.0 install assembly:assembly "$@"
-tar --strip-components=1 -xzf target/hbase*.tar.gz
+mvn clean -DskipTests -Dhadoop.profile=23 -Dhadoop.version=0.23.1 install site assembly:assembly "$@"
+tar --strip-components=2 -xzf target/hbase*.tar.gz

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hbase/install_hbase.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hbase/install_hbase.sh?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hbase/install_hbase.sh (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hbase/install_hbase.sh Mon Apr 16 16:10:22 2012
@@ -110,14 +110,19 @@ install -d -m 0755 $PREFIX/$BIN_DIR
 install -d -m 0755 $PREFIX/$ETC_DIR
 install -d -m 0755 $PREFIX/$MAN_DIR
 
-cp -ra lib/* ${PREFIX}/${LIB_DIR}/lib/
-cp hbase*.jar $PREFIX/$LIB_DIR
-cp -a docs/* $PREFIX/$DOC_DIR
-cp *.txt $PREFIX/$DOC_DIR/
-cp -a hbase-webapps $PREFIX/$LIB_DIR
-
-cp -a conf $PREFIX/$CONF_DIR
-cp -a bin/* $PREFIX/$BIN_DIR
+cp -ra $BUILD_DIR/lib/* ${PREFIX}/${LIB_DIR}/lib/
+cp $BUILD_DIR/hbase*.jar $PREFIX/$LIB_DIR
+cp -a $BUILD_DIR/docs/* $PREFIX/$DOC_DIR
+cp $BUILD_DIR/*.txt $PREFIX/$DOC_DIR/
+cp -a $BUILD_DIR/hbase-webapps $PREFIX/$LIB_DIR
+
+cp -a $BUILD_DIR/conf $PREFIX/$CONF_DIR
+cp -a $BUILD_DIR/bin/* $PREFIX/$BIN_DIR
+# Purge scripts that don't work with packages
+for file in rolling-restart.sh graceful_stop.sh local-regionservers.sh \
+            start-hbase.sh stop-hbase.sh local-master-backup.sh ; do
+  rm -f $PREFIX/$BIN_DIR/$file
+done
 
 ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/conf
 

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hive/do-component-build Mon Apr 16 16:10:22 2012
@@ -15,5 +15,4 @@
 # limitations under the License.
 
 set -ex
-
-ant -f src/build.xml -Dhadoop.mirror=http://archive.cloudera.com/hive-deps package "$@"
+ant -f src/build.xml -Dhadoop.version=0.23.0 -Dhadoop.security.version=0.23.0 -Dhadoop.security.version.prefix=0.23 package "$@"

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh Mon Apr 16 16:10:22 2012
@@ -65,6 +65,7 @@ HIVE_USER="hive"
 HIVE_HOME="`eval echo ~$HIVE_USER`"
 NICENESS="+0"
 TIMEOUT=3
+USER="hive"
 
 [ -f $SYS_FILE ] && . $SYS_FILE
 
@@ -77,8 +78,14 @@ hive_start() {
       exec_env="HADOOP_OPTS=\"-Dhive.log.dir=`dirname $LOG_FILE`\""
     fi
 
+    if [ -x /sbin/runuser ]; then
+      SU="runuser -s /bin/bash $USER"
+    else
+      SU="su -s /bin/sh $USER"
+    fi
+
     log_success_msg "Starting $desc (${NAME}): "
-    start_daemon -u $HIVE_USER -p $PID_FILE -n $NICENESS  /bin/sh -c "cd $HIVE_HOME ; $exec_env nohup \
+     $SU -c "cd $HIVE_HOME ; $exec_env nohup \ 
            $EXE_FILE --service $service_name $PORT \
              > $LOG_FILE 2>&1 < /dev/null & "'echo $! '"> $PID_FILE"
 

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh.suse (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh.suse?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh.suse&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh.suse Mon Apr 16 16:10:22 2012
@@ -14,15 +14,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-#
-#
+
 # Starts a Hive @HIVE_DAEMON@
 #
 # chkconfig: 345 90 10
 # description: Starts a Hive @HIVE_DAEMON@
 # processname: hive
-# pidfile: /var/run/hive/hive-@HIVE_DAEMON@.pid
+#
 ### BEGIN INIT INFO
 # Provides:          hive-@HIVE_DAEMON@
 # Required-Start:    $syslog $remote_fs
@@ -31,10 +29,10 @@
 # Should-Stop:
 # Default-Start:     3 4 5
 # Default-Stop:      0 1 2 6
-# Short-Description: Starts a Hive @HIVE_DAEMON@
+# Short-Description: Hive @HIVE_DAEMON@
 ### END INIT INFO
 
-source /lib/lsb/init-functions
+. /lib/lsb/init-functions
 
 # Autodetect JAVA_HOME if not defined
 if [ -e /usr/libexec/bigtop-detect-javahome ]; then
@@ -43,92 +41,158 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-d
   . /usr/lib/bigtop-utils/bigtop-detect-javahome
 fi
 
-RETVAL_SUCCESS=0
-
-STATUS_RUNNING=0
-STATUS_DEAD=1
-STATUS_DEAD_AND_LOCK=2
-STATUS_NOT_RUNNING=3
-
-ERROR_PROGRAM_NOT_INSTALLED=5
-ERROR_PROGRAM_NOT_CONFIGURED=6
+# Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh
 
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 NAME="hive-@HIVE_DAEMON@"
-DESC="Hive @HIVE_DAEMON@ daemon"
 SYS_FILE="/etc/default/${NAME}"
 EXE_FILE="/usr/lib/hive/bin/hive"
 PID_FILE="/var/run/hive/${NAME}.pid"
 LOCKFILE="/var/lock/subsys/${NAME}"
 LOG_FILE="/var/log/hive/${NAME}.log"
 HIVE_USER="hive"
-HIVE_HOME="`eval echo ~$HIVE_USER`"
-NICENESS="+0"
-TIMEOUT=3
+NICENESS="0"
+DESC="Hive daemon"
+ 
+DODTIME=3
+SLAVE_TIMEOUT=300
 
 [ -f $SYS_FILE ] && . $SYS_FILE
 
+hive_die() {
+    echo "$@"
+    exit 1
+}
+hive_is_process_alive() {
+    local pid="$1"
+    kill -0 $pid > /dev/null 2>&1
+}
+hive_check_pidfile() {
+    local pidfile="$1" # IN
+    local pid
+
+    pid=`cat "$pidfile" 2>/dev/null`
+    if [ "$pid" = '' ]; then
+    # The file probably does not exist or is empty. 
+	return 1
+    fi
+    
+    set -- $pid
+    pid="$1"
+
+    hive_is_process_alive $pid
+}
+hive_process_kill() {
+    local pid="$1"    # IN
+    local signal="$2" # IN
+    local second
+
+    kill -$signal $pid 2>/dev/null
+
+    for second in 0 1 2 3 4 5 6 7 8 9 10; do
+      hive_is_process_alive "$pid" || return 0
+      sleep 1
+    done
+
+    return 1
+}
+hive_stop_pidfile() {
+    local pidfile="$1" # IN
+    local pid
+
+    pid=`cat "$pidfile" 2>/dev/null`
+    if [ "$pid" = '' ]; then
+      # The file probably does not exist or is empty. Success
+	return 0
+    fi
+    
+    set -- $pid
+    pid="$1"
+
+    # First try the easy way
+    if hive_process_kill "$pid" 15; then
+	return 0
+    fi
+
+    # Otherwise try the hard way
+    if hive_process_kill "$pid" 9; then
+	return 0
+    fi
+
+    return 1
+}
+
 hive_start() {
+    ERROR_PROGRAM_NOT_INSTALLED=5
+
     [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+    log_success_msg "Starting $desc (${NAME}): "
+
+    if hive_check_pidfile $PID_FILE ; then
+      exit 0
+    fi
 
     service_name="@HIVE_DAEMON@"
     if [ $service_name = "server" ] ; then
       service_name="hiveserver"
       exec_env="HADOOP_OPTS=\"-Dhive.log.dir=`dirname $LOG_FILE`\""
     fi
-
-    log_success_msg "Starting $desc (${NAME}): "
-    start_daemon -u $HIVE_USER -p $PID_FILE -n $NICENESS  /bin/sh -c "cd $HIVE_HOME ; $exec_env nohup \
+    su -s /bin/sh $HIVE_USER \
+       -c "$exec_env nohup nice -n $NICENESS       \
            $EXE_FILE --service $service_name $PORT \
              > $LOG_FILE 2>&1 < /dev/null & "'echo $! '"> $PID_FILE"
+    sleep 3
 
+    hive_check_pidfile $PID_FILE
     RETVAL=$?
-    [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE
+    echo
+    [ $RETVAL -eq 0 ] && touch $LOCKFILE
     return $RETVAL
 }
-
 hive_stop() {
     log_success_msg "Stopping $desc (${NAME}): "
-    killproc -p $PID_FILE java
-    RETVAL=$?
-
-    [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PID_FILE
-    return $RETVAL
+    if [ -f $PID_FILE ]; then
+      hive_stop_pidfile $PID_FILE
+      RETVAL=$?
+      echo
+      [ $RETVAL -eq 0 ] && rm -f $LOCKFILE $PID_FILE
+    fi
 }
-
 hive_restart() {
     hive_stop
-    [ -n "$TIMEOUT" ] && sleep $TIMEOUT
+    [ -n "$DODTIME" ] && sleep $DODTIME
     hive_start
 }
-
 hive_status() {
-    echo -n "Checking for service $desc: "
+    local STATUS_RUNNING=0
+    local STATUS_DEAD=1
+    local STATUS_DEAD_AND_LOCK=2
+    local STATUS_NOT_RUNNING=3
+    local DAEMON="${NAME}"
+
     pidofproc -p $PID_FILE java > /dev/null
     status=$?
 
     case "$status" in
       $STATUS_RUNNING)
-        log_success_msg "@HIVE_DAEMON@ is running"
+        log_success_msg "$DAEMON is running"
         ;;
       $STATUS_DEAD)
-        log_failure_msg "@HIVE_DAEMON@ is dead and pid file exists"
+        log_failure_msg "$DAEMON is dead and pid file exists"
         ;;
       $STATUS_DEAD_AND_LOCK)
-        log_failure_msg "@HIVE_DAEMON@ is dead and lock file exists"
+        log_failure_msg "$DAEMON is dead and lock file exists"
         ;;
       $STATUS_NOT_RUNNING)
-        log_failure_msg "@HIVE_DAEMON@ is not running"
+        log_failure_msg "$DAEMON is not running"
         ;;
       *)
-        log_failure_msg "@HIVE_DAEMON@ status is unknown"
+        log_failure_msg "$DAEMON status is unknown"
         ;;
     esac
     return $status
 }
 
-RETVAL=0
-
 case "$1" in
     start)
       hive_start

Propchange: incubator/bigtop/trunk/bigtop-packages/src/common/hive/hive.sh.suse
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hive/install_hive.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hive/install_hive.sh?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hive/install_hive.sh (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hive/install_hive.sh Mon Apr 16 16:10:22 2012
@@ -119,7 +119,6 @@ do
   wrapper=$BIN_DIR/$file
   cat >>$wrapper <<EOF
 #!/bin/sh
-. /etc/default/hadoop
 
 # Autodetect JAVA_HOME if not defined
 if [ -e /usr/libexec/bigtop-detect-javahome ]; then
@@ -128,6 +127,9 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-d
   . /usr/lib/bigtop-utils/bigtop-detect-javahome
 fi
 
+#FIXME: workaround for HIVE-2757 (NOTE: HADOOP_HOME is obsolete for Hadoop .23)
+export HADOOP_HOME=\${HADOOP_HOME:-/usr/lib/hadoop}
+
 export HIVE_HOME=$INSTALLED_LIB_DIR
 exec $INSTALLED_LIB_DIR/bin/$file "\$@"
 EOF



Mime
View raw message