hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject svn commit: r1618847 - in /hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn: bin/slaves.sh bin/start-yarn.sh bin/stop-yarn.sh bin/yarn bin/yarn-config.sh bin/yarn-daemon.sh bin/yarn-daemons.sh conf/yarn-env.sh
Date Tue, 19 Aug 2014 12:11:19 GMT
Author: aw
Date: Tue Aug 19 12:11:17 2014
New Revision: 1618847

URL: http://svn.apache.org/r1618847
Log:
HADOOP-9902. Shell script rewrite (aw)

Removed:
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/slaves.sh
Modified:
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh?rev=1618847&r1=1618846&r2=1618847&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh Tue Aug 19 12:11:17
2014
@@ -16,20 +16,34 @@
 # limitations under the License.
 
 
-# Start all yarn daemons.  Run this on master node.
-
-echo "starting yarn daemons"
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
+function hadoop_usage
+{
+  echo "Usage: start-yarn.sh [--config confdir]"
+}
+
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+  DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+  DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
+  exit 1
+fi
 
 # start resourceManager
-"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR  start resourcemanager
+"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}"  start resourcemanager
 # start nodeManager
-"$bin"/yarn-daemons.sh --config $YARN_CONF_DIR  start nodemanager
+"${bin}/yarn-daemons.sh" --config "${YARN_CONF_DIR}"  start nodemanager
 # start proxyserver
-#"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR  start proxyserver
+#"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}"  start proxyserver

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh?rev=1618847&r1=1618846&r2=1618847&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh Tue Aug 19 12:11:17
2014
@@ -18,18 +18,34 @@
 
 # Stop all yarn daemons.  Run this on master node.
 
-echo "stopping yarn daemons"
+function hadoop_usage
+{
+  echo "Usage: stop-yarn.sh [--config confdir]"
+}
 
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
 
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
-
-# stop resourceManager
-"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR  stop resourcemanager
-# stop nodeManager
-"$bin"/yarn-daemons.sh --config $YARN_CONF_DIR  stop nodemanager
-# stop proxy server
-"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR  stop proxyserver
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+  DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+  DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
+  exit 1
+fi
+
+# start resourceManager
+"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}"  stop resourcemanager
+# start nodeManager
+"${bin}/yarn-daemons.sh" --config "${YARN_CONF_DIR}"  stop nodemanager
+# start proxyserver
+#"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}"  stop proxyserver

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1618847&r1=1618846&r2=1618847&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn Tue Aug 19 12:11:17 2014
@@ -15,266 +15,182 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-# The Hadoop command script
-#
-# Environment Variables
-#
-#   JAVA_HOME        The java implementation to use.  Overrides JAVA_HOME.
-#
-#   YARN_USER_CLASSPATH Additional user CLASSPATH entries.
-#
-#   YARN_USER_CLASSPATH_FIRST  If set to non empty value then the user classpath
-#                              specified in YARN_USER_CLASSPATH will be
-#                              appended at the beginning of YARN's final
-#                              classpath instead of at the end.
-#
-#   YARN_HEAPSIZE  The maximum amount of heap to use, in MB. 
-#                    Default is 1000.
-#
-#   YARN_{COMMAND}_HEAPSIZE overrides YARN_HEAPSIZE for a given command
-#                           eg YARN_NODEMANAGER_HEAPSIZE sets the heap
-#                           size for the NodeManager.  If you set the
-#                           heap size in YARN_{COMMAND}_OPTS or YARN_OPTS
-#                           they take precedence.
-#
-#   YARN_OPTS      Extra Java runtime options.
-#   
-#   YARN_CLIENT_OPTS         when the respective command is run.
-#   YARN_{COMMAND}_OPTS etc  YARN_NODEMANAGER_OPTS applies to NodeManager 
-#                              for e.g.  YARN_CLIENT_OPTS applies to 
-#                              more than one command (fs, dfs, fsck, 
-#                              dfsadmin etc)  
-#
-#   YARN_CONF_DIR  Alternate conf dir. Default is ${HADOOP_YARN_HOME}/conf.
-#
-#   YARN_ROOT_LOGGER The root appender. Default is INFO,console
-#
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin" > /dev/null; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
-
-function print_usage(){
-  echo "Usage: yarn [--config confdir] COMMAND"
+function hadoop_usage
+{
+  echo "Usage: yarn [--config confdir] [--daemon (start|stop|status)] COMMAND"
   echo "where COMMAND is one of:"
-  echo "  resourcemanager -format-state-store   deletes the RMStateStore"
-  echo "  resourcemanager                       run the ResourceManager"
+  echo "  application                           prints application(s) report/kill application"
+  echo "  applicationattempt                    prints applicationattempt(s) report"
+  echo "  classpath                             prints the class path needed to get the"
+  echo "                                        Hadoop jar and the required libraries"
+  echo "  container                             prints container(s) report"
+  echo "  daemonlog                             get/set the log level for each daemon"
+  echo "  jar <jar>                             run a jar file"
+  echo "  logs                                  dump container logs"
+  echo "  node                                  prints node report(s)"
   echo "  nodemanager                           run a nodemanager on each slave"
-  echo "  timelineserver                        run the timeline server"
+  echo "  proxyserver                           run the web app proxy server"
+  echo "  resourcemanager                       run the ResourceManager"
+  echo "  resourcemanager -format-state-store   deletes the RMStateStore"
   echo "  rmadmin                               admin tools"
+  echo "  timelineserver                        run the timeline server"
   echo "  version                               print the version"
-  echo "  jar <jar>                             run a jar file"
-  echo "  application                           prints application(s)"
-  echo "                                        report/kill application"
-  echo "  applicationattempt                    prints applicationattempt(s)"
-  echo "                                        report"
-  echo "  container                             prints container(s) report"
-  echo "  node                                  prints node report(s)"
-  echo "  logs                                  dump container logs"
-  echo "  classpath                             prints the class path needed to"
-  echo "                                        get the Hadoop jar and the"
-  echo "                                        required libraries"
-  echo "  daemonlog                             get/set the log level for each"
-  echo "                                        daemon"
   echo " or"
   echo "  CLASSNAME                             run the class named CLASSNAME"
   echo "Most commands print help when invoked w/o parameters."
 }
 
-# if no args specified, show usage
-if [ $# = 0 ]; then
-  print_usage
+
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+  DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+  this="${BASH_SOURCE-$0}"
+  bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+  DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
   exit 1
 fi
 
+# if no args specified, show usage
+if [[ $# = 0 ]]; then
+  hadoop_exit_with_usage 1
+fi
+
 # get arguments
 COMMAND=$1
 shift
 
-case $COMMAND in
-  # usage flags
-  --help|-help|-h)
-    print_usage
+case "${COMMAND}" in
+  application|applicationattempt|container)
+    CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
+    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+  ;;
+  classpath)
+    hadoop_finalize
+    echo "${CLASSPATH}"
     exit
-    ;;
+  ;;
+  daemonlog)
+    CLASS=org.apache.hadoop.log.LogLevel
+    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+  ;;
+  jar)
+    CLASS=org.apache.hadoop.util.RunJar
+    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+  ;;
+  historyserver)
+    daemon="true"
+    echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
+    echo "Instead use the timelineserver command for it." 1>&2
+    echo "Starting the History Server anyway..." 1>&2
+    CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+  ;;
+  logs)
+    CLASS=org.apache.hadoop.yarn.logaggregation.LogDumper
+    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+  ;;
+  node)
+    CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
+    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+  ;;
+  nodemanager)
+    daemon="true"
+    CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
+    YARN_OPTS="${YARN_OPTS} ${YARN_NODEMANAGER_OPTS}"
+    if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
+      JAVA_HEAP_MAX="-Xmx${YARN_NODEMANAGER_HEAPSIZE}m"
+    fi
+  ;;
+  proxyserver)
+    daemon="true"
+    CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
+    YARN_OPTS="${YARN_OPTS} ${YARN_PROXYSERVER_OPTS}"
+    if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
+      JAVA_HEAP_MAX="-Xmx${YARN_PROXYSERVER_HEAPSIZE}m"
+    fi
+  ;;
+  resourcemanager)
+    daemon="true"
+    CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
+    YARN_OPTS="${YARN_OPTS} ${YARN_RESOURCEMANAGER_OPTS}"
+    if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
+      JAVA_HEAP_MAX="-Xmx${YARN_RESOURCEMANAGER_HEAPSIZE}m"
+    fi
+  ;;
+  rmadmin)
+    CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
+    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+  ;;
+  timelineserver)
+    daemon="true"
+    CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+    YARN_OPTS="${YARN_OPTS} ${YARN_TIMELINESERVER_OPTS}"
+    if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
+      JAVA_HEAP_MAX="-Xmx${YARN_TIMELINESERVER_HEAPSIZE}m"
+    fi
+  ;;
+  version)
+    CLASS=org.apache.hadoop.util.VersionInfo
+    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+  ;;
+  -*)
+    hadoop_exit_with_usage 1
+  ;;
+  *)
+    CLASS="${COMMAND}"
+  ;;
 esac
 
-if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then
-  . "${YARN_CONF_DIR}/yarn-env.sh"
-fi
+# set HADOOP_OPTS to YARN_OPTS so that we can use
+# finalize, etc, without doing anything funky
+HADOOP_OPTS="${YARN_OPTS}"
 
-# some Java parameters
-if [ "$JAVA_HOME" != "" ]; then
-  #echo "run java in $JAVA_HOME"
-  JAVA_HOME=$JAVA_HOME
-fi
-  
-if [ "$JAVA_HOME" = "" ]; then
-  echo "Error: JAVA_HOME is not set."
-  exit 1
-fi
-
-JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m 
+daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
+daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
 
-# check envvars which might override default args
-if [ "$YARN_HEAPSIZE" != "" ]; then
-  #echo "run with heapsize $YARN_HEAPSIZE"
-  JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
-  #echo $JAVA_HEAP_MAX
+if [[  "${HADOOP_DAEMON_MODE}" != "default" ]]; then
+  # shellcheck disable=SC2034
+  HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
+  YARN_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
+  HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
 fi
 
-# CLASSPATH initially contains $HADOOP_CONF_DIR & $YARN_CONF_DIR
-if [ ! -d "$HADOOP_CONF_DIR" ]; then
-  echo No HADOOP_CONF_DIR set. 
-  echo Please specify it either in yarn-env.sh or in the environment.
-  exit 1
-fi
+hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
 
-CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}:${CLASSPATH}"
+# Add YARN custom options to comamnd line in case someone actaully 
+# used these.
+#
+# Note that we are replacing ' ' with '\ ' so that when we exec
+# stuff it works
+#
+hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${HADOOP_LOG_DIR/ /\ }"
+hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE/ /\ }"
+hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${HADOOP_YARN_HOME/ /\ }"
+hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
 
-# for developers, add Hadoop classes to CLASSPATH
-if [ -d "$HADOOP_YARN_HOME/yarn-api/target/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-api/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-common/target/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-common/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-mapreduce/target/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-mapreduce/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-master-worker/target/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-master-worker/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-common/target/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-common/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes"
]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/build/test/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/target/test/classes
-fi
-if [ -d "$HADOOP_YARN_HOME/build/tools" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/build/tools
-fi
+hadoop_finalize
 
-CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/${YARN_DIR}/*
-CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/${YARN_LIB_JARS_DIR}/*
+export CLASSPATH
 
-# Add user defined YARN_USER_CLASSPATH to the class path (if defined)
-if [ -n "$YARN_USER_CLASSPATH" ]; then
-  if [ -n "$YARN_USER_CLASSPATH_FIRST" ]; then
-    # User requested to add the custom entries at the beginning
-    CLASSPATH=${YARN_USER_CLASSPATH}:${CLASSPATH}
+if [[ -n "${daemon}" ]]; then
+  if [[ -n "${secure_service}" ]]; then
+    hadoop_secure_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" \
+    "${CLASS}" "${daemon_pidfile}" "${daemon_outfile}" \
+    "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@"
   else
-    # By default we will just append the extra entries at the end
-    CLASSPATH=${CLASSPATH}:${YARN_USER_CLASSPATH}
+    hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}" \
+    "${daemon_pidfile}" "${daemon_outfile}" "$@"
   fi
-fi
-
-# so that filenames w/ spaces are handled correctly in loops below
-IFS=
-
-# default log directory & file
-if [ "$YARN_LOG_DIR" = "" ]; then
-  YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-fi
-if [ "$YARN_LOGFILE" = "" ]; then
-  YARN_LOGFILE='yarn.log'
-fi
-
-# restore ordinary behaviour
-unset IFS
-
-# figure out which class to run
-if [ "$COMMAND" = "classpath" ] ; then
-  echo $CLASSPATH
-  exit
-elif [ "$COMMAND" = "rmadmin" ] ; then
-  CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
-  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "application" ] || 
-     [ "$COMMAND" = "applicationattempt" ] || 
-     [ "$COMMAND" = "container" ]; then
-  CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
-  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-  set -- $COMMAND $@
-elif [ "$COMMAND" = "node" ] ; then
-  CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
-  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "resourcemanager" ] ; then
-  CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
-  CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
-  YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS"
-  if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then
-    JAVA_HEAP_MAX="-Xmx""$YARN_RESOURCEMANAGER_HEAPSIZE""m"
-  fi
-elif [ "$COMMAND" = "historyserver" ] ; then
-  echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
-  echo "Instead use the timelineserver command for it." 1>&2
-  CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/ahs-config/log4j.properties
-  CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
-  YARN_OPTS="$YARN_OPTS $YARN_HISTORYSERVER_OPTS"
-  if [ "$YARN_HISTORYSERVER_HEAPSIZE" != "" ]; then
-    JAVA_HEAP_MAX="-Xmx""$YARN_HISTORYSERVER_HEAPSIZE""m"
-  fi
-elif [ "$COMMAND" = "timelineserver" ] ; then
-  CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/timelineserver-config/log4j.properties
-  CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
-  YARN_OPTS="$YARN_OPTS $YARN_TIMELINESERVER_OPTS"
-  if [ "$YARN_TIMELINESERVER_HEAPSIZE" != "" ]; then
-    JAVA_HEAP_MAX="-Xmx""$YARN_TIMELINESERVER_HEAPSIZE""m"
-  fi
-elif [ "$COMMAND" = "nodemanager" ] ; then
-  CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties
-  CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
-  YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS"
-  if [ "$YARN_NODEMANAGER_HEAPSIZE" != "" ]; then
-    JAVA_HEAP_MAX="-Xmx""$YARN_NODEMANAGER_HEAPSIZE""m"
-  fi
-elif [ "$COMMAND" = "proxyserver" ] ; then
-  CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
-  YARN_OPTS="$YARN_OPTS $YARN_PROXYSERVER_OPTS"
-  if [ "$YARN_PROXYSERVER_HEAPSIZE" != "" ]; then
-    JAVA_HEAP_MAX="-Xmx""$YARN_PROXYSERVER_HEAPSIZE""m"
-  fi
-elif [ "$COMMAND" = "version" ] ; then
-  CLASS=org.apache.hadoop.util.VersionInfo
-  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "jar" ] ; then
-  CLASS=org.apache.hadoop.util.RunJar
-  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "logs" ] ; then
-  CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI
-  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "daemonlog" ] ; then
-  CLASS=org.apache.hadoop.log.LogLevel
-  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
+  exit $?
 else
-  CLASS=$COMMAND
+  hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
 fi
-
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
-YARN_OPTS="$YARN_OPTS -Dhadoop.home.dir=$HADOOP_YARN_HOME"
-YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
-  YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
-fi  
-
-exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $YARN_OPTS -classpath "$CLASSPATH" $CLASS "$@"

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh?rev=1618847&r1=1618846&r2=1618847&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh Tue Aug 19 12:11:17
2014
@@ -13,53 +13,81 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# included in all the hadoop scripts with source command
-# should not be executable directly
-bin=`which "$0"`
-bin=`dirname "${bin}"`
-bin=`cd "$bin"; pwd`
+function hadoop_subproject_init
+{
+  
+  # at some point in time, someone thought it would be a good idea to
+  # create separate vars for every subproject.  *sigh*
+  # let's perform some overrides and setup some defaults for bw compat
+  # this way the common hadoop var's == subproject vars and can be
+  # used interchangeable from here on out
+  # ...
+  # this should get deprecated at some point.
+  
+  if [[ -e "${YARN_CONF_DIR}/yarn-env.sh" ]]; then
+    . "${YARN_CONF_DIR}/yarn-env.sh"
+  elif [[ -e "${HADOOP_CONF_DIR}/yarn-env.sh" ]]; then
+    . "${HADOOP_CONF_DIR}/yarn-env.sh"
+  fi
+  
+  if [[ -n "${YARN_CONF_DIR}" ]]; then
+    HADOOP_CONF_DIR="${YARN_CONF_DIR}"
+  fi
+  
+  YARN_CONF_DIR="${HADOOP_CONF_DIR}"
+  
+  # YARN_CONF_DIR needs precedence over HADOOP_CONF_DIR
+  # and the various jar dirs
+  hadoop_add_classpath "${YARN_CONF_DIR}" before
+  
+  HADOOP_LOG_DIR="${YARN_LOG_DIR:-$HADOOP_LOG_DIR}"
+  YARN_LOG_DIR="${HADOOP_LOG_DIR}"
+  
+  HADOOP_LOGFILE="${YARN_LOGFILE:-$HADOOP_LOGFILE}"
+  YARN_LOGFILE="${HADOOP_LOGFILE}"
+  
+  HADOOP_NICENESS="${YARN_NICENESS:-$HADOOP_NICENESS}"
+  YARN_NICENESS="${HADOOP_NICENESS}"
+  
+  HADOOP_STOP_TIMEOUT="${YARN_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT}"
+  YARN_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}"
+  
+  HADOOP_PID_DIR="${YARN_PID_DIR:-$HADOOP_PID_DIR}"
+  YARN_PID_DIR="${HADOOP_PID_DIR}"
+  
+  HADOOP_ROOT_LOGGER="${YARN_ROOT_LOGGER:-INFO,console}"
+  YARN_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
+  
+  HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_PREFIX}"
+  
+  HADOOP_IDENT_STRING="${YARN_IDENT_STRING:-$HADOOP_IDENT_STRING}"
+  YARN_IDENT_STRING="${HADOOP_IDENT_STRING}"
+  
+  YARN_OPTS="${YARN_OPTS:-$HADOOP_OPTS}"
+  
+  # YARN-1429 added the completely superfluous YARN_USER_CLASSPATH
+  # env var.  We're going to override HADOOP_USER_CLASSPATH to keep
+  # consistency with the rest of the duplicate/useless env vars
+  HADOOP_USER_CLASSPATH="${YARN_USER_CLASSPATH:-$HADOOP_USER_CLASSPATH}"
+  YARN_USER_CLASSPATH="${HADOOP_USER_CLASSPATH}"
+  
+  HADOOP_USER_CLASSPATH_FIRST="${YARN_USER_CLASSPATH_FIRST:-$HADOOP_USER_CLASSPATH_FIRST}"
+  YARN_USER_CLASSPATH_FIRST="${HADOOP_USER_CLASSPATH_FIRST}"
+}
 
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then
-  . ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh
-elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then
-  . "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh
-elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
-  . "$HADOOP_HOME"/libexec/hadoop-config.sh
+if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
+  _yc_this="${BASH_SOURCE-$0}"
+  HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_yc_this}")" >/dev/null && pwd
-P)
+fi
+
+if [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
+  . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then
+  . "${HADOOP_HOME}/libexec/hadoop-config.sh"
 else
   echo "Hadoop common not found."
   exit
 fi
 
-# Same glibc bug that discovered in Hadoop.
-# Without this you can see very large vmem settings on containers.
-export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
-
-#check to see if the conf dir is given as an optional argument
-if [ $# -gt 1 ]
-then
-    if [ "--config" = "$1" ]
-	  then
-	      shift
-	      confdir=$1
-	      shift
-	      YARN_CONF_DIR=$confdir
-    fi
-fi
- 
-# Allow alternate conf dir location.
-export YARN_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
-
-#check to see it is specified whether to use the slaves or the
-# masters file
-if [ $# -gt 1 ]
-then
-    if [ "--hosts" = "$1" ]
-    then
-        shift
-        slavesfile=$1
-        shift
-        export YARN_SLAVES="${YARN_CONF_DIR}/$slavesfile"
-    fi
-fi

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh?rev=1618847&r1=1618846&r2=1618847&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh Tue Aug 19 12:11:17
2014
@@ -15,147 +15,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-# Runs a yarn command as a daemon.
-#
-# Environment Variables
-#
-#   YARN_CONF_DIR  Alternate conf dir. Default is ${HADOOP_YARN_HOME}/conf.
-#   YARN_LOG_DIR   Where log files are stored.  PWD by default.
-#   YARN_MASTER    host:path where hadoop code should be rsync'd from
-#   YARN_PID_DIR   The pid files are stored. /tmp by default.
-#   YARN_IDENT_STRING   A string representing this instance of hadoop. $USER by default
-#   YARN_NICENESS The scheduling priority for daemons. Defaults to 0.
-##
-
-usage="Usage: yarn-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop)
<yarn-command> "
-
-# if no args specified, show usage
-if [ $# -le 1 ]; then
-  echo $usage
-  exit 1
-fi
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
-
-# get arguments
-startStop=$1
-shift
-command=$1
-shift
-
-hadoop_rotate_log ()
+function hadoop_usage
 {
-    log=$1;
-    num=5;
-    if [ -n "$2" ]; then
-	num=$2
-    fi
-    if [ -f "$log" ]; then # rotate logs
-	while [ $num -gt 1 ]; do
-	    prev=`expr $num - 1`
-	    [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
-	    num=$prev
-	done
-	mv "$log" "$log.$num";
-    fi
+  echo "Usage: yarn-daemon.sh [--config confdir] (start|stop|status) <hadoop-command>
<args...>"
 }
 
-if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then
-  . "${YARN_CONF_DIR}/yarn-env.sh"
-fi
-
-if [ "$YARN_IDENT_STRING" = "" ]; then
-  export YARN_IDENT_STRING="$USER"
-fi
-
-# get log directory
-if [ "$YARN_LOG_DIR" = "" ]; then
-  export YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-fi
-
-if [ ! -w "$YARN_LOG_DIR" ] ; then
-  mkdir -p "$YARN_LOG_DIR"
-  chown $YARN_IDENT_STRING $YARN_LOG_DIR 
-fi
-
-if [ "$YARN_PID_DIR" = "" ]; then
-  YARN_PID_DIR=/tmp
-fi
-
-# some variables
-export YARN_LOGFILE=yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.log
-export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,RFA}
-log=$YARN_LOG_DIR/yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.out
-pid=$YARN_PID_DIR/yarn-$YARN_IDENT_STRING-$command.pid
-YARN_STOP_TIMEOUT=${YARN_STOP_TIMEOUT:-5}
-
-# Set default scheduling priority
-if [ "$YARN_NICENESS" = "" ]; then
-    export YARN_NICENESS=0
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+  DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+  this="${BASH_SOURCE-$0}"
+  bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+  DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
+  exit 1
 fi
 
-case $startStop in
-
-  (start)
-
-    [ -w "$YARN_PID_DIR" ] || mkdir -p "$YARN_PID_DIR"
-
-    if [ -f $pid ]; then
-      if kill -0 `cat $pid` > /dev/null 2>&1; then
-        echo $command running as process `cat $pid`.  Stop it first.
-        exit 1
-      fi
-    fi
-
-    if [ "$YARN_MASTER" != "" ]; then
-      echo rsync from $YARN_MASTER
-      rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*'
$YARN_MASTER/ "$HADOOP_YARN_HOME"
-    fi
-
-    hadoop_rotate_log $log
-    echo starting $command, logging to $log
-    cd "$HADOOP_YARN_HOME"
-    nohup nice -n $YARN_NICENESS "$HADOOP_YARN_HOME"/bin/yarn --config $YARN_CONF_DIR $command
"$@" > "$log" 2>&1 < /dev/null &
-    echo $! > $pid
-    sleep 1
-    head "$log"
-    # capture the ulimit output
-    echo "ulimit -a" >> $log
-    ulimit -a >> $log 2>&1
-    ;;
-          
-  (stop)
-
-    if [ -f $pid ]; then
-      TARGET_PID=`cat $pid`
-      if kill -0 $TARGET_PID > /dev/null 2>&1; then
-        echo stopping $command
-        kill $TARGET_PID
-        sleep $YARN_STOP_TIMEOUT
-        if kill -0 $TARGET_PID > /dev/null 2>&1; then
-          echo "$command did not stop gracefully after $YARN_STOP_TIMEOUT seconds: killing
with kill -9"
-          kill -9 $TARGET_PID
-        fi
-      else
-        echo no $command to stop
-      fi
-      rm -f $pid
-    else
-      echo no $command to stop
-    fi
-    ;;
-
-  (*)
-    echo $usage
-    exit 1
-    ;;
-
-esac
-
+daemonmode=$1
+shift
 
+exec "${HADOOP_YARN_HOME}/bin/yarn" \
+--config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh?rev=1618847&r1=1618846&r2=1618847&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh Tue Aug 19 12:11:17
2014
@@ -16,23 +16,31 @@
 # limitations under the License.
 
 
-# Run a Yarn command on all slave hosts.
-
-usage="Usage: yarn-daemons.sh [--config confdir] [--hosts hostlistfile] [start
-|stop] command args..."
+function hadoop_usage
+{
+  echo "Usage: yarn-daemons.sh [--config confdir] [--hosts hostlistfile] (start|stop|status)
<yarn-command> <args...>"
+}
+
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+  DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+  DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
 
-# if no args specified, show usage
-if [ $# -le 1 ]; then
-  echo $usage
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
+else
+  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
   exit 1
 fi
 
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
-
-exec "$bin/slaves.sh" --config $YARN_CONF_DIR cd "$HADOOP_YARN_HOME" \; "$bin/yarn-daemon.sh"
--config $YARN_CONF_DIR "$@"
+hadoop_connect_to_hosts "${bin}/yarn-daemon.sh" \
+--config "${HADOOP_CONF_DIR}" "$@"
 

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh?rev=1618847&r1=1618846&r2=1618847&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh Tue Aug 19 12:11:17
2014
@@ -13,118 +13,115 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+##
+## THIS FILE ACTS AS AN OVERRIDE FOR hadoop-env.sh FOR ALL
+## WORK DONE BY THE yarn AND RELATED COMMANDS.
+##
+## Precedence rules:
+##
+## yarn-env.sh > hadoop-env.sh > hard-coded defaults
+##
+## YARN_xyz > HADOOP_xyz > hard-coded defaults
+##
+
+###
+# Generic settings for YARN
+###
+
 # User for YARN daemons
 export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
 
-# resolve links - $0 may be a softlink
-export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+#
+# By default, YARN will use HADOOP_CONF_DIR. Specify a custom
+# YARN_CONF_DIR here
+# export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+#
+
+# Override Hadoop's log directory & file
+# export YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
+# export YARN_LOGFILE='yarn.log'
+
+# Need a custom-to-YARN service-level authorization policy file?
+# export YARN_POLICYFILE="yarn-policy.xml"
 
-# some Java parameters
-# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
-if [ "$JAVA_HOME" != "" ]; then
-  #echo "run java in $JAVA_HOME"
-  JAVA_HOME=$JAVA_HOME
-fi
-  
-if [ "$JAVA_HOME" = "" ]; then
-  echo "Error: JAVA_HOME is not set."
-  exit 1
-fi
-
-JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m 
-
-# For setting YARN specific HEAP sizes please use this
-# Parameter and set appropriately
-# YARN_HEAPSIZE=1000
-
-# check envvars which might override default args
-if [ "$YARN_HEAPSIZE" != "" ]; then
-  JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
-fi
+#Override the log4j settings for all YARN apps
+# export YARN_ROOT_LOGGER="INFO,console"
 
+###
 # Resource Manager specific parameters
+###
 
-# Specify the max Heapsize for the ResourceManager using a numerical value
+# Specify the max heapsize for the ResourceManager using a numerical value
 # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
 # the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_RESOURCEMANAGER_OPTS.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
+# HADOOP_OPTS, and/or YARN_RESOURCEMANAGER_OPTS.
 # If not specified, the default value will be picked from either YARN_HEAPMAX
 # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#
 #export YARN_RESOURCEMANAGER_HEAPSIZE=1000
 
-# Specify the max Heapsize for the timeline server using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_TIMELINESERVER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-#export YARN_TIMELINESERVER_HEAPSIZE=1000
-
 # Specify the JVM options to be used when starting the ResourceManager.
 # These options will be appended to the options specified as YARN_OPTS
 # and therefore may override any similar flags set in YARN_OPTS
-#export YARN_RESOURCEMANAGER_OPTS=
+#
+# Examples for a Sun/Oracle JDK:
+# a) override the appsummary log file:
+# export YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log
-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY"
+#
+# b) Set JMX options
+# export YARN_RESOURCEMANAGER_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
+#
+# c) Set garbage collection logs from hadoop-env.sh
+# export YARN_RESOURCE_MANAGER_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date
+'%Y%m%d%H%M')"
+#
+# d) ... or set them directly
+# export YARN_RESOURCEMANAGER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
+#
+#
+# export YARN_RESOURCEMANAGER_OPTS=
 
+###
 # Node Manager specific parameters
+###
 
 # Specify the max Heapsize for the NodeManager using a numerical value
 # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
 # the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_NODEMANAGER_OPTS.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
+# HADOOP_OPTS, and/or YARN_NODEMANAGER_OPTS.
 # If not specified, the default value will be picked from either YARN_HEAPMAX
 # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#
 #export YARN_NODEMANAGER_HEAPSIZE=1000
 
 # Specify the JVM options to be used when starting the NodeManager.
 # These options will be appended to the options specified as YARN_OPTS
 # and therefore may override any similar flags set in YARN_OPTS
+#
+# See ResourceManager for some examples
+#
 #export YARN_NODEMANAGER_OPTS=
 
-# so that filenames w/ spaces are handled correctly in loops below
-IFS=
+###
+# TimeLineServer specifc parameters
+###
 
+# Specify the max Heapsize for the timeline server using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
+# HADOOP_OPTS, and/or YARN_TIMELINESERVER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#
+#export YARN_TIMELINESERVER_HEAPSIZE=1000
 
-# default log directory & file
-if [ "$YARN_LOG_DIR" = "" ]; then
-  YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-fi
-if [ "$YARN_LOGFILE" = "" ]; then
-  YARN_LOGFILE='yarn.log'
-fi
-
-# default policy file for service-level authorization
-if [ "$YARN_POLICYFILE" = "" ]; then
-  YARN_POLICYFILE="hadoop-policy.xml"
-fi
-
-# restore ordinary behaviour
-unset IFS
-
-MAC_OSX=false
-case "`uname`" in
-Darwin*) MAC_OSX=true;;
-esac
-
-if $MAC_OSX; then
-  YARN_OPTS="$YARN_OPTS -Djava.security.krb5.realm= -Djava.security.krb5.kdc="
-fi
-
-
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
-YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
-YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
-  YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
-fi  
-YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
-
+# Specify the JVM options to be used when starting the TimeLineServer.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_TIMELINESERVER_OPTS=
 



Mime
View raw message