hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [1/2] hadoop git commit: HADOOP-12930. Dynamic subcommands for hadoop shell scripts (aw)
Date Tue, 17 May 2016 00:56:17 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 6a6e74acf -> 730bc746f


http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-gridmix/src/main/shellprofile.d/hadoop-gridmix.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/shellprofile.d/hadoop-gridmix.sh b/hadoop-tools/hadoop-gridmix/src/main/shellprofile.d/hadoop-gridmix.sh
new file mode 100755
index 0000000..b7887ba
--- /dev/null
+++ b/hadoop-tools/hadoop-gridmix/src/main/shellprofile.d/hadoop-gridmix.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f hadoop_subcommand_gridmix >/dev/null 2>/dev/null; then
+
+  if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+    hadoop_add_subcommand "gridmix" "submit a mix of synthetic job, modeling a profiled from
production load"
+  fi
+
+## @description  gridmix command for hadoop
+## @audience     public
+## @stability    stable
+## @replaceable  yes
+function hadoop_subcommand_gridmix
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.mapred.gridmix.Gridmix
+  hadoop_add_to_classpath_tools hadoop-rumen
+  hadoop_add_to_classpath_tools hadoop-gridmix
+}
+
+fi

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh b/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
new file mode 100755
index 0000000..d7d4022
--- /dev/null
+++ b/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f hadoop_subcommand_rumenfolder >/dev/null 2>/dev/null; then
+
+  if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+    hadoop_add_subcommand "rumenfolder" "scale a rumen input trace"
+  fi
+
+## @description  rumenfolder command for hadoop
+## @audience     public
+## @stability    stable
+## @replaceable  yes
+function hadoop_subcommand_rumenfolder
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.rumen.Folder
+  hadoop_add_to_classpath_tools hadoop-rumen
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+fi
+
+if ! declare -f hadoop_subcommand_rumentrace >/dev/null 2>/dev/null; then
+
+  if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+    hadoop_add_subcommand "rumentrace" "convert logs into a rumen trace"
+  fi
+
+## @description  rumentrace command for hadoop
+## @audience     public
+## @stability    stable
+## @replaceable  yes
+function hadoop_subcommand_rumentrace
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.rumen.TraceBuilder
+  hadoop_add_to_classpath_tools hadoop-rumen
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+fi

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm b/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
index bee976a..34dfd0b 100644
--- a/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
+++ b/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
@@ -50,8 +50,8 @@ but a simulation of the scheduler elects to run that task on a remote
 rack, the simulator requires a runtime its input cannot provide. 
 To fill in these gaps, Rumen performs a statistical analysis of the 
 digest to estimate the variables the trace doesn't supply. Rumen traces 
-drive both Gridmix (a benchmark of Hadoop MapReduce clusters) and Mumak 
-(a simulator for the JobTracker).
+drive both Gridmix (a benchmark of Hadoop MapReduce clusters) and SLS
+(a simulator for the resource manager scheduler).
 
 
 $H3 Motivation
@@ -126,16 +126,13 @@ can use the `Folder` utility to fold the current trace to the
 desired length. The remaining part of this section explains these 
 utilities in detail.
     
-Examples in this section assumes that certain libraries are present 
-in the java CLASSPATH. See [Dependencies](#Dependencies) for more details.
-
 
 $H3 Trace Builder
       
 $H4 Command
 
 ```
-java org.apache.hadoop.tools.rumen.TraceBuilder [options] <jobtrace-output> <topology-output>
<inputs>
+hadoop rumentrace [options] <jobtrace-output> <topology-output> <inputs>
 ```
   
 This command invokes the `TraceBuilder` utility of *Rumen*.
@@ -205,12 +202,8 @@ $H4 Options
 
 $H4 Example
 
-*Rumen* expects certain library *JARs* to be present in  the *CLASSPATH*.
-One simple way to run Rumen is to use
-`$HADOOP_HOME/bin/hadoop jar` command to run it as example below.
-
 ```
-java org.apache.hadoop.tools.rumen.TraceBuilder \
+hadoop rumentrace \
   file:///tmp/job-trace.json \
   file:///tmp/job-topology.json \
   hdfs:///tmp/hadoop-yarn/staging/history/done_intermediate/testuser
@@ -229,7 +222,7 @@ $H3 Folder
 $H4 Command
 
 ```
-java org.apache.hadoop.tools.rumen.Folder [options] [input] [output]
+hadoop rumenfolder [options] [input] [output]
 ```
       
 This command invokes the `Folder` utility of 
@@ -350,7 +343,7 @@ $H4 Examples
 $H5 Folding an input trace with 10 hours of total runtime to generate an output trace with
1 hour of total runtime
 
 ```
-java org.apache.hadoop.tools.rumen.Folder \
+hadoop rumenfolder \
   -output-duration 1h \
   -input-cycle 20m \
   file:///tmp/job-trace.json \
@@ -362,7 +355,7 @@ If the folded jobs are out of order then the command will bail out.
 $H5 Folding an input trace with 10 hours of total runtime to generate an output trace with
1 hour of total runtime and tolerate some skewness
 
 ```
-java org.apache.hadoop.tools.rumen.Folder \
+hadoop rumenfolder \
   -output-duration 1h \
   -input-cycle 20m \
   -allow-missorting \
@@ -378,7 +371,7 @@ If the folded jobs are out of order, then atmost
 $H5 Folding an input trace with 10 hours of total runtime to generate an output trace with
1 hour of total runtime in debug mode
 
 ```
-java org.apache.hadoop.tools.rumen.Folder \
+hadoop rumenfolder \
   -output-duration 1h \
   -input-cycle 20m \
   -debug -temp-directory file:///tmp/debug \
@@ -395,7 +388,7 @@ up.
 $H5 Folding an input trace with 10 hours of total runtime to generate an output trace with
1 hour of total runtime with custom concentration.
 
 ```
-java org.apache.hadoop.tools.rumen.Folder \
+hadoop rumenfolder \
   -output-duration 1h \
   -input-cycle 20m \
   -concentration 2 \
@@ -421,18 +414,3 @@ Look at the MapReduce
 <a href="https://issues.apache.org/jira/browse/MAPREDUCE/component/12313617">rumen-component</a>
 for further details.
 
-
-$H3 Dependencies
-
-*Rumen* expects certain library *JARs* to be present in  the *CLASSPATH*.
-One simple way to run Rumen is to use
-`hadoop jar` command to run it as example below.
-
-```
-$HADOOP_HOME/bin/hadoop jar \
-  $HADOOP_HOME/share/hadoop/tools/lib/hadoop-rumen-2.5.1.jar \
-  org.apache.hadoop.tools.rumen.TraceBuilder \
-  file:///tmp/job-trace.json \
-  file:///tmp/job-topology.json \
-  hdfs:///tmp/hadoop-yarn/staging/history/done_intermediate/testuser
-```

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
index 5a07cc3..ffddc7c 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
@@ -91,8 +91,7 @@ public class DumpTypedBytes implements Tool {
   }
 
   private void printUsage() {
-    System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
-        + " dumptb <glob-pattern>");
+    System.out.println("Usage: mapred streaming dumptb <glob-pattern>");
     System.out.println("  Dumps all files that match the given pattern to " +
         "standard output as typed bytes.");
     System.out.println("  The files can be text or sequence files");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
index eabf46c..92f9d03 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
@@ -56,8 +56,7 @@ public class HadoopStreaming {
   }
   
   private static void printUsage() {
-    System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
-        + " [options]");
+    System.out.println("Usage: mapred streaming [options]");
     System.out.println("Options:");
     System.out.println("  dumptb <glob-pattern> Dumps all files that match the" 
         + " given pattern to ");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
index a7a001c..838cfa1 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
@@ -89,8 +89,7 @@ public class LoadTypedBytes implements Tool {
   }
 
   private void printUsage() {
-    System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
-        + " loadtb <path>");
+    System.out.println("Usage: mapred streaming loadtb <path>");
     System.out.println("  Reads typed bytes from standard input" +
     " and stores them in a sequence file in");
     System.out.println("  the specified path");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh b/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
new file mode 100755
index 0000000..cca016d
--- /dev/null
+++ b/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f mapred_subcommand_streaming >/dev/null 2>/dev/null; then
+
+  if [[ "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+    hadoop_add_subcommand "streaming" "launch a mapreduce streaming job"
+  fi
+
+## @description  streaming command for mapred
+## @audience     public
+## @stability    stable
+## @replaceable  yes
+function mapred_subcommand_streaming
+{
+  declare jarname
+  declare oldifs
+
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
+  hadoop_add_to_classpath_tools hadoop-streaming
+
+  # locate the streaming jar so we have something to
+  # give to RunJar
+  oldifs=${IFS}
+  IFS=:
+  for jarname in ${CLASSPATH}; do
+    if [[ "${jarname}" =~ hadoop-streaming-[0-9] ]]; then
+      HADOOP_SUBCMD_ARGS=("${jarname}" "${HADOOP_SUBCMD_ARGS[@]}")
+      break
+    fi
+  done
+
+  IFS=${oldifs}
+
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+
+}
+
+fi

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm b/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
index cc8ed69..072a68b 100644
--- a/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
+++ b/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
@@ -62,7 +62,7 @@ Hadoop Streaming
 
 Hadoop streaming is a utility that comes with the Hadoop distribution. The utility allows
you to create and run Map/Reduce jobs with any executable or script as the mapper and/or the
reducer. For example:
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -input myInputDirs \
       -output myOutputDir \
       -mapper /bin/cat \
@@ -88,7 +88,7 @@ Streaming supports streaming command options as well as [generic command
options
 
 **Note:** Be sure to place the generic options before the streaming options, otherwise the
command will fail. For an example, see [Making Archives Available to Tasks](#Making_Archives_Available_to_Tasks).
 
-    hadoop command [genericOptions] [streamingOptions]
+    mapred streaming [genericOptions] [streamingOptions]
 
 The Hadoop streaming command options are listed here:
 
@@ -115,7 +115,7 @@ $H3 Specifying a Java Class as the Mapper/Reducer
 
 You can supply a Java class as the mapper and/or the reducer.
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -input myInputDirs \
       -output myOutputDir \
       -inputformat org.apache.hadoop.mapred.KeyValueTextInputFormat \
@@ -128,7 +128,7 @@ $H3 Packaging Files With Job Submissions
 
 You can specify any executable as the mapper and/or the reducer. The executables do not need
to pre-exist on the machines in the cluster; however, if they don't, you will need to use
"-file" option to tell the framework to pack your executable files as a part of job submission.
For example:
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -input myInputDirs \
       -output myOutputDir \
       -mapper myPythonScript.py \
@@ -139,7 +139,7 @@ The above example specifies a user defined Python executable as the mapper.
The
 
 In addition to executable files, you can also package other auxiliary files (such as dictionaries,
configuration files, etc) that may be used by the mapper and/or the reducer. For example:
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -input myInputDirs \
       -output myOutputDir \
       -mapper myPythonScript.py \
@@ -216,7 +216,7 @@ $H4 Specifying the Number of Reducers
 
 To specify the number of reducers, for example two, use:
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -D mapreduce.job.reduces=2 \
       -input myInputDirs \
       -output myOutputDir \
@@ -229,7 +229,7 @@ As noted earlier, when the Map/Reduce framework reads a line from the
stdout of
 
 However, you can customize this default. You can specify a field separator other than the
tab character (the default), and you can specify the nth (n \>= 1) character rather than
the first character in a line (the default) as the separator between the key and value. For
example:
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -D stream.map.output.field.separator=. \
       -D stream.num.map.output.key.fields=4 \
       -input myInputDirs \
@@ -279,7 +279,7 @@ User can specify a different symlink name for -archives using \#.
 
 In this example, the input.txt file has two lines specifying the names of the two files:
cachedir.jar/cache.txt and cachedir.jar/cache2.txt. "cachedir.jar" is a symlink to the archived
directory, which has the files "cache.txt" and "cache2.txt".
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
                     -archives 'hdfs://hadoop-nn1.example.com/user/me/samples/cachefile/cachedir.jar'
\
                     -D mapreduce.job.maps=1 \
                     -D mapreduce.job.reduces=1 \
@@ -325,7 +325,7 @@ $H3 Hadoop Partitioner Class
 
 Hadoop has a library class, [KeyFieldBasedPartitioner](../api/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.html),
that is useful for many applications. This class allows the Map/Reduce framework to partition
the map outputs based on certain key fields, not the whole keys. For example:
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -D stream.map.output.field.separator=. \
       -D stream.num.map.output.key.fields=4 \
       -D map.output.key.field.separator=. \
@@ -375,7 +375,7 @@ $H3 Hadoop Comparator Class
 
 Hadoop has a library class, [KeyFieldBasedComparator](../api/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.html),
that is useful for many applications. This class provides a subset of features provided by
the Unix/GNU Sort. For example:
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -D mapreduce.job.output.key.comparator.class=org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator
\
       -D stream.map.output.field.separator=. \
       -D stream.num.map.output.key.fields=4 \
@@ -411,7 +411,7 @@ Hadoop has a library package called [Aggregate](../api/org/apache/hadoop/mapred/
 
 To use Aggregate, simply specify "-reducer aggregate":
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -input myInputDirs \
       -output myOutputDir \
       -mapper myAggregatorForKeyCount.py \
@@ -444,7 +444,7 @@ $H3 Hadoop Field Selection Class
 
 Hadoop has a library class, [FieldSelectionMapReduce](../api/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.html),
that effectively allows you to process text data like the unix "cut" utility. The map function
defined in the class treats each input key/value pair as a list of fields. You can specify
the field separator (the default is the tab character). You can select an arbitrary list of
fields as the map output key, and an arbitrary list of fields as the map output value. Similarly,
the reduce function defined in the class treats each input key/value pair as a list of fields.
You can select an arbitrary list of fields as the reduce output key, and an arbitrary list
of fields as the reduce output value. For example:
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -D mapreduce.map.output.key.field.separator=. \
       -D mapreduce.partition.keypartitioner.options=-k1,2 \
       -D mapreduce.fieldsel.data.field.separator=. \
@@ -495,7 +495,7 @@ Using an alias will not work, but variable substitution is allowed as
shown in t
     charlie 80
     dan     75
 
-    $ c2='cut -f2'; hadoop jar hadoop-streaming-${project.version}.jar \
+    $ c2='cut -f2'; mapred streaming \
       -D mapreduce.job.name='Experiment' \
       -input /user/me/samples/student_marks \
       -output /user/me/samples/student_out \
@@ -525,7 +525,7 @@ $H3 How do I specify multiple input directories?
 
 You can specify multiple input directories with multiple '-input' options:
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -input '/user/foo/dir1' -input '/user/foo/dir2' \
         (rest of the command)
 
@@ -541,7 +541,7 @@ $H3 How do I parse XML documents using streaming?
 
 You can use the record reader StreamXmlRecordReader to process XML documents.
 
-    hadoop jar hadoop-streaming-${project.version}.jar \
+    mapred streaming \
       -inputreader "StreamXmlRecord,begin=BEGIN_STRING,end=END_STRING" \
         (rest of the command)
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index cac3bb6..878c95f 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -16,7 +16,12 @@
 # limitations under the License.
 
 MYNAME="${BASH_SOURCE-$0}"
+HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
 
+## @description  build up the yarn command's usage text.
+## @audience     public
+## @stability    stable
+## @replaceable  no
 function hadoop_usage
 {
   hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
@@ -46,9 +51,180 @@ function hadoop_usage
   hadoop_add_subcommand "timelineserver" "run the timeline server"
   hadoop_add_subcommand "top" "view cluster information"
   hadoop_add_subcommand "version" "print the version"
-  hadoop_generate_usage "${MYNAME}" true
+  hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
 }
 
+## @description  Default command handler for yarn command
+## @audience     public
+## @stability    stable
+## @replaceable  no
+## @param        CLI arguments
+function yarncmd_case
+{
+  subcmd=$1
+  shift
+
+  case ${subcmd} in
+    application|applicationattempt|container)
+      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+      set -- "${subcmd}" "$@"
+    ;;
+    classpath)
+      hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
+    ;;
+    cluster)
+      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ClusterCLI
+      hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
+      YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    daemonlog)
+      HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    envvars)
+      echo "JAVA_HOME='${JAVA_HOME}'"
+      echo "HADOOP_YARN_HOME='${HADOOP_YARN_HOME}'"
+      echo "YARN_DIR='${YARN_DIR}'"
+      echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'"
+      echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
+      echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+      echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+      echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
+      exit 0
+    ;;
+    jar)
+      HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    historyserver)
+      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+      echo "DEPRECATED: Use of this command to start the timeline server is deprecated."
1>&2
+      echo "Instead use the timelineserver command for it." 1>&2
+      echo "Starting the History Server anyway..." 1>&2
+      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+    ;;
+    logs)
+      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.LogsCLI
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    node)
+      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.NodeCLI
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    nodemanager)
+      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
+      hadoop_debug "Append YARN_NODEMANAGER_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_NODEMANAGER_OPTS}"
+      # Backwards compatibility
+      if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
+        HADOOP_HEAPSIZE_MAX="${YARN_NODEMANAGER_HEAPSIZE}"
+      fi
+    ;;
+    proxyserver)
+      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
+      hadoop_debug "Append YARN_PROXYSERVER_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_PROXYSERVER_OPTS}"
+      # Backwards compatibility
+      if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
+        # shellcheck disable=SC2034
+        HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}"
+      fi
+    ;;
+    queue)
+      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    resourcemanager)
+      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_RESOURCEMANAGER_OPTS}"
+      hadoop_debug "Append YARN_RESOURCEMANAGER_OPTS onto HADOOP_OPTS"
+      # Backwards compatibility
+      if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
+        # shellcheck disable=SC2034
+        HADOOP_HEAPSIZE_MAX="${YARN_RESOURCEMANAGER_HEAPSIZE}"
+      fi
+    ;;
+    rmadmin)
+      HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    scmadmin)
+      HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    sharedcachemanager)
+      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
+      hadoop_debug "Append YARN_SHAREDCACHEMANAGER_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_SHAREDCACHEMANAGER_OPTS}"
+    ;;
+    timelineserver)
+      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+      hadoop_debug "Append YARN_TIMELINESERVER_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_TIMELINESERVER_OPTS}"
+      # Backwards compatibility
+      if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
+        # shellcheck disable=SC2034
+        HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
+      fi
+    ;;
+    version)
+      HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    top)
+      doNotSetCols=0
+      doNotSetRows=0
+      for i in "$@"; do
+        if [[ $i == "-cols" ]]; then
+          doNotSetCols=1
+        fi
+        if [[ $i == "-rows" ]]; then
+          doNotSetRows=1
+        fi
+      done
+      if [ $doNotSetCols == 0 ] && [ -n "${TERM}" ]; then
+        cols=$(tput cols)
+        if [ -n "$cols" ]; then
+          args=( $@ )
+          args=("${args[@]}" "-cols" "$cols")
+          set -- "${args[@]}"
+        fi
+      fi
+      if [ $doNotSetRows == 0 ] && [ -n "${TERM}" ]; then
+        rows=$(tput lines)
+        if [ -n "$rows" ]; then
+          args=( $@ )
+          args=("${args[@]}" "-rows" "$rows")
+          set -- "${args[@]}"
+        fi
+      fi
+      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.TopCLI
+      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+    ;;
+    *)
+      HADOOP_CLASSNAME="${subcmd}"
+      if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
+        hadoop_exit_with_usage 1
+      fi
+    ;;
+  esac
+}
 
 # let's locate libexec...
 if [[ -n "${HADOOP_HOME}" ]]; then
@@ -74,195 +250,71 @@ if [[ $# = 0 ]]; then
 fi
 
 # get arguments
-COMMAND=$1
+HADOOP_SUBCMD=$1
 shift
 
-case "${COMMAND}" in
-  application|applicationattempt|container)
-    CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-    set -- "${COMMAND}" "$@"
-  ;;
-  classpath)
-    hadoop_do_classpath_subcommand CLASS "$@"
-  ;;
-  cluster)
-    CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  daemonlog)
-    CLASS=org.apache.hadoop.log.LogLevel
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  envvars)
-    echo "JAVA_HOME='${JAVA_HOME}'"
-    echo "HADOOP_YARN_HOME='${HADOOP_YARN_HOME}'"
-    echo "YARN_DIR='${YARN_DIR}'"
-    echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'"
-    echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
-    echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
-    echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
-    echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
-    exit 0
-  ;;
-  jar)
-    CLASS=org.apache.hadoop.util.RunJar
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  historyserver)
-    supportdaemonization="true"
-    echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
-    echo "Instead use the timelineserver command for it." 1>&2
-    echo "Starting the History Server anyway..." 1>&2
-    CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
-  ;;
-  logs)
-    CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  node)
-    CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  nodemanager)
-    supportdaemonization="true"
-    CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
-    hadoop_debug "Append YARN_NODEMANAGER_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_NODEMANAGER_OPTS}"
-    # Backwards compatibility
-    if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
-      HADOOP_HEAPSIZE_MAX="${YARN_NODEMANAGER_HEAPSIZE}"
-    fi
-  ;;
-  proxyserver)
-    supportdaemonization="true"
-    CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
-    hadoop_debug "Append YARN_PROXYSERVER_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_PROXYSERVER_OPTS}"
-    # Backwards compatibility
-    if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
-      HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}"
-    fi
-  ;;
-  queue)
-    CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  resourcemanager)
-    supportdaemonization="true"
-    CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_RESOURCEMANAGER_OPTS}"
-    hadoop_debug "Append YARN_RESOURCEMANAGER_OPTS onto HADOOP_OPTS"
-    # Backwards compatibility
-    if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
-      HADOOP_HEAPSIZE_MAX="${YARN_RESOURCEMANAGER_HEAPSIZE}"
-    fi
-  ;;
-  rmadmin)
-    CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  scmadmin)
-    CLASS='org.apache.hadoop.yarn.client.SCMAdmin'
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  sharedcachemanager)
-    supportdaemonization="true"
-    CLASS='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
-    hadoop_debug "Append YARN_SHAREDCACHEMANAGER_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_SHAREDCACHEMANAGER_OPTS}"
-  ;;
-  timelineserver)
-    supportdaemonization="true"
-    CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
-    hadoop_debug "Append YARN_TIMELINESERVER_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_TIMELINESERVER_OPTS}"
-    # Backwards compatibility
-    if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
-      HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
-    fi
-  ;;
-  version)
-    CLASS=org.apache.hadoop.util.VersionInfo
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  top)
-    doNotSetCols=0
-    doNotSetRows=0
-    for i in "$@"; do
-      if [[ $i == "-cols" ]]; then
-        doNotSetCols=1
-      fi
-      if [[ $i == "-rows" ]]; then
-        doNotSetRows=1
-      fi
-    done
-    if [ $doNotSetCols == 0 ] && [ -n "${TERM}" ]; then
-      cols=$(tput cols)
-      if [ -n "$cols" ]; then
-        args=( $@ )
-        args=("${args[@]}" "-cols" "$cols")
-        set -- "${args[@]}"
-      fi
-    fi
-    if [ $doNotSetRows == 0 ] && [ -n "${TERM}" ]; then
-      rows=$(tput lines)
-      if [ -n "$rows" ]; then
-        args=( $@ )
-        args=("${args[@]}" "-rows" "$rows")
-        set -- "${args[@]}"
-      fi
-    fi
-    CLASS=org.apache.hadoop.yarn.client.cli.TopCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
-  ;;
-  *)
-    CLASS="${COMMAND}"
-    if ! hadoop_validate_classname "${CLASS}"; then
-      hadoop_exit_with_usage 1
-    fi
-  ;;
-esac
+HADOOP_SUBCMD_ARGS=("$@")
+
+if declare -f yarn_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
+  hadoop_debug "Calling dynamically: yarn_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}"
+  "yarn_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+else
+  yarncmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+fi
 
-hadoop_verify_user "${COMMAND}"
+hadoop_verify_user "${HADOOP_SUBCMD}"
 
 if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
   hadoop_common_slave_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
   exit $?
 fi
 
-daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
-daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
+if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
+  HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
+  hadoop_verify_secure_prereq
+  hadoop_setup_secure_service
+  priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+  priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err"
+  priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+  daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+  daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+else
+  daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+  daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+fi
 
 if [[  "${HADOOP_DAEMON_MODE}" != "default" ]]; then
   # shellcheck disable=SC2034
   HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
-  HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
+  # shellcheck disable=SC2034
+  HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
 fi
 
 hadoop_finalize
 
-if [[ -n "${supportdaemonization}" ]]; then
-  if [[ -n "${secure_service}" ]]; then
-    hadoop_secure_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" \
-    "${CLASS}" "${daemon_pidfile}" "${daemon_outfile}" \
-    "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@"
+if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then
+  if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
+    hadoop_secure_daemon_handler \
+      "${HADOOP_DAEMON_MODE}" \
+      "${HADOOP_SUBCMD}" \
+      "${HADOOP_CLASSNAME}" \
+      "${daemon_pidfile}" \
+      "${daemon_outfile}" \
+      "${priv_pidfile}" \
+      "${priv_outfile}" \
+      "${priv_errfile}" \
+      "${HADOOP_SUBCMD_ARGS[@]}"
   else
-    hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}" \
-    "${daemon_pidfile}" "${daemon_outfile}" "$@"
+    hadoop_daemon_handler \
+      "${HADOOP_DAEMON_MODE}" \
+      "${HADOOP_SUBCMD}" \
+      "${HADOOP_CLASSNAME}" \
+      "${daemon_pidfile}" \
+      "${daemon_outfile}" \
+      "${HADOOP_SUBCMD_ARGS[@]}"
   fi
   exit $?
 else
-  hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
+  # shellcheck disable=SC2086
+  hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}"
 fi

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-yarn-project/hadoop-yarn/shellprofile.d/hadoop-yarn.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/shellprofile.d/hadoop-yarn.sh b/hadoop-yarn-project/hadoop-yarn/shellprofile.d/hadoop-yarn.sh
new file mode 100644
index 0000000..4602293
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/shellprofile.d/hadoop-yarn.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop_add_profile yarn
+
+function _yarn_hadoop_classpath
+{
+  local i
+  #
+  # get all of the yarn jars+config in the path
+  #
+  # developers
+  if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
+    for i in yarn-api yarn-common yarn-mapreduce yarn-master-worker \
+             yarn-server/yarn-server-nodemanager \
+             yarn-server/yarn-server-common \
+             yarn-server/yarn-server-resourcemanager; do
+      hadoop_add_classpath "${HADOOP_YARN_HOME}/$i/target/classes"
+    done
+
+    hadoop_add_classpath "${HADOOP_YARN_HOME}/build/test/classes"
+    hadoop_add_classpath "${HADOOP_YARN_HOME}/build/tools"
+  fi
+
+  if [[ -d "${HADOOP_YARN_HOME}/${YARN_DIR}/webapps" ]]; then
+    hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"
+  fi
+
+  hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}"'/*'
+  hadoop_add_classpath  "${HADOOP_YARN_HOME}/${YARN_DIR}"'/*'
+}
+
+function _yarn_hadoop_finalize
+{
+  # Add YARN custom options to comamnd line in case someone actaully
+  # used these.
+  #
+  # Note that we are replacing ' ' with '\ ' so that when we exec
+  # stuff it works
+  #
+  local yld=$HADOOP_LOG_DIR
+  hadoop_translate_cygwin_path yld
+  hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${yld}"
+  hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE}"
+  local yhd=$HADOOP_YARN_HOME
+  hadoop_translate_cygwin_path yhd
+  hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${yhd}"
+  hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn.sh b/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn.sh
deleted file mode 100644
index 4aa20b1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-hadoop_add_profile yarn
-
-function _yarn_hadoop_classpath
-{
-  local i
-  #
-  # get all of the yarn jars+config in the path
-  #
-  # developers
-  if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
-    for i in yarn-api yarn-common yarn-mapreduce yarn-master-worker \
-             yarn-server/yarn-server-nodemanager \
-             yarn-server/yarn-server-common \
-             yarn-server/yarn-server-resourcemanager; do
-      hadoop_add_classpath "${HADOOP_YARN_HOME}/$i/target/classes"
-    done
-
-    hadoop_add_classpath "${HADOOP_YARN_HOME}/build/test/classes"
-    hadoop_add_classpath "${HADOOP_YARN_HOME}/build/tools"
-  fi
-
-  if [[ -d "${HADOOP_YARN_HOME}/${YARN_DIR}/webapps" ]]; then
-    hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"
-  fi
-
-  hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}"'/*'
-  hadoop_add_classpath  "${HADOOP_YARN_HOME}/${YARN_DIR}"'/*'
-}
-
-function _yarn_hadoop_finalize
-{
-  # Add YARN custom options to comamnd line in case someone actaully
-  # used these.
-  #
-  # Note that we are replacing ' ' with '\ ' so that when we exec
-  # stuff it works
-  #
-  local yld=$HADOOP_LOG_DIR
-  hadoop_translate_cygwin_path yld
-  hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${yld}"
-  hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE}"
-  local yhd=$HADOOP_YARN_HOME
-  hadoop_translate_cygwin_path yhd
-  hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${yhd}"
-  hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message