hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [10/16] hadoop git commit: HADOOP-12936. modify hadoop-tools to take advantage of dynamic subcommands
Date Fri, 06 May 2016 21:11:28 GMT
HADOOP-12936. modify hadoop-tools to take advantage of dynamic subcommands


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b7c1d88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b7c1d88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b7c1d88

Branch: refs/heads/HADOOP-12930
Commit: 1b7c1d887d966aae0b611b81ec56718cdf8872b2
Parents: 4247c06
Author: Allen Wittenauer <aw@apache.org>
Authored: Wed May 4 07:21:36 2016 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Fri May 6 14:10:19 2016 -0700

----------------------------------------------------------------------
 .../main/resources/assemblies/hadoop-tools.xml  | 32 ++++++++++++++
 .../hadoop-common/src/main/bin/hadoop           | 45 +++++++-------------
 hadoop-mapreduce-project/bin/mapred             | 21 ---------
 .../main/shellprofile.d/hadoop-archive-logs.sh  | 33 ++++++++++++++
 .../src/main/shellprofile.d/hadoop-archives.sh  | 41 ++++++++++++++++++
 .../src/main/shellprofile.d/hadoop-distcp.sh    | 43 +++++++++++++++++++
 .../src/main/shellprofile.d/hadoop-extras.sh    | 33 ++++++++++++++
 7 files changed, 197 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index f8ba48e..c5ea6ad 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -24,6 +24,38 @@
   <includeBaseDirectory>false</includeBaseDirectory>
   <fileSets>
     <fileSet>
+      <directory>../hadoop-archive-logs/src/main/shellprofile.d</directory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-archives/src/main/shellprofile.d</directory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-distcp/src/main/shellprofile.d</directory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-extras/src/main/shellprofile.d</directory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
       <directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
       <includes>
         <include>*.hh</include>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 453ac83..ce695cb 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -30,14 +30,11 @@ function hadoop_usage
   hadoop_add_option "hosts filename" "list of hosts to use in slave mode"
   hadoop_add_option "slaves" "turn on slave mode"
 
-  hadoop_add_subcommand "archive" "create a Hadoop archive"
   hadoop_add_subcommand "checknative" "check native Hadoop and compression libraries availability"
   hadoop_add_subcommand "classpath" "prints the class path needed to get the Hadoop jar and
the required libraries"
   hadoop_add_subcommand "conftest" "validate configuration XML files"
   hadoop_add_subcommand "credential" "interact with credential providers"
   hadoop_add_subcommand "daemonlog" "get/set the log level for each daemon"
-  hadoop_add_subcommand "distch" "distributed metadata changer"
-  hadoop_add_subcommand "distcp" "copy file or directories recursively"
   hadoop_add_subcommand "dtutil" "operations related to delegation tokens"
   hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
   hadoop_add_subcommand "fs" "run a generic filesystem user client"
@@ -102,35 +99,23 @@ function hadoopcmd_case
         exit 1
       fi
     ;;
-    archive)
-      HADOOP_CLASS=org.apache.hadoop.tools.HadoopArchives
-      hadoop_add_to_classpath_tools hadoop-archives
-    ;;
     checknative)
-      HADOOP_CLASS=org.apache.hadoop.util.NativeLibraryChecker
+      HADOOP_CLASSNAME=org.apache.hadoop.util.NativeLibraryChecker
     ;;
     classpath)
-      hadoop_do_classpath_subcommand HADOOP_CLASS "$@"
+      hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
     ;;
     conftest)
-      HADOOP_CLASS=org.apache.hadoop.util.ConfTest
+      HADOOP_CLASSNAME=org.apache.hadoop.util.ConfTest
     ;;
     credential)
-      HADOOP_CLASS=org.apache.hadoop.security.alias.CredentialShell
+      HADOOP_CLASSNAME=org.apache.hadoop.security.alias.CredentialShell
     ;;
     daemonlog)
-      HADOOP_CLASS=org.apache.hadoop.log.LogLevel
-    ;;
-    distch)
-      HADOOP_CLASS=org.apache.hadoop.tools.DistCh
-      hadoop_add_to_classpath_tools hadoop-extras
-    ;;
-    distcp)
-      HADOOP_CLASS=org.apache.hadoop.tools.DistCp
-      hadoop_add_to_classpath_tools hadoop-distcp
+      HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
     ;;
     dtutil)
-      HADOOP_CLASS=org.apache.hadoop.security.token.DtUtilShell
+      HADOOP_CLASSNAME=org.apache.hadoop.security.token.DtUtilShell
     ;;
     envvars)
       echo "JAVA_HOME='${JAVA_HOME}'"
@@ -145,7 +130,7 @@ function hadoopcmd_case
       exit 0
     ;;
     fs)
-      HADOOP_CLASS=org.apache.hadoop.fs.FsShell
+      HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
     ;;
     gridmix)
       CLASS=org.apache.hadoop.mapred.gridmix.Gridmix
@@ -156,7 +141,7 @@ function hadoopcmd_case
       if [[ -n "${YARN_OPTS}" ]] || [[ -n "${YARN_CLIENT_OPTS}" ]]; then
         hadoop_error "WARNING: Use \"yarn jar\" to launch YARN applications."
       fi
-      HADOOP_CLASS=org.apache.hadoop.util.RunJar
+      HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
     ;;
     jnipath)
       hadoop_finalize
@@ -164,20 +149,20 @@ function hadoopcmd_case
       exit 0
     ;;
     kerbname)
-      HADOOP_CLASS=org.apache.hadoop.security.HadoopKerberosName
+      HADOOP_CLASSNAME=org.apache.hadoop.security.HadoopKerberosName
     ;;
     key)
-      HADOOP_CLASS=org.apache.hadoop.crypto.key.KeyShell
+      HADOOP_CLASSNAME=org.apache.hadoop.crypto.key.KeyShell
     ;;
     trace)
-      HADOOP_CLASS=org.apache.hadoop.tracing.TraceAdmin
+      HADOOP_CLASSNAME=org.apache.hadoop.tracing.TraceAdmin
     ;;
     version)
-      HADOOP_CLASS=org.apache.hadoop.util.VersionInfo
+      HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
     ;;
     *)
-      HADOOP_CLASS="${subcmd}"
-      if ! hadoop_validate_classname "${HADOOP_CLASS}"; then
+      HADOOP_CLASSNAME="${subcmd}"
+      if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
         hadoop_exit_with_usage 1
       fi
     ;;
@@ -229,5 +214,5 @@ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
 HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 
 hadoop_finalize
-hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASS}" "$@"
+hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "$@"
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index 69b00b3..84f03e4 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -24,10 +24,7 @@ HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
 ## @replaceable  no
 function hadoop_usage
 {
-  hadoop_add_subcommand "archive" "create a hadoop archive"
-  hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives"
   hadoop_add_subcommand "classpath" "prints the class path needed for running mapreduce subcommands"
-  hadoop_add_subcommand "distcp" "copy file or directories recursively"
   hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
   hadoop_add_subcommand "historyserver" "run job history servers as a standalone daemon"
   hadoop_add_subcommand "hsadmin" "job history server admin interface"
@@ -55,27 +52,9 @@ function mapredcmd_case
       hadoop_error "You may find similar functionality with the \"yarn\" shell command."
       hadoop_exit_with_usage 1
     ;;
-    archive)
-      HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives
-      hadoop_add_to_classpath_tools hadoop-archives
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
-    ;;
-    archive-logs)
-      HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs
-      hadoop_add_to_classpath_tools hadoop-archive-logs
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
-    ;;
     classpath)
       hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
     ;;
-    distcp)
-      HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
-      hadoop_add_to_classpath_tools hadoop-distcp
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
-    ;;
     envvars)
       echo "JAVA_HOME='${JAVA_HOME}'"
       echo "HADOOP_MAPRED_HOME='${HADOOP_MAPRED_HOME}'"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
new file mode 100755
index 0000000..d37411e
--- /dev/null
+++ b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+  hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives"
+fi
+
+## @description  archive-logs command for mapred
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function mapred_subcommand_archive-logs
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs
+  hadoop_add_to_classpath_tools hadoop-archive-logs
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh b/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh
new file mode 100755
index 0000000..b85ff25
--- /dev/null
+++ b/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop
+   || "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+  hadoop_add_subcommand "archive" "create a Hadoop archive"
+fi
+
+## @description  archive command for hadoop (and mapred)
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function hadoop_subcommand_archive
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives
+  hadoop_add_to_classpath_tools hadoop-archives
+}
+
+## @description  archive-logs command for mapred (calls hadoop version)
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function mapred_subcommand_archive
+{
+  hadoop_subcommand_archive
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
new file mode 100755
index 0000000..6cb13fc
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop
+   || "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+  hadoop_add_subcommand "distcp" "copy file or directories recursively"
+fi
+
+## @description  distcp command for hadoop
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function hadoop_subcommand_distcp
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
+  hadoop_add_to_classpath_tools hadoop-distcp
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+## @description  distcp command for mapred (calls hadoop)
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function mapred_subcommand_distcp
+{
+  hadoop_subcommand_distcp
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b7c1d88/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
new file mode 100755
index 0000000..e599334
--- /dev/null
+++ b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+  hadoop_add_subcommand "distch" "distributed metadata changer"
+fi
+
+## @description  distch command for hadoop
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function hadoop_subcommand_distch
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCh
+  hadoop_add_to_classpath_tools hadoop-extras
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message