hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject [16/50] [abbrv] hadoop git commit: HADOOP-11565. Add --slaves shell option (aw)
Date Mon, 23 Feb 2015 19:36:13 GMT
HADOOP-11565. Add --slaves shell option (aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/812817c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/812817c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/812817c0

Branch: refs/heads/HDFS-EC
Commit: 812817c0ba56b1b1aae5fe81d45915bc2ddaf1be
Parents: 82c93b0
Author: Allen Wittenauer <aw@apache.org>
Authored: Thu Feb 12 18:01:28 2015 -0800
Committer: Zhe Zhang <zhz@apache.org>
Committed: Mon Feb 16 10:29:48 2015 -0800

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt  |  2 ++
 .../hadoop-common/src/main/bin/hadoop            |  5 +++++
 .../hadoop-common/src/main/bin/hadoop-config.sh  |  9 +++++++--
 .../hadoop-common/src/main/bin/hadoop-daemons.sh | 16 ++++++++++++++--
 .../src/main/bin/hadoop-functions.sh             | 14 ++++++++++++++
 .../hadoop-hdfs/src/main/bin/hdfs                |  5 +++++
 hadoop-mapreduce-project/bin/mapred              |  5 +++++
 hadoop-yarn-project/hadoop-yarn/bin/yarn         |  5 +++++
 .../hadoop-yarn/bin/yarn-daemons.sh              | 19 +++++++++++++++++--
 9 files changed, 74 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/812817c0/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c11e340..0d8c02f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -29,6 +29,8 @@ Trunk (Unreleased)
 
     HADOOP-11554. Expose HadoopKerberosName as a hadoop subcommand (aw)
 
+    HADOOP-11565. Add --slaves shell option (aw)
+
   IMPROVEMENTS
 
     HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812817c0/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 42e4e83..6003927 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -179,6 +179,11 @@ esac
 
 hadoop_verify_user "${COMMAND}"
 
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+  hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
+  exit $?
+fi
+
 # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
 hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
 HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812817c0/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
index f0cabbe..58b871e 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
@@ -1,4 +1,4 @@
-
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -91,7 +91,7 @@ hadoop_bootstrap
 
 # save these off in case our caller needs them
 # shellcheck disable=SC2034
-HADOOP_USER_PARAMS="$@"
+HADOOP_USER_PARAMS=("$@")
 
 HADOOP_DAEMON_MODE="default"
 
@@ -152,6 +152,11 @@ while [[ -z "${_hadoop_common_done}" ]]; do
       HADOOP_LOGLEVEL="$1"
       shift
     ;;
+    --slaves)
+      shift
+      # shellcheck disable=SC2034
+      HADOOP_SLAVE_MODE=true
+    ;;
     *)
       _hadoop_common_done=true
     ;;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812817c0/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
index 20d7e4a..9e4e6b0 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
@@ -56,5 +56,17 @@ else
   hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs"
 fi
 
-hadoop_connect_to_hosts "$hdfsscript" \
-    --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"
+hadoop_error "WARNING: Use of this script to ${daemonmode} HDFS daemons is deprecated."
+hadoop_error "WARNING: Attempting to execute replacement \"hdfs --slaves --daemon ${daemonmode}\"
instead."
+
+#
+# Original input was usually:
+#  hadoop-daemons.sh (shell options) (start|stop) (datanode|...) (daemon options)
+# we're going to turn this into
+#  hdfs --slaves --daemon (start|stop) (rest of options)
+#
+argv=(${HADOOP_USER_PARAMS[@]/start})
+argv=(${argv[@]/stop})
+argv=(${argv[@]/status})
+
+${hdfsscript} --slaves --daemon "${daemonmode}" "${argv[@]}"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812817c0/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 142ac22..cec6b2c 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -405,6 +405,20 @@ function hadoop_connect_to_hosts
   fi
 }
 
+function hadoop_common_slave_mode_execute
+{
+  #
+  # input should be the command line as given by the user
+  # in the form of an array
+  #
+  local argv=("$@")
+
+  # if --slaves is still on the command line, remove it
+  # to prevent loops
+  argv=(${argv[@]/--slaves})
+  hadoop_connect_to_hosts -- "${argv[@]}"
+}
+
 function hadoop_validate_classname
 {
   local class=$1

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812817c0/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 5f8ff18..830ca36 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -247,6 +247,11 @@ esac
 
 hadoop_verify_user "${COMMAND}"
 
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+  hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
+  exit $?
+fi
+
 if [[ -n "${secure_service}" ]]; then
   HADOOP_SECURE_USER="${secure_user}"
   hadoop_verify_secure_prereq

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812817c0/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index 6d0c781..d199128 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -135,6 +135,11 @@ esac
 
 hadoop_verify_user "${COMMAND}"
 
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+  hadoop_common_slave_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}"
+  exit $?
+fi
+
 daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
 daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812817c0/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index b3fe1f3..f1a06a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -186,6 +186,11 @@ esac
 
 hadoop_verify_user "${COMMAND}"
 
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+  hadoop_common_slave_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
+  exit $?
+fi
+
 daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
 daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812817c0/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
index 17d3db6..c6963d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
@@ -41,8 +41,23 @@ else
   exit 1
 fi
 
+yarnscript="${HADOOP_YARN_HOME}/bin/yarn"
+
 daemonmode=$1
 shift
 
-hadoop_connect_to_hosts "${HADOOP_YARN_HOME}/bin/yarn" \
-    --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"
+hadoop_error "WARNING: Use of this script to ${daemonmode} YARN daemons is deprecated."
+hadoop_error "WARNING: Attempting to execute replacement \"yarn --slaves --daemon ${daemonmode}\"
instead."
+
+#
+# Original input was usually:
+#  yarn-daemons.sh (shell options) (start|stop) nodemanager (daemon options)
+# we're going to turn this into
+#  yarn --slaves --daemon (start|stop) (rest of options)
+#
+argv=(${HADOOP_USER_PARAMS[@]/start})
+argv=(${argv[@]/stop})
+argv=(${argv[@]/status})
+
+${yarnscript} --slaves --daemon "${daemonmode}" "${argv[@]}"
+


Mime
View raw message