hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject git commit: HADOOP-11022. User replaced functions get lost 2-3 levels deep (e.g., sbin) (aw)
Date Tue, 16 Sep 2014 23:06:34 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3e85f5b60 -> 7971c97ec


HADOOP-11022. User replaced functions get lost 2-3 levels deep (e.g., sbin) (aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7971c97e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7971c97e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7971c97e

Branch: refs/heads/trunk
Commit: 7971c97ec18f4fc6504f6a87d9ad02a1f5f01eea
Parents: 3e85f5b
Author: Allen Wittenauer <aw@apache.org>
Authored: Tue Sep 16 16:06:23 2014 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Tue Sep 16 16:06:23 2014 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +
 .../hadoop-common/src/main/bin/hadoop-config.sh |  1 +
 .../src/main/bin/hadoop-functions.sh            |  9 ++
 .../hadoop-common/src/main/conf/hadoop-env.sh   | 81 -----------------
 .../main/conf/hadoop-user-functions.sh.example  | 94 ++++++++++++++++++++
 .../hadoop-hdfs/src/main/bin/hdfs-config.sh     |  7 +-
 hadoop-mapreduce-project/bin/mapred-config.sh   |  7 +-
 .../hadoop-yarn/bin/yarn-config.sh              | 11 ++-
 8 files changed, 124 insertions(+), 89 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7971c97e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 5e57c62..194b4be 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -327,6 +327,9 @@ Trunk (Unreleased)
 
     HADOOP-11055. non-daemon pid files are missing (aw)
 
+    HADOOP-11022. User replaced functions get lost 2-3 levels deep (e.g., 
+    sbin) (aw)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7971c97e/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
index 40494b3..81a5867 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
@@ -156,6 +156,7 @@ done
 
 hadoop_find_confdir
 hadoop_exec_hadoopenv
+hadoop_exec_userfuncs
 
 #
 # IMPORTANT! User provided code is now available!

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7971c97e/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index db7dd5e..dfdb101 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -104,6 +104,15 @@ function hadoop_exec_hadoopenv
   fi
 }
 
+function hadoop_exec_userfuncs
+{
+  # NOTE: This function is not user replaceable.
+
+  if [[ -e "${HADOOP_CONF_DIR}/hadoop-user-functions.sh" ]]; then
+    . "${HADOOP_CONF_DIR}/hadoop-user-functions.sh"
+  fi
+}
+
 function hadoop_basic_init
 {
   # Some of these are also set in hadoop-env.sh.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7971c97e/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index eda47c9..754601a 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -337,84 +337,3 @@ esac
 # via this special env var:
 # HADOOP_ENABLE_BUILD_PATHS="true"
 
-# You can do things like replace parts of the shell underbelly.
-# Most of this code is in hadoop-functions.sh.
-#
-#
-# For example, if you want to add compression to the rotation
-# menthod for the .out files that daemons generate, you can do
-# that by redefining the hadoop_rotate_log function by
-# uncommenting this code block:
-
-#function hadoop_rotate_log
-#{
-#  #
-#  # log rotation (mainly used for .out files)
-#  # Users are likely to replace this one for something
-#  # that gzips or uses dates or who knows what.
-#  #
-#  # be aware that &1 and &2 might go through here
-#  # so don't do anything too crazy...
-#  #
-#  local log=$1;
-#  local num=${2:-5};
-#
-#  if [[ -f "${log}" ]]; then # rotate logs
-#    while [[ ${num} -gt 1 ]]; do
-#      #shellcheck disable=SC2086
-#      let prev=${num}-1
-#      if [[ -f "${log}.${prev}" ]]; then
-#        mv "${log}.${prev}" "${log}.${num}"
-#      fi
-#      num=${prev}
-#    done
-#    mv "${log}" "${log}.${num}"
-#    gzip -9 "${log}.${num}"
-#  fi
-#}
-#
-#
-# Another example:  finding java
-#
-# By default, Hadoop assumes that $JAVA_HOME is always defined
-# outside of its configuration. Eons ago, Apple standardized
-# on a helper program called java_home to find it for you.
-#
-#function hadoop_java_setup
-#{
-#
-#  if [[ -z "${JAVA_HOME}" ]]; then
-#     case $HADOOP_OS_TYPE in
-#       Darwin*)
-#          JAVA_HOME=$(/usr/libexec/java_home)
-#          ;;
-#     esac
-#  fi
-#
-#  # Bail if we did not detect it
-#  if [[ -z "${JAVA_HOME}" ]]; then
-#    echo "ERROR: JAVA_HOME is not set and could not be found." 1>&2
-#    exit 1
-#  fi
-#
-#  if [[ ! -d "${JAVA_HOME}" ]]; then
-#     echo "ERROR: JAVA_HOME (${JAVA_HOME}) does not exist." 1>&2
-#     exit 1
-#  fi
-#
-#  JAVA="${JAVA_HOME}/bin/java"
-#
-#  if [[ ! -x ${JAVA} ]]; then
-#    echo "ERROR: ${JAVA} is not executable." 1>&2
-#    exit 1
-#  fi
-#  JAVA_HEAP_MAX=-Xmx1g
-#  HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-128}
-#
-#  # check envvars which might override default args
-#  if [[ -n "$HADOOP_HEAPSIZE" ]]; then
-#    JAVA_HEAP_MAX="-Xmx${HADOOP_HEAPSIZE}m"
-#  fi
-#}
-
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7971c97e/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
new file mode 100644
index 0000000..7699c40
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
@@ -0,0 +1,94 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#######
+# Advanced Users Only
+######
+
+# You can do things like replace parts of the shell underbelly.
+# Most of this code is in hadoop-functions.sh.
+#
+#
+# For example, if you want to add compression to the rotation
+# menthod for the .out files that daemons generate, you can do
+# that by redefining the hadoop_rotate_log function by
+# uncommenting this code block:
+
+#function hadoop_rotate_log
+#{
+#  local log=$1;
+#  local num=${2:-5};
+#
+#  if [[ -f "${log}" ]]; then
+#    while [[ ${num} -gt 1 ]]; do
+#      #shellcheck disable=SC2086
+#      let prev=${num}-1
+#      if [[ -f "${log}.${prev}.gz" ]]; then
+#        mv "${log}.${prev}.gz" "${log}.${num}.gz"
+#      fi
+#      num=${prev}
+#    done
+#    mv "${log}" "${log}.${num}"
+#    gzip -9 "${log}.${num}"
+#  fi
+#}
+#
+#
+
+#
+# Another example:  finding java
+#
+# By default, Hadoop assumes that $JAVA_HOME is always defined
+# outside of its configuration. Eons ago, Apple standardized
+# on a helper program called java_home to find it for you.
+#
+#function hadoop_java_setup
+#{
+#
+#  if [[ -z "${JAVA_HOME}" ]]; then
+#     case $HADOOP_OS_TYPE in
+#       Darwin*)
+#          JAVA_HOME=$(/usr/libexec/java_home)
+#          ;;
+#     esac
+#  fi
+#
+#  # Bail if we did not detect it
+#  if [[ -z "${JAVA_HOME}" ]]; then
+#    echo "ERROR: JAVA_HOME is not set and could not be found." 1>&2
+#    exit 1
+#  fi
+#
+#  if [[ ! -d "${JAVA_HOME}" ]]; then
+#     echo "ERROR: JAVA_HOME (${JAVA_HOME}) does not exist." 1>&2
+#     exit 1
+#  fi
+#
+#  JAVA="${JAVA_HOME}/bin/java"
+#
+#  if [[ ! -x ${JAVA} ]]; then
+#    echo "ERROR: ${JAVA} is not executable." 1>&2
+#    exit 1
+#  fi
+#  JAVA_HEAP_MAX=-Xmx1g
+#  HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-128}
+#
+#  # check envvars which might override default args
+#  if [[ -n "$HADOOP_HEAPSIZE" ]]; then
+#    JAVA_HEAP_MAX="-Xmx${HADOOP_HEAPSIZE}m"
+#  fi
+#}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7971c97e/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
index 6824028..bb17aa1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
@@ -20,8 +20,11 @@
 
 function hadoop_subproject_init
 {
-  if [[ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]]; then
-    . "${HADOOP_CONF_DIR}/hdfs-env.sh"
+  if [[ -z "${HADOOP_HDFS_ENV_PROCESSED}" ]]; then
+    if [[ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]]; then
+      . "${HADOOP_CONF_DIR}/hdfs-env.sh"
+      export HADOOP_HDFS_ENV_PROCESSED=true
+    fi
   fi
   
   # at some point in time, someone thought it would be a good idea to

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7971c97e/hadoop-mapreduce-project/bin/mapred-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh
index c2681ac..cc62b9e 100644
--- a/hadoop-mapreduce-project/bin/mapred-config.sh
+++ b/hadoop-mapreduce-project/bin/mapred-config.sh
@@ -20,8 +20,11 @@
 
 function hadoop_subproject_init
 {
-  if [[ -e "${HADOOP_CONF_DIR}/mapred-env.sh" ]]; then
-    . "${HADOOP_CONF_DIR}/mapred-env.sh"
+  if [[ -z "${HADOOP_MAPRED_ENV_PROCESSED}" ]]; then
+    if [[ -e "${HADOOP_CONF_DIR}/mapred-env.sh" ]]; then
+      . "${HADOOP_CONF_DIR}/mapred-env.sh"
+      export HADOOP_MAPRED_ENV_PROCESSED=true
+    fi
   fi
   
   # at some point in time, someone thought it would be a good idea to

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7971c97e/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
index d83e998..372e8ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
@@ -24,10 +24,13 @@ function hadoop_subproject_init
   # ...
   # this should get deprecated at some point.
   
-  if [[ -e "${YARN_CONF_DIR}/yarn-env.sh" ]]; then
-    . "${YARN_CONF_DIR}/yarn-env.sh"
-  elif [[ -e "${HADOOP_CONF_DIR}/yarn-env.sh" ]]; then
-    . "${HADOOP_CONF_DIR}/yarn-env.sh"
+  if [[ -z "${HADOOP_YARN_ENV_PROCESSED}" ]]; then
+    if [[ -e "${YARN_CONF_DIR}/yarn-env.sh" ]]; then
+      . "${YARN_CONF_DIR}/yarn-env.sh"
+    elif [[ -e "${HADOOP_CONF_DIR}/yarn-env.sh" ]]; then
+      . "${HADOOP_CONF_DIR}/yarn-env.sh"
+    fi
+    export HADOOP_YARN_ENV_PROCESSED=true
   fi
   
   if [[ -n "${YARN_CONF_DIR}" ]]; then


Mime
View raw message