incubator-ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject git commit: AMBARI-2907. yarn-env.sh is missing few critical attributes. (smohanty)
Date Thu, 15 Aug 2013 05:51:22 GMT
Updated Branches:
  refs/heads/trunk 9c3940bd0 -> 32399749c


AMBARI-2907. yarn-env.sh is missing few critical attributes. (smohanty)


Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/32399749
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/32399749
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/32399749

Branch: refs/heads/trunk
Commit: 32399749c9394cef0088328583f192c6e548165c
Parents: 9c3940b
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Wed Aug 14 22:50:58 2013 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Wed Aug 14 22:50:58 2013 -0700

----------------------------------------------------------------------
 .../puppet/modules/hdp-yarn/manifests/params.pp |   9 +-
 .../modules/hdp-yarn/templates/yarn-env.sh.erb  | 172 +++++++++++--------
 .../services/YARN/configuration/global.xml      |  15 ++
 .../services/YARN/configuration/global.xml      |  15 ++
 4 files changed, 130 insertions(+), 81 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
index d09483b..8916196 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
@@ -32,12 +32,11 @@ class hdp-yarn::params(
 
   ## yarn-env 
   $hadoop_libexec_dir = hdp_default("yarn/yarn-env/hadoop_libexec_dir","/usr/lib/hadoop/libexec")
-  
-  $hadoop_common_home = hdp_default("yarn/yarn-env/hadoop_common_home","/usr/lib/hadoop")
-  $hadoop_hdfs_home = hdp_default("yarn/yarn-env/hadoop_hdfs_home","/usr/lib/hadoop-hdfs")
-  $hadoop_mapred_home = hdp_default("yarn/yarn-env/hadoop_mapred_home","/usr/lib/hadoop-mapreduce")
   $hadoop_yarn_home = hdp_default("yarn/yarn-env/hadoop_yarn_home","/usr/lib/hadoop-yarn")
-  
+  $yarn_heapsize = hdp_default("yarn/yarn-env/yarn_heapsize","1024")
+  $resourcemanager_heapsize = hdp_default("yarn/yarn-env/resourcemanager_heapsize","1024")
+  $nodemanager_heapsize = hdp_default("yarn/yarn-env/nodemanager_heapsize","1024")
+
   $yarn_log_dir_prefix = hdp_default("hadoop/yarn-env/yarn_log_dir_prefix","/var/log/hadoop-yarn")
   $yarn_pid_dir_prefix = hdp_default("hadoop/yarn-env/yarn_pid_dir_prefix","/var/run/hadoop-yarn")
   

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb b/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb
index d1bd482..4bf1779 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb
+++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb
@@ -16,84 +16,104 @@
 # * limitations under the License.
 # */
 
-# Set Hadoop-specific environment variables here.
-
-# The only required environment variable is JAVA_HOME.  All others are
-# optional.  When running a distributed configuration it is best to
-# set JAVA_HOME in this file, so that it is correctly defined on
-# remote nodes.
-
-export HADOOP_COMMON_HOME=<%=scope.function_hdp_template_var("hadoop_common_home")%>
-export HADOOP_HDFS_HOME=<%=scope.function_hdp_template_var("hadoop_hdfs_home")%>
-export HADOOP_MAPRED_HOME=<%=scope.function_hdp_template_var("hadoop_mapred_home")%>
 export HADOOP_YARN_HOME=<%=scope.function_hdp_template_var("hadoop_yarn_home")%>
-
 export YARN_LOG_DIR=<%=scope.function_hdp_template_var("yarn_log_dir_prefix")%>/$USER
 export YARN_PID_DIR=<%=scope.function_hdp_template_var("yarn_pid_dir_prefix")%>/$USER
-
-export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec
-
-# The java implementation to use.  Required.
+export HADOOP_LIBEXEC_DIR=<%=scope.function_hdp_template_var("hadoop_libexec_dir")%>
 export JAVA_HOME=<%=scope.function_hdp_java_home()%>
-export HADOOP_HOME_WARN_SUPPRESS=1
-
-# Hadoop Configuration Directory
-#TODO: if env var set that can cause problems
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-export HADOOP_HEAPSIZE="1024"
-
-export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms1024m"
-
-# Extra Java runtime options.  Empty by default.
-export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}"
-
-# Command specific options appended to HADOOP_OPTS when specified
-export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log
-XX:NewSize=200m -XX:MaxNewSize=640m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'`
-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m
-Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}"
-HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log
-XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'`
-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS
-Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
-
-HADOOP_TASKTRACKER_OPTS="-server -Xmx1024m -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console
${HADOOP_TASKTRACKER_OPTS}"
-HADOOP_DATANODE_OPTS="-Xmx1024m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}"
-HADOOP_BALANCER_OPTS="-server -Xmx1024m ${HADOOP_BALANCER_OPTS}"
-
-export HADOOP_SECONDARYNAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC
-XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=640m
-Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails
-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS
-Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}"
-
-# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
-export HADOOP_CLIENT_OPTS="-Xmx128m ${HADOOP_CLIENT_OPTS}"
-#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData ${HADOOP_JAVA_PLATFORM_OPTS}"
-
-# On secure datanodes, user to run the datanode as after dropping privileges
-export HADOOP_SECURE_DN_USER=hdfs
-
-# Extra ssh options.  Empty by default.
-export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR"
-
-# Where log files are stored.  $HADOOP_HOME/logs by default.
-export HADOOP_LOG_DIR=/var/log/hadoop/$USER
-
-
-# Where log files are stored in the secure data environment.
-export HADOOP_SECURE_DN_LOG_DIR=/var/log/hadoop/$HADOOP_SECURE_DN_USER
-
-# File naming remote slave hosts.  $HADOOP_HOME/conf/slaves by default.
-# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves
-
-# host:path where hadoop code should be rsync'd from.  Unset by default.
-# export HADOOP_MASTER=master:/home/$USER/src/hadoop
-
-# Seconds to sleep between slave commands.  Unset by default.  This
-# can be useful in large clusters, where, e.g., slave rsyncs can
-# otherwise arrive faster than the master can service them.
-# export HADOOP_SLAVE_SLEEP=0.1
-
-# The directory where pid files are stored. /tmp by default.
-export HADOOP_PID_DIR=/var/run/hadoop/$USER
-export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop/$HADOOP_SECURE_DN_USER
-
-# A string representing this instance of hadoop. $USER by default.
-export HADOOP_IDENT_STRING=$USER
-
-# The scheduling priority for daemon processes.  See 'man nice'.
 
-# export HADOOP_NICENESS=10
+# User for YARN daemons
+export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+# resolve links - $0 may be a softlink
+export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+
+# some Java parameters
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+if [ "$JAVA_HOME" != "" ]; then
+  #echo "run java in $JAVA_HOME"
+  JAVA_HOME=$JAVA_HOME
+fi
+
+if [ "$JAVA_HOME" = "" ]; then
+  echo "Error: JAVA_HOME is not set."
+  exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m
+
+# For setting YARN specific HEAP sizes please use this
+# Parameter and set appropriately
+YARN_HEAPSIZE=<%=scope.function_hdp_template_var("yarn_heapsize")%>
+
+# check envvars which might override default args
+if [ "$YARN_HEAPSIZE" != "" ]; then
+  JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+fi
+
+# Resource Manager specific parameters
+
+# Specify the max Heapsize for the ResourceManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_RESOURCEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_RESOURCEMANAGER_HEAPSIZE=<%=scope.function_hdp_template_var("resourcemanager_heapsize")%>
+
+# Specify the JVM options to be used when starting the ResourceManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_RESOURCEMANAGER_OPTS=
+
+# Node Manager specific parameters
+
+# Specify the max Heapsize for the NodeManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_NODEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_NODEMANAGER_HEAPSIZE=<%=scope.function_hdp_template_var("nodemanager_heapsize")%>
+
+# Specify the JVM options to be used when starting the NodeManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_NODEMANAGER_OPTS=
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+
+# default log directory & file
+if [ "$YARN_LOG_DIR" = "" ]; then
+  YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
+fi
+if [ "$YARN_LOGFILE" = "" ]; then
+  YARN_LOGFILE='yarn.log'
+fi
+
+# default policy file for service-level authorization
+if [ "$YARN_POLICYFILE" = "" ]; then
+  YARN_POLICYFILE="hadoop-policy.xml"
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
+YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
+YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+  YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi
+YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml
b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml
index 7ddd6e3..edd1636 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml
@@ -46,4 +46,19 @@
     <value>yarn</value>
     <description>YARN User</description>
   </property>
+  <property>
+    <name>yarn_heapsize</name>
+    <value>1024</value>
+    <description>Max heapsize for all YARN components using a numerical value in the
scale of MB</description>
+  </property>
+  <property>
+    <name>resourcemanager_heapsize</name>
+    <value>1024</value>
+    <description>Max heapsize for ResourceManager using a numerical value in the scale
of MB</description>
+  </property>
+  <property>
+    <name>nodemanager_heapsize</name>
+    <value>1024</value>
+    <description>Max heapsize for NodeManager using a numerical value in the scale
of MB</description>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml
b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml
index 7ddd6e3..7d43a97 100644
--- a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml
+++ b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml
@@ -46,4 +46,19 @@
     <value>yarn</value>
     <description>YARN User</description>
   </property>
+  <property>
+    <name>yarn_heapsize</name>
+    <value>1000</value>
+    <description>Max heapsize for all YARN components using a numerical value in the
scale of MB</description>
+  </property>
+  <property>
+    <name>resourcemanager_heapsize</name>
+    <value>1000</value>
+    <description>Max heapsize for ResourceManager using a numerical value in the scale
of MB</description>
+  </property>
+  <property>
+    <name>nodemanager_heapsize</name>
+    <value>1000</value>
+    <description>Max heapsize for NodeManager using a numerical value in the scale
of MB</description>
+  </property>
 </configuration>


Mime
View raw message