ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [2/2] ambari git commit: HADOOP-11764. [ HADOOP-11764] NodeManager should use directory other than tmp for extracting and loading leveldbjni (aonishuk)
Date Fri, 11 Sep 2015 14:16:35 GMT
HADOOP-11764. [ HADOOP-11764] NodeManager should use directory other than tmp for extracting
and loading leveldbjni (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d9f600e2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d9f600e2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d9f600e2

Branch: refs/heads/branch-2.1
Commit: d9f600e23d42544a3d8f67e2da75dc3db98bc555
Parents: cfa8023
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Fri Sep 11 17:16:26 2015 +0300
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Fri Sep 11 17:16:26 2015 +0300

----------------------------------------------------------------------
 .../common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml | 2 ++
 .../YARN/2.1.0.2.0/package/scripts/params_linux.py            | 1 +
 .../stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py       | 2 ++
 .../2.0.6/hooks/before-ANY/scripts/shared_initialization.py   | 7 +++++++
 .../stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml   | 2 ++
 .../HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml | 2 ++
 .../stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml   | 2 ++
 .../python/stacks/2.0.6/hooks/before-ANY/test_before_any.py   | 5 +++++
 8 files changed, 23 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
index a1dfa57..6eb1a1d 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
@@ -85,6 +85,7 @@ export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER
 export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
 export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
 export JAVA_HOME={{java64_home}}
+export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
 
 # User for YARN daemons
 export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
@@ -190,6 +191,7 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
   YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
 fi
 YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
     </value>
   </property>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 70ae4b8..9b242ea 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -74,6 +74,7 @@ hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce"
 mapred_bin = "/usr/lib/hadoop-mapreduce/sbin"
 yarn_bin = "/usr/lib/hadoop-yarn/sbin"
 yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
+hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir")
 
 # hadoop parameters for 2.2+
 if Script.is_hdp_stack_greater_or_equal("2.2"):

http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
index 8abd72b..d871751 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
@@ -19,6 +19,7 @@ limitations under the License.
 
 import collections
 import re
+import os
 
 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json
module and has the same functions set.
 
@@ -94,6 +95,7 @@ hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
 hadoop_secure_dn_user = hdfs_user
 hadoop_dir = "/etc/hadoop"
 versioned_hdp_root = '/usr/hdp/current'
+hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir")
 
 # HDP 2.2+ params
 if Script.is_hdp_stack_greater_or_equal("2.2"):

http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
index bd44caf..c2cbe3c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
@@ -155,6 +155,13 @@ def setup_hadoop_env():
         group=params.user_group,
         content=InlineTemplate(params.hadoop_env_sh_template))
 
+    # Create tmp dir for java.io.tmpdir
+    # Handle a situation when /tmp is set to noexec
+    Directory(params.hadoop_java_io_tmpdir,
+              owner=params.hdfs_user,
+              group=params.user_group,
+              mode=0777
+    )
 
 def setup_java():
   """

http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
index 4c99823..9ad95ac 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
@@ -37,6 +37,7 @@ export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER
 export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
 export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
 export JAVA_HOME={{java64_home}}
+export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
 
 # User for YARN daemons
 export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
@@ -142,6 +143,7 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
   YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
 fi
 YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
     </value>
   </property>  
   

http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml
index b5280ce..0046eb9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml
@@ -41,6 +41,8 @@ export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
 #export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER
by default
 #export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
 export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS"
+export HADOOP_OPTS="-Djava.io.tmpdir={{hadoop_java_io_tmpdir}} $HADOOP_OPTS"
+export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
     </value>
   </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml
index 44f3dc2..699463b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml
@@ -35,6 +35,7 @@
       export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
       export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
       export JAVA_HOME={{java64_home}}
+      export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
 
       # We need to add the EWMA appender for the yarn daemons only;
       # however, YARN_ROOT_LOGGER is shared by the yarn client and the
@@ -148,6 +149,7 @@
       YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
       fi
       YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+      YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
     </value>
   </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
index 39fd3d5..242dc44 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
@@ -163,6 +163,11 @@ class TestHookBeforeInstall(RMFTestCase):
         owner = 'hdfs',
         group = 'hadoop'
     )
+    self.assertResourceCalled('Directory', '/tmp/hadoop_java_io_tmpdir',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              mode = 0777
+    )
 
     self.assertResourceCalled('Directory', '/tmp/AMBARI-artifacts/',
                               recursive = True,


Mime
View raw message