ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject ambari git commit: AMBARI-14099. Tez config relationship dependency not taking effect (aonishuk)
Date Fri, 27 Nov 2015 14:51:39 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk ebdda3bcf -> 01f7d709e


AMBARI-14099. Tez config relationship dependency not taking effect  (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/01f7d709
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/01f7d709
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/01f7d709

Branch: refs/heads/trunk
Commit: 01f7d709e8f49b3d2095ebb4de6c269e4aad1230
Parents: ebdda3b
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Fri Nov 27 16:51:28 2015 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Fri Nov 27 16:51:28 2015 +0200

----------------------------------------------------------------------
 .../HDP/2.2/services/TEZ/configuration/tez-site.xml     | 12 ++++++++++++
 .../resources/stacks/HDP/2.2/services/stack_advisor.py  |  3 ++-
 .../resources/stacks/HDP/2.3/services/stack_advisor.py  |  3 +--
 3 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/01f7d709/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
index 33920d6..3a04f70 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
@@ -304,12 +304,24 @@
     <name>tez.runtime.io.sort.mb</name>
     <value>512</value>
     <description>The size of the sort buffer when output needs to be sorted</description>
+    <depends-on>
+      <property>
+        <type>tez-site</type>
+        <name>tez.task.resource.memory.mb</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>
     <name>tez.runtime.unordered.output.buffer.size-mb</name>
     <value>100</value>
     <description>The size of the buffer when output does not require to be sorted</description>
+    <depends-on>
+      <property>
+        <type>tez-site</type>
+        <name>tez.task.resource.memory.mb</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/01f7d709/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index c9b3aa3..a33be1e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -744,12 +744,13 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     #properties below should be always present as they are provided in HDP206 stack advisor
     yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),
int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
 
-    putTezProperty = self.putProperty(configurations, "tez-site")
+    putTezProperty = self.putProperty(configurations, "tez-site", services)
     putTezProperty("tez.am.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]),
int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory'])))
 
     taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else
int(clusterData['reduceMemory'])
     taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'],
taskResourceMemory, yarnMaxAllocationSize)
     putTezProperty("tez.task.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]),
taskResourceMemory))
+    taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])
     putTezProperty("tez.runtime.io.sort.mb", min(int(taskResourceMemory * 0.4), 2047))
     putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory
* 0.075))
     putTezProperty("tez.session.am.dag.submit.timeout.secs", "600")

http://git-wip-us.apache.org/repos/asf/ambari/blob/01f7d709/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 419fb3a..a5e419a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -128,8 +128,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     putTezProperty = self.putProperty(configurations, "tez-site")
     # remove 2gb limit for tez.runtime.io.sort.mb
     # in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in other case
comment calculation code below
-    taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else
int(clusterData['reduceMemory'])
-    taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'],
taskResourceMemory)
+    taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])
     putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4))
 
     if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]:


Mime
View raw message