ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srima...@apache.org
Subject ambari git commit: AMBARI-11627. Default Map memory should be less than default reduce memory (srimanth)
Date Tue, 02 Jun 2015 18:13:02 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk b40d808d3 -> 8dcade74e


AMBARI-11627. Default Map memory should be less than default reduce memory (srimanth)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8dcade74
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8dcade74
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8dcade74

Branch: refs/heads/trunk
Commit: 8dcade74e2c669f0ec15e5562232fe12d51ca57e
Parents: b40d808
Author: Srimanth Gunturi <sgunturi@hortonworks.com>
Authored: Tue Jun 2 09:50:16 2015 -0700
Committer: Srimanth Gunturi <sgunturi@hortonworks.com>
Committed: Tue Jun 2 11:12:56 2015 -0700

----------------------------------------------------------------------
 .../stacks/HDP/2.2/services/stack_advisor.py       |  6 ++++--
 .../stacks/HDPWIN/2.2/services/stack_advisor.py    | 17 +++++++++++++----
 .../python/stacks/2.2/common/test_stack_advisor.py | 10 +++++-----
 3 files changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8dcade74/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 1e31662..3ea3167 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -634,14 +634,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       nodemanagerMinRam = min(nodemanager["Hosts"]["total_mem"]/1024, nodemanagerMinRam)
     putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
     putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"])))
+ "m" + " -Dhdp.version=${hdp.version}")
-    putMapredProperty('mapreduce.reduce.memory.mb', min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),
int(nodemanagerMinRam)))
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
     min_mapreduce_map_memory_mb = 0
+    min_mapreduce_reduce_memory_mb = 0
     min_mapreduce_map_java_opts = 0
     if ("PIG" in servicesList):
-      min_mapreduce_map_memory_mb = 1500
+      min_mapreduce_map_memory_mb = 1536
+      min_mapreduce_reduce_memory_mb = 1536
       min_mapreduce_map_java_opts = 1024
     putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
+    putMapredProperty('mapreduce.reduce.memory.mb', max(min_mapreduce_reduce_memory_mb, min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),
int(nodemanagerMinRam))))
     mapredMapXmx = int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]));
     putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(max(min_mapreduce_map_java_opts,
mapredMapXmx)) + "m")
     putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"])))
+ "m")

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dcade74/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
index 3b7c948..058b618 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
@@ -230,11 +230,20 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor):
     putMapredProperty = self.putProperty(configurations, "mapred-site", services)
     putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
     putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"])))
+ "m" + " -Dhdp.version=${hdp.version}")
-    putMapredProperty('mapreduce.map.memory.mb', int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
-    putMapredProperty('mapreduce.reduce.memory.mb', int(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
-    putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"])))
+ "m")
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    min_mapreduce_map_memory_mb = 0
+    min_mapreduce_reduce_memory_mb = 0
+    min_mapreduce_map_java_opts = 0
+    if ("PIG" in servicesList):
+      min_mapreduce_map_memory_mb = 1536
+      min_mapreduce_reduce_memory_mb = 1536
+      min_mapreduce_map_java_opts = 1024
+    putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
+    putMapredProperty('mapreduce.reduce.memory.mb', max(min_mapreduce_reduce_memory_mb, min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]),
int(nodemanagerMinRam))))
+    mapredMapXmx = int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]));
+    putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(max(min_mapreduce_map_java_opts,
mapredMapXmx)) + "m")
     putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"])))
+ "m")
-    putMapredProperty('mapreduce.task.io.sort.mb', str(int(0.7*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]))))
+    putMapredProperty('mapreduce.task.io.sort.mb', str(min(int(0.7*mapredMapXmx), 2047)))
     # Property Attributes
     putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site")
     yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dcade74/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 3a641f1..8d462d4 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1209,13 +1209,13 @@ class TestHDP22StackAdvisor(TestCase):
       },
       "mapred-site": {
         "properties": {
-          "mapreduce.map.memory.mb": "1500",
-          "mapreduce.reduce.memory.mb": "200",
+          "mapreduce.map.memory.mb": "1536",
+          "mapreduce.reduce.memory.mb": "1536",
           "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}",
-          "mapreduce.reduce.java.opts": "-Xmx160m",
+          "mapreduce.reduce.java.opts": "-Xmx1228m",
           "yarn.app.mapreduce.am.resource.mb": "100",
-          "mapreduce.map.java.opts": "-Xmx1200m",
-          "mapreduce.task.io.sort.mb": "840"
+          "mapreduce.map.java.opts": "-Xmx1228m",
+          "mapreduce.task.io.sort.mb": "859"
         },
         "property_attributes": {
           'mapreduce.task.io.sort.mb': {'maximum': '2047'},


Mime
View raw message