ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-17818. Tez & Hive: Memory management should use scaled down Xmx for sizing buffers. (Gopal V via Swapan Shridhar)
Date Thu, 21 Jul 2016 21:39:14 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 f270a5071 -> 72d34c96e


AMBARI-17818. Tez & Hive: Memory management should use scaled down Xmx for sizing buffers.
(Gopal V via Swapan Shridhar)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/72d34c96
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/72d34c96
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/72d34c96

Branch: refs/heads/branch-2.4
Commit: 72d34c96ecf366d529ead19fffd5e52ef80aae42
Parents: f270a50
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Thu Jul 21 14:38:58 2016 -0700
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Thu Jul 21 14:38:58 2016 -0700

----------------------------------------------------------------------
 .../resources/stacks/HDP/2.2/services/stack_advisor.py    |  2 +-
 .../resources/stacks/HDP/2.3/services/stack_advisor.py    |  3 ++-
 .../test/python/stacks/2.2/common/test_stack_advisor.py   |  4 ++--
 .../test/python/stacks/2.3/common/test_stack_advisor.py   | 10 +++++-----
 4 files changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/72d34c96/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 1bc5da7..fc38ced 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -364,7 +364,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     putHiveSiteProperty("hive.tez.dynamic.partition.pruning", "true")
 
     container_size = configurations["hive-site"]["properties"]["hive.tez.container.size"]
-    container_size_bytes = int(container_size)*1024*1024
+    container_size_bytes = int(int(container_size)*0.8*1024*1024) # Xmx == 80% of container
     # Memory
     putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", int(round(container_size_bytes/3)))
     putHiveSitePropertyAttribute("hive.auto.convert.join.noconditionaltask.size", "maximum",
container_size_bytes)

http://git-wip-us.apache.org/repos/asf/ambari/blob/72d34c96/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 24047ce..8d5cdc9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -92,7 +92,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     # remove 2gb limit for tez.runtime.io.sort.mb
     # in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in other case
comment calculation code below
     taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])
-    putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4))
+    # fit io.sort.mb into tenured regions
+    putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.8 * 0.33))
 
     if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]:
       if services["configurations"]["tez-site"]["properties"]["tez.runtime.sorter.class"]
== "LEGACY":

http://git-wip-us.apache.org/repos/asf/ambari/blob/72d34c96/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 208ee8c..818390d 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1203,7 +1203,7 @@ class TestHDP22StackAdvisor(TestCase):
           'hive.server2.tez.default.queues': "queue1,queue2",
           'hive.server2.tez.initialize.default.sessions': 'false',
           'hive.server2.tez.sessions.per.default.queue': '1',
-          'hive.auto.convert.join.noconditionaltask.size': '268435456',
+          'hive.auto.convert.join.noconditionaltask.size': '214748364',
           'hive.compactor.initiator.on': 'false',
           'hive.compactor.worker.threads': '0',
           'hive.compute.query.using.stats': 'true',
@@ -1238,7 +1238,7 @@ class TestHDP22StackAdvisor(TestCase):
           'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
         },
        'property_attributes': {
-         'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
+         'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'},
          'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'},
          'hive.server2.authentication.pam.services': {'delete': 'true'},
          'hive.server2.custom.authentication.class': {'delete': 'true'},

http://git-wip-us.apache.org/repos/asf/ambari/blob/72d34c96/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index faad01c..34bebe8 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -617,7 +617,7 @@ class TestHDP23StackAdvisor(TestCase):
           'hive.server2.tez.default.queues': "queue1,queue2",
           'hive.server2.tez.initialize.default.sessions': 'false',
           'hive.server2.tez.sessions.per.default.queue': '1',
-          'hive.auto.convert.join.noconditionaltask.size': '268435456',
+          'hive.auto.convert.join.noconditionaltask.size': '214748364',
           'hive.compactor.initiator.on': 'false',
           'hive.compactor.worker.threads': '0',
           'hive.compute.query.using.stats': 'true',
@@ -652,7 +652,7 @@ class TestHDP23StackAdvisor(TestCase):
           'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
         },
        'property_attributes': {
-         'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
+         'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'},
          'hive.server2.authentication.pam.services': {'delete': 'true'},
          'hive.server2.custom.authentication.class': {'delete': 'true'},
          'hive.server2.authentication.kerberos.principal': {'delete': 'true'},
@@ -884,7 +884,7 @@ class TestHDP23StackAdvisor(TestCase):
           'hive.server2.tez.default.queues': "queue1,queue2",
           'hive.server2.tez.initialize.default.sessions': 'false',
           'hive.server2.tez.sessions.per.default.queue': '1',
-          'hive.auto.convert.join.noconditionaltask.size': '268435456',
+          'hive.auto.convert.join.noconditionaltask.size': '214748364',
           'hive.compactor.initiator.on': 'false',
           'hive.compactor.worker.threads': '0',
           'hive.compute.query.using.stats': 'true',
@@ -919,7 +919,7 @@ class TestHDP23StackAdvisor(TestCase):
           'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
         },
         'property_attributes': {
-          'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
+          'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'},
           'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'},
           'hive.server2.authentication.pam.services': {'delete': 'true'},
           'hive.server2.custom.authentication.class': {'delete': 'true'},
@@ -1161,7 +1161,7 @@ class TestHDP23StackAdvisor(TestCase):
           "tez.task.resource.memory.mb": "768",
           "tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps
-XX:+UseNUMA -XX:+UseParallelGC",
           "tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps
-XX:+UseNUMA -XX:+UseParallelGC",
-          "tez.runtime.io.sort.mb": "307",
+          "tez.runtime.io.sort.mb": "202",
           "tez.session.am.dag.submit.timeout.secs": "600",
           "tez.runtime.unordered.output.buffer.size-mb": "57",
           "tez.am.resource.memory.mb": "4000",


Mime
View raw message