ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [44/50] [abbrv] ambari git commit: AMBARI-20821. Update default values for YARN preemption in Ambari.(vbrodetskyi)
Date Mon, 24 Apr 2017 12:01:59 GMT
AMBARI-20821. Update default values for YARN preemption in Ambari.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d1dce4c8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d1dce4c8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d1dce4c8

Branch: refs/heads/branch-3.0-perf
Commit: d1dce4c8077e38bc8c67ccae4934819ebb869876
Parents: 978c9d7
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Fri Apr 21 18:53:06 2017 +0300
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Mon Apr 24 14:59:56 2017 +0300

----------------------------------------------------------------------
 .../YARN/3.0.0.3.0/configuration/yarn-site.xml          | 12 ++++++++++++
 .../common-services/YARN/3.0.0.3.0/service_advisor.py   |  4 ++++
 .../HDP/2.6/services/YARN/configuration/yarn-site.xml   | 12 ++++++++++++
 .../resources/stacks/HDP/2.6/services/stack_advisor.py  |  5 +++++
 .../test/python/stacks/2.6/common/test_stack_advisor.py |  3 +++
 5 files changed, 36 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/d1dce4c8/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
index 394fae3..cc96cd7 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
@@ -1177,4 +1177,16 @@ yarn.node-labels.manager-class
     </value-attributes>
     <on-ambari-upgrade add="false"/>
   </property>
+  <property>
+    <name>yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round</name>
+    <value>0.1</value>
+    <description>This option controls the pace at which containers-marked-for-preemption
are actually preempted in each period.</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor</name>
+    <value>1</value>
+    <description>Similar to total_preemption_per_round, we can apply this factor to
slowdown resource preemption after preemption-target is computed for each queue.</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/d1dce4c8/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
index 1ac7849..34fe117 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
@@ -430,6 +430,10 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
       else:
         putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled',
"false")
 
+    # calculate total_preemption_per_round
+    total_preemption_per_round = str(round(max(float(1)/len(hosts['items']), 0.1),2))
+    putYarnSiteProperty('yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round',
total_preemption_per_round)
+
     if 'yarn-env' in services['configurations'] and 'yarn_user' in services['configurations']['yarn-env']['properties']:
       yarn_user = services['configurations']['yarn-env']['properties']['yarn_user']
     else:

http://git-wip-us.apache.org/repos/asf/ambari/blob/d1dce4c8/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
index acdae65..c434c12 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
@@ -42,6 +42,18 @@
     <on-ambari-upgrade add="false"/>
   </property>
   <property>
+    <name>yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round</name>
+    <value>0.1</value>
+    <description>This option controls the pace at which containers-marked-for-preemption
are actually preempted in each period.</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor</name>
+    <value>1</value>
+    <description>Similar to total_preemption_per_round, we can apply this factor to
slowdown resource preemption after preemption-target is computed for each queue.</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
     <name>yarn.timeline-service.entity-group-fs-store.app-cache-size</name>
     <value>10</value>
     <description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/d1dce4c8/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 4e1b4b6..da3c98f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -189,6 +189,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
       else:
         putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled',
"false")
 
+    # calculate total_preemption_per_round
+    total_preemption_per_round = str(round(max(float(1)/len(hosts['items']), 0.1),2))
+    putYarnSiteProperty('yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round',
total_preemption_per_round)
+
+
     if 'yarn-env' in services['configurations'] and 'yarn_user' in services['configurations']['yarn-env']['properties']:
       yarn_user = services['configurations']['yarn-env']['properties']['yarn_user']
     else:

http://git-wip-us.apache.org/repos/asf/ambari/blob/d1dce4c8/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
index 114dd3a..d4d28c9 100644
--- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
@@ -1162,6 +1162,7 @@ class TestHDP26StackAdvisor(TestCase):
           'yarn.timeline-service.entity-group-fs-store.app-cache-size': '10',
           'yarn.scheduler.minimum-allocation-mb': '256',
           'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
+          'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round':
'1.0',
           'yarn.nodemanager.resource.cpu-vcores': '4',
           'yarn.scheduler.maximum-allocation-mb': '768',
           'yarn.nodemanager.linux-container-executor.group': 'hadoop',
@@ -1336,6 +1337,7 @@ class TestHDP26StackAdvisor(TestCase):
           'yarn.nodemanager.resource.memory-mb': '1280',
           'yarn.scheduler.minimum-allocation-mb': '256',
           'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
+          'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round':
'1.0',
           'yarn.nodemanager.resource.cpu-vcores': '4',
           'yarn.scheduler.maximum-allocation-mb': '1280',
           'yarn.nodemanager.linux-container-executor.group': 'hadoop',
@@ -1442,6 +1444,7 @@ class TestHDP26StackAdvisor(TestCase):
           'yarn.nodemanager.resource.memory-mb': '1280',
           'yarn.scheduler.minimum-allocation-mb': '256',
           'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
+          'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round':
'1.0',
           'yarn.nodemanager.resource.cpu-vcores': '4',
           'yarn.scheduler.maximum-allocation-mb': '1280',
           'yarn.nodemanager.linux-container-executor.group': 'hadoop',


Mime
View raw message