ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [21/34] ambari git commit: AMBARI-20763. Update YARN's ATS configs 'apptimelineserver_heapsize' and 'yarn.timeline-service.entity-group-fs-store.app-cache-size' logic in 2.6.
Date Mon, 17 Apr 2017 20:18:15 GMT
AMBARI-20763. Update YARN's ATS configs 'apptimelineserver_heapsize' and 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
logic in 2.6.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/38f84bf1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/38f84bf1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/38f84bf1

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 38f84bf12d04981017bbcfc3d44c243f062ed055
Parents: f894e48
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Thu Apr 13 15:57:10 2017 -0700
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Thu Apr 13 15:57:10 2017 -0700

----------------------------------------------------------------------
 .../YARN/3.0.0.3.0/service_advisor.py           |  76 ++++
 .../services/YARN/configuration/yarn-env.xml    |  18 +
 .../stacks/HDP/2.6/services/stack_advisor.py    |  80 ++++
 .../stacks/2.5/common/test_stack_advisor.py     |   7 +
 .../stacks/2.6/common/test_stack_advisor.py     | 452 ++++++++++++++++++-
 5 files changed, 627 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
index fc32001..1ac7849 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
@@ -420,6 +420,7 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
 
   def recommendYARNConfigurationsFromHDP26(self, configurations, clusterData, services, hosts):
     putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
+    putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services)
 
     if "yarn-site" in services["configurations"] and \
                     "yarn.resourcemanager.scheduler.monitor.enable" in services["configurations"]["yarn-site"]["properties"]:
@@ -466,6 +467,81 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
     else:
       self.logger.info("Not setting Yarn Repo user for Ranger.")
 
+    yarn_timeline_app_cache_size = None
+    host_mem = None
+    for host in hosts["items"]:
+      host_mem = host["Hosts"]["total_mem"]
+      break
+    # Check if 'yarn.timeline-service.entity-group-fs-store.app-cache-size' in changed configs.
+    changed_configs_has_ats_cache_size = self.isConfigPropertiesChanged(
+      services, "yarn-site", ['yarn.timeline-service.entity-group-fs-store.app-cache-size'],
False)
+    # Check if it's : 1. 'apptimelineserver_heapsize' changed detected in changed-configurations)
+    # OR 2. cluster initialization (services['changed-configurations'] should be empty in
this case)
+    if changed_configs_has_ats_cache_size:
+      yarn_timeline_app_cache_size = self.read_yarn_apptimelineserver_cache_size(services)
+    elif 0 == len(services['changed-configurations']):
+      # Fetch host memory from 1st host, to be used for ATS config calculations below.
+      if host_mem is not None:
+        yarn_timeline_app_cache_size = self.calculate_yarn_apptimelineserver_cache_size(host_mem)
+        putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.app-cache-size',
yarn_timeline_app_cache_size)
+        self.logger.info("Updated YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
as : {0}, "
+                         "using 'host_mem' = {1}".format(yarn_timeline_app_cache_size, host_mem))
+      else:
+        self.logger.info("Couldn't update YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
as "
+                         "'host_mem' read = {0}".format(host_mem))
+
+    if yarn_timeline_app_cache_size is not None:
+      # Calculation for 'ats_heapsize' is in MB.
+      ats_heapsize = self.calculate_yarn_apptimelineserver_heapsize(host_mem, yarn_timeline_app_cache_size)
+      putYarnEnvProperty('apptimelineserver_heapsize', ats_heapsize) # Value in MB
+      self.logger.info("Updated YARN config 'apptimelineserver_heapsize' as : {0}, ".format(ats_heapsize))
+
+  """
+  Calculate YARN config 'apptimelineserver_heapsize' in MB.
+  """
+  def calculate_yarn_apptimelineserver_heapsize(self, host_mem, yarn_timeline_app_cache_size):
+    ats_heapsize = None
+    if host_mem < 4096:
+      ats_heapsize = 1024
+    else:
+      ats_heapsize = long(min(math.floor(host_mem/2), long(yarn_timeline_app_cache_size)
* 500 + 3072))
+    return ats_heapsize
+
+  """
+  Calculates for YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size',
based on YARN's NodeManager size.
+  """
+  def calculate_yarn_apptimelineserver_cache_size(self, host_mem):
+    yarn_timeline_app_cache_size = None
+    if host_mem < 4096:
+      yarn_timeline_app_cache_size = 3
+    elif host_mem >= 4096 and host_mem < 8192:
+      yarn_timeline_app_cache_size = 7
+    elif host_mem >= 8192:
+      yarn_timeline_app_cache_size = 10
+    self.logger.info("Calculated and returning 'yarn_timeline_app_cache_size' : {0}".format(yarn_timeline_app_cache_size))
+    return yarn_timeline_app_cache_size
+
+
+  """
+  Reads YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size'.
+  """
+  def read_yarn_apptimelineserver_cache_size(self, services):
+    """
+    :type services dict
+    :rtype str
+    """
+    yarn_ats_app_cache_size = None
+    yarn_ats_app_cache_size_config = "yarn.timeline-service.entity-group-fs-store.app-cache-size"
+    yarn_site_in_services = self.getServicesSiteProperties(services, "yarn-site")
+
+    if yarn_site_in_services and yarn_ats_app_cache_size_config in yarn_site_in_services:
+      yarn_ats_app_cache_size = yarn_site_in_services[yarn_ats_app_cache_size_config]
+      self.logger.info("'yarn.scheduler.minimum-allocation-mb' read from services as : {0}".format(yarn_ats_app_cache_size))
+
+    if not yarn_ats_app_cache_size:
+      self.logger.error("'{0}' was not found in the services".format(yarn_ats_app_cache_size_config))
+
+    return yarn_ats_app_cache_size
 
   #region LLAP
   def updateLlapConfigs(self, configurations, services, hosts, llap_queue_name):

http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml
index d04c3c5..1b2ca68 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml
@@ -25,6 +25,24 @@
     <description>Set to false by default,  needs to be set to true in stacks that use
Ranger Yarn Plugin</description>
     <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>apptimelineserver_heapsize</name>
+    <value>1024</value>
+    <display-name>AppTimelineServer Java heap size</display-name>
+    <description>Max heapsize for AppTimelineServer using a numerical value in the
scale of MB</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <unit>MB</unit>
+      <type>int</type>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+    <depends-on>
+      <property>
+        <type>yarn-site</type>
+        <name>yarn.timeline-service.entity-group-fs-store.app-cache-size</name>
+      </property>
+    </depends-on>
+  </property>
   <!-- yarn-env.sh -->
   <property>
     <name>content</name>

http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 7881917..38f46d7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -16,6 +16,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+import math
+
 import json
 import re
 from resource_management.libraries.functions import format
@@ -146,6 +148,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
   def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP26StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData,
services, hosts)
     putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
+    putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services)
 
     if "yarn-site" in services["configurations"] and \
                     "yarn.resourcemanager.scheduler.monitor.enable" in services["configurations"]["yarn-site"]["properties"]:
@@ -186,6 +189,83 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
     else:
       self.logger.info("Not setting Yarn Repo user for Ranger.")
 
+
+    yarn_timeline_app_cache_size = None
+    host_mem = None
+    for host in hosts["items"]:
+      host_mem = host["Hosts"]["total_mem"]
+      break
+    # Check if 'yarn.timeline-service.entity-group-fs-store.app-cache-size' in changed configs.
+    changed_configs_has_ats_cache_size = self.isConfigPropertiesChanged(
+      services, "yarn-site", ['yarn.timeline-service.entity-group-fs-store.app-cache-size'],
False)
+    # Check if it's : 1. 'apptimelineserver_heapsize' changed detected in changed-configurations)
+    # OR 2. cluster initialization (services['changed-configurations'] should be empty in
this case)
+    if changed_configs_has_ats_cache_size:
+      yarn_timeline_app_cache_size = self.read_yarn_apptimelineserver_cache_size(services)
+    elif 0 == len(services['changed-configurations']):
+      # Fetch host memory from 1st host, to be used for ATS config calculations below.
+      if host_mem is not None:
+        yarn_timeline_app_cache_size = self.calculate_yarn_apptimelineserver_cache_size(host_mem)
+        putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.app-cache-size',
yarn_timeline_app_cache_size)
+        self.logger.info("Updated YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
as : {0}, "
+                    "using 'host_mem' = {1}".format(yarn_timeline_app_cache_size, host_mem))
+      else:
+        self.logger.info("Couldn't update YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
as "
+                    "'host_mem' read = {0}".format(host_mem))
+
+    if yarn_timeline_app_cache_size is not None:
+      # Calculation for 'ats_heapsize' is in MB.
+      ats_heapsize = self.calculate_yarn_apptimelineserver_heapsize(host_mem, yarn_timeline_app_cache_size)
+      putYarnEnvProperty('apptimelineserver_heapsize', ats_heapsize) # Value in MB
+      self.logger.info("Updated YARN config 'apptimelineserver_heapsize' as : {0}, ".format(ats_heapsize))
+
+  """
+  Calculate YARN config 'apptimelineserver_heapsize' in MB.
+  """
+  def calculate_yarn_apptimelineserver_heapsize(self, host_mem, yarn_timeline_app_cache_size):
+    ats_heapsize = None
+    if host_mem < 4096:
+      ats_heapsize = 1024
+    else:
+      ats_heapsize = long(min(math.floor(host_mem/2), long(yarn_timeline_app_cache_size)
* 500 + 3072))
+    return ats_heapsize
+
+  """
+  Calculates for YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size',
based on YARN's NodeManager size.
+  """
+  def calculate_yarn_apptimelineserver_cache_size(self, host_mem):
+    yarn_timeline_app_cache_size = None
+    if host_mem < 4096:
+      yarn_timeline_app_cache_size = 3
+    elif host_mem >= 4096 and host_mem < 8192:
+      yarn_timeline_app_cache_size = 7
+    elif host_mem >= 8192:
+      yarn_timeline_app_cache_size = 10
+    self.logger.info("Calculated and returning 'yarn_timeline_app_cache_size' : {0}".format(yarn_timeline_app_cache_size))
+    return yarn_timeline_app_cache_size
+
+
+  """
+  Reads YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size'.
+  """
+  def read_yarn_apptimelineserver_cache_size(self, services):
+    """
+    :type services dict
+    :rtype str
+    """
+    yarn_ats_app_cache_size = None
+    yarn_ats_app_cache_size_config = "yarn.timeline-service.entity-group-fs-store.app-cache-size"
+    yarn_site_in_services = self.getServicesSiteProperties(services, "yarn-site")
+
+    if yarn_site_in_services and yarn_ats_app_cache_size_config in yarn_site_in_services:
+      yarn_ats_app_cache_size = yarn_site_in_services[yarn_ats_app_cache_size_config]
+      self.logger.info("'yarn.scheduler.minimum-allocation-mb' read from services as : {0}".format(yarn_ats_app_cache_size))
+
+    if not yarn_ats_app_cache_size:
+      self.logger.error("'{0}' was not found in the services".format(yarn_ats_app_cache_size_config))
+
+    return yarn_ats_app_cache_size
+
   def getMetadataConnectionString(self, database_type):
       driverDict = {
           'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true',

http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 4250681..77a06fe 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -926,6 +926,8 @@ class TestHDP25StackAdvisor(TestCase):
         ]
       }
       ],
+      "changed-configurations": [
+      ],
       "configurations": {
         "capacity-scheduler": {
           "properties": {
@@ -4918,6 +4920,9 @@ class TestHDP25StackAdvisor(TestCase):
                                   "capacity-scheduler":{"properties":{
                                     "capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue,leaf\n"
+
                                                           "yarn.scheduler.capacity.root.ndfqueue.queues=ndfqueue1,ndfqueue2\n"}}}
+    services["changed-configurations"]= []
+
+
     hosts = self.prepareHosts([])
     result = self.stackAdvisor.validateConfigurations(services, hosts)
     expectedItems = [
@@ -4948,6 +4953,8 @@ class TestHDP25StackAdvisor(TestCase):
           "stack_versions": ["2.4", "2.3", "2.2", "2.1", "2.0.6"]
         }
       },
+      "changed-configurations": [
+      ],
       "configurations": configurations,
       "services": [],
       "ambari-server-properties": {}}

http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
index 5bfa1a9..2d7322d 100644
--- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
@@ -989,7 +989,26 @@ class TestHDP26StackAdvisor(TestCase):
       "yarnMinContainerSize": 256
     }
 
-    hosts = {}
+    hosts = {
+      "items": [
+      {
+        "Hosts": {
+          "cpu_count": 6,
+          "total_mem": 50331648,
+          "disk_info": [
+            {"mountpoint": "/"},
+            {"mountpoint": "/dev/shm"},
+            {"mountpoint": "/vagrant"},
+            {"mountpoint": "/"},
+            {"mountpoint": "/dev/shm"},
+            {"mountpoint": "/vagrant"}
+          ],
+          "public_host_name": "c6401.ambari.apache.org",
+          "host_name": "c6401.ambari.apache.org"
+        },
+      }
+      ]
+    }
 
     services = {
       "services":
@@ -1007,6 +1026,8 @@ class TestHDP26StackAdvisor(TestCase):
         "stack_name" : "HDP",
         "stack_version": "2.6"
       },
+      "changed-configurations": [
+      ],
       "configurations": configurations,
       "ambari-server-properties": {"ambari-server.user":"ambari_user"}
     }
@@ -1016,6 +1037,7 @@ class TestHDP26StackAdvisor(TestCase):
       'yarn-env': {
         'properties': {
           'min_user_id': '500',
+          'apptimelineserver_heapsize': '8072',
           'service_check.queue.name': 'default'
         }
       },
@@ -1131,11 +1153,16 @@ class TestHDP26StackAdvisor(TestCase):
           'yarn.scheduler.minimum-allocation-vcores': '1',
           'yarn.scheduler.maximum-allocation-vcores': '4',
           'yarn.nodemanager.resource.memory-mb': '768',
+          'yarn.nodemanager.local-dirs': '/hadoop/yarn/local,/dev/shm/hadoop/yarn/local,/vagrant/hadoop/yarn/local',
+          'yarn.nodemanager.log-dirs': '/hadoop/yarn/log,/dev/shm/hadoop/yarn/log,/vagrant/hadoop/yarn/log',
+          'yarn.timeline-service.entity-group-fs-store.app-cache-size': '10',
           'yarn.scheduler.minimum-allocation-mb': '256',
           'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
           'yarn.nodemanager.resource.cpu-vcores': '4',
           'yarn.scheduler.maximum-allocation-mb': '768',
-          'yarn.nodemanager.linux-container-executor.group': 'hadoop'
+          'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+          'yarn.timeline-service.leveldb-state-store.path': '/hadoop/yarn/timeline',
+          'yarn.timeline-service.leveldb-timeline-store.path': '/hadoop/yarn/timeline'
         },
         'property_attributes': {
           'yarn.authorization-provider': {
@@ -1267,6 +1294,8 @@ class TestHDP26StackAdvisor(TestCase):
         "components": []
       }
       ],
+      "changed-configurations": [
+      ],
       "configurations": configurations
     }
 
@@ -1282,7 +1311,8 @@ class TestHDP26StackAdvisor(TestCase):
         'properties': {
           'yarn_user': 'custom_yarn',
           'service_check.queue.name': 'default',
-          'min_user_id': '500'
+          'min_user_id': '500',
+          'apptimelineserver_heapsize': '2048'
         }
       },
       'ranger-yarn-plugin-properties': {
@@ -1304,19 +1334,429 @@ class TestHDP26StackAdvisor(TestCase):
           'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
           'yarn.nodemanager.resource.cpu-vcores': '4',
           'yarn.scheduler.maximum-allocation-mb': '1280',
-          'yarn.nodemanager.linux-container-executor.group': 'hadoop'
+          'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+          'yarn.nodemanager.local-dirs': '/hadoop/yarn/local,/dev/shm/hadoop/yarn/local,/vagrant/hadoop/yarn/local',
+          'yarn.nodemanager.log-dirs': '/hadoop/yarn/log,/dev/shm/hadoop/yarn/log,/vagrant/hadoop/yarn/log',
+          'yarn.timeline-service.entity-group-fs-store.app-cache-size': '7',
+          'yarn.timeline-service.leveldb-state-store.path': '/hadoop/yarn/timeline',
+          'yarn.timeline-service.leveldb-timeline-store.path': '/hadoop/yarn/timeline'
+
         }
       }
     }
 
-    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
None)
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 6,
+            "total_mem": 4096,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          },
+        }
+      ]
+    }
+
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
     self.assertEquals(configurations, expected)
+
     configurations['yarn-env']['properties']['yarn_user'] = 'yarn'
     expected['yarn-env']['properties']['yarn_user'] = 'yarn'
     expected['ranger-yarn-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME']
= 'yarn'
-    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
None)
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+
+
+  def test_recommendYARNConfigurations_for_ats_heapsize_and_cache(self):
+    configurations = {
+      "yarn-env": {
+        "properties": {
+          "yarn_user" : "custom_yarn"
+        }
+      },
+      "ranger-yarn-plugin-properties": {
+        "properties": {
+          "ranger-yarn-plugin-enabled" : "Yes",
+          "REPOSITORY_CONFIG_USERNAME":"yarn"
+        }
+      }
+    }
+    services = {
+      "services" : [{
+        "StackServices": {
+          "service_name" : "YARN",
+          "service_version" : "2.7.3.2.6"
+        },
+        "components": []
+      }
+      ],
+      "changed-configurations": [
+      ],
+      "configurations": configurations
+    }
+
+
+    clusterData = {
+      "cpu": 4,
+      "containers" : 5,
+      "ramPerContainer": 256,
+      "yarnMinContainerSize": 256
+    }
+    expected = {
+      'yarn-env': {
+        'properties': {
+          'yarn_user': 'custom_yarn',
+          'service_check.queue.name': 'default',
+          'min_user_id': '500',
+          'apptimelineserver_heapsize': '1024'
+        }
+      },
+      'ranger-yarn-plugin-properties': {
+        'properties': {
+          'ranger-yarn-plugin-enabled': 'Yes',
+          'REPOSITORY_CONFIG_USERNAME': 'custom_yarn'
+        }
+      },
+      'yarn-site': {
+        'properties': {
+          'hadoop.registry.rm.enabled': 'false',
+          'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes': '',
+          'yarn.authorization-provider': 'org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer',
+          'yarn.acl.enable': 'true',
+          'yarn.scheduler.minimum-allocation-vcores': '1',
+          'yarn.scheduler.maximum-allocation-vcores': '4',
+          'yarn.nodemanager.resource.memory-mb': '1280',
+          'yarn.scheduler.minimum-allocation-mb': '256',
+          'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
+          'yarn.nodemanager.resource.cpu-vcores': '4',
+          'yarn.scheduler.maximum-allocation-mb': '1280',
+          'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+          'yarn.nodemanager.local-dirs': '/hadoop/yarn/local,/dev/shm/hadoop/yarn/local,/vagrant/hadoop/yarn/local',
+          'yarn.nodemanager.log-dirs': '/hadoop/yarn/log,/dev/shm/hadoop/yarn/log,/vagrant/hadoop/yarn/log',
+          'yarn.timeline-service.entity-group-fs-store.app-cache-size': '3',
+          'yarn.timeline-service.leveldb-state-store.path': '/hadoop/yarn/timeline',
+          'yarn.timeline-service.leveldb-timeline-store.path': '/hadoop/yarn/timeline'
+
+        }
+      }
+    }
+
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 6,
+            "total_mem": 2048,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          },
+        }
+      ]
+    }
+
+
+
+
+    '''
+    Test 1 :
+    I/P:
+       - 'changed-configurations' is empty (doesnt have 'yarn.timeline-service.entity-group-fs-store.app-cache-size')
+       - 'host_mem' = 2048
+    O/P :
+       -  Config value recommended for:
+           - yarn.timeline-service.entity-group-fs-store.app-cache-size = 3
+           - apptimelineserver_heapsize = 1024
+    '''
+
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+
+
+    '''
+    Test 2 :
+    I/P:
+       - 'changed-configurations' is empty (doesnt have 'yarn.timeline-service.entity-group-fs-store.app-cache-size')
+       - 'host_mem' = 4096
+    O/P :
+       -  Config value recommended for:
+           - yarn.timeline-service.entity-group-fs-store.app-cache-size = 7
+           - apptimelineserver_heapsize = 2048
+    '''
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 6,
+            "total_mem": 4096,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          },
+        }
+      ]
+    }
+
+    expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '2048'
+    expected['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.app-cache-size']
= '7'
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+
+
+    '''
+    Test 3 :
+    I/P:
+       - 'changed-configurations' is empty (doesnt have 'yarn.timeline-service.entity-group-fs-store.app-cache-size')
+       - 'host_mem' = 8192
+    O/P :
+       -  Config value recommended for:
+           - yarn.timeline-service.entity-group-fs-store.app-cache-size = 10
+           - apptimelineserver_heapsize = 4096
+    '''
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 6,
+            "total_mem": 8192,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          },
+        }
+      ]
+    }
+
+    expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '4096'
+    expected['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.app-cache-size']
= '10'
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+
+
+    '''
+    Test 4 :
+    I/P:
+       - 'changed-configurations' has 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+       - 'host_mem' = 2048
+    O/P :
+       -  Config value recommended for:
+           - apptimelineserver_heapsize = 4096
+    '''
+
+    services["changed-configurations"] = [
+      {
+        u'old_value': u'10',
+        u'type': u'yarn-site',
+        u'name': u'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+      }
+    ]
+
+    services["configurations"] = {
+      "yarn-env": {
+        "properties": {
+          "yarn_user" : "custom_yarn",
+        }
+      },
+      "yarn-site": {
+        "properties": {
+          "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "7"
+        }
+      },
+      "ranger-yarn-plugin-properties": {
+        "properties": {
+          "ranger-yarn-plugin-enabled" : "Yes",
+          "REPOSITORY_CONFIG_USERNAME":"yarn"
+        }
+      }
+    }
+
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 6,
+            "total_mem": 4096,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          },
+        }
+      ]
+    }
+
+
+
+    '''
+    Test 5 :
+    I/P:
+       - 'changed-configurations' has 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+       - 'host_mem' = 4096
+    O/P :
+       -  Config value recommended for:
+           - apptimelineserver_heapsize = 2048
+    '''
+
+    services["changed-configurations"] = [
+      {
+        u'old_value': u'10',
+        u'type': u'yarn-site',
+        u'name': u'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+      }
+    ]
+
+    services["configurations"] = {
+      "yarn-env": {
+        "properties": {
+          "yarn_user" : "custom_yarn",
+        }
+      },
+      "yarn-site": {
+        "properties": {
+          "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "7"
+        }
+      },
+      "ranger-yarn-plugin-properties": {
+        "properties": {
+          "ranger-yarn-plugin-enabled" : "Yes",
+          "REPOSITORY_CONFIG_USERNAME":"yarn"
+        }
+      }
+    }
+
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 6,
+            "total_mem": 4096,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          },
+        }
+      ]
+    }
+
+
+    expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '2048'
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
     self.assertEquals(configurations, expected)
 
+
+
+    '''
+    Test 6 :
+    I/P:
+       - 'changed-configurations' has 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+       - 'host_mem' = 8196
+    O/P :
+       -  Config value recommended for:
+           - Shouldn't have yarn.timeline-service.entity-group-fs-store.app-cache-size
+           - apptimelineserver_heapsize = 4572
+    '''
+
+    services["changed-configurations"] = [
+      {
+        u'old_value': u'10',
+        u'type': u'yarn-site',
+        u'name': u'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+      }
+    ]
+
+    services["configurations"] = {
+      "yarn-env": {
+        "properties": {
+          "yarn_user" : "custom_yarn",
+        }
+      },
+      "yarn-site": {
+        "properties": {
+          "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "3"
+        }
+      },
+      "ranger-yarn-plugin-properties": {
+        "properties": {
+          "ranger-yarn-plugin-enabled" : "Yes",
+          "REPOSITORY_CONFIG_USERNAME":"yarn"
+        }
+      }
+    }
+
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "cpu_count": 6,
+            "total_mem": 16392,
+            "disk_info": [
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"},
+              {"mountpoint": "/"},
+              {"mountpoint": "/dev/shm"},
+              {"mountpoint": "/vagrant"}
+            ],
+            "public_host_name": "c6401.ambari.apache.org",
+            "host_name": "c6401.ambari.apache.org"
+          },
+        }
+      ]
+    }
+
+
+    expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '4572'
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+
   def test_recommendKAFKAConfigurations(self):
     configurations = {
       "kafka-env": {


Mime
View raw message