ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject [1/2] ambari git commit: AMBARI-16919. Handle the scenario when 'capacity-scheduler' is passed-in as dictionary to Stack Advisor and services['configurations'][capacity-scheduler][properties][capacity-scheduler] is null.
Date Fri, 27 May 2016 02:58:40 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 4acca9415 -> 59364bc87


AMBARI-16919. Handle the scenario when 'capacity-scheduler' is passed-in as dictionary to
Stack Advisor and services['configurations'][capacity-scheduler][properties][capacity-scheduler]
is null.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/59364bc8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/59364bc8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/59364bc8

Branch: refs/heads/trunk
Commit: 59364bc87d0ad3c6fa16918329d40230b34646c8
Parents: 08d70f5
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Thu May 26 18:47:21 2016 -0700
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Thu May 26 19:58:44 2016 -0700

----------------------------------------------------------------------
 .../BlueprintConfigurationProcessor.java        |  4 ++-
 .../stacks/HDP/2.5/services/stack_advisor.py    | 28 +++++++++++++++-----
 .../stacks/2.5/common/test_stack_advisor.py     |  7 +++--
 3 files changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/59364bc8/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 4e404bc..a0af813 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -448,6 +448,8 @@ public class BlueprintConfigurationProcessor {
   private void doRecommendConfigurations(Configuration configuration, Set<String> configTypesUpdated)
{
     ConfigRecommendationStrategy configRecommendationStrategy = clusterTopology.getConfigRecommendationStrategy();
     Map<String, AdvisedConfiguration> advisedConfigurations = clusterTopology.getAdvisedConfigurations();
+    LOG.info("Config recommendation strategy being used is {})", configRecommendationStrategy);
+
     if (ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY.equals(configRecommendationStrategy))
{
       LOG.info("Filter out recommended configurations. Keep only the stack defaults.");
       doFilterStackDefaults(advisedConfigurations);
@@ -465,7 +467,7 @@ public class BlueprintConfigurationProcessor {
         }
       }
     } else {
-      LOG.info("No any recommended configuration applied. (strategy: {})", ConfigRecommendationStrategy.NEVER_APPLY);
+      LOG.info("No recommended configurations are applied. (strategy: {})", ConfigRecommendationStrategy.NEVER_APPLY);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/59364bc8/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 44407ca..8c5351f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -1025,17 +1025,33 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     received_as_key_value_pair = True
     if "capacity-scheduler" in services['configurations']:
       if "capacity-scheduler" in services['configurations']["capacity-scheduler"]["properties"]:
-        properties = str(services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"]).split('\n')
-        if properties:
-          if properties[0] != 'null':
+        cap_sched_props_as_str = services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"]
+        if cap_sched_props_as_str:
+          cap_sched_props_as_str = str(cap_sched_props_as_str).split('\n')
+          if len(cap_sched_props_as_str) > 0 and cap_sched_props_as_str[0] != 'null':
             # Received confgs as one "\n" separated string
-            for property in properties:
+            for property in cap_sched_props_as_str:
               key, sep, value = property.partition("=")
               capacity_scheduler_properties[key] = value
+            Logger.info("'capacity-scheduler' configs is passed-in as a single '\\n' separated
string. "
+                        "count(services['configurations']['capacity-scheduler']['properties']['capacity-scheduler'])
= "
+                        "{0}".format(len(capacity_scheduler_properties)))
             received_as_key_value_pair = False
           else:
-            # Received configs as a dictionary (Generally on 1st invocation).
-            capacity_scheduler_properties = services['configurations']["capacity-scheduler"]["properties"]
+            Logger.info("Passed-in services['configurations']['capacity-scheduler']['properties']['capacity-scheduler']
is 'null'.")
+        else:
+          Logger.info("'capacity-schdeuler' configs not passed-in as single '\\n' string
in "
+                      "services['configurations']['capacity-scheduler']['properties']['capacity-scheduler'].")
+      if not capacity_scheduler_properties:
+        # Received configs as a dictionary (Generally on 1st invocation).
+        capacity_scheduler_properties = services['configurations']["capacity-scheduler"]["properties"]
+        Logger.info("'capacity-scheduler' configs is passed-in as a dictionary. "
+                    "count(services['configurations']['capacity-scheduler']['properties'])
= {0}".format(len(capacity_scheduler_properties)))
+    else:
+      Logger.error("Couldn't retrieve 'capacity-scheduler' from services.")
+
+    Logger.info("Retrieved 'capacity-scheduler' received as dictionary : '{0}'. configs :
{1}"\
+                .format(received_as_key_value_pair, capacity_scheduler_properties.items()))
     return capacity_scheduler_properties, received_as_key_value_pair
 
   def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts):

http://git-wip-us.apache.org/repos/asf/ambari/blob/59364bc8/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 01064a6..0066e1d 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -636,6 +636,7 @@ class TestHDP25StackAdvisor(TestCase):
 
 
   # Test 2 : (1). Only default queue exists in capacity-scheduler and capacity-scheduler
is passed-in as a dictionary,
+  # and services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"]
is set to value "null"
   # (2). enable_hive_interactive' is 'On' and 'llap_queue_capacity is 0.
   def test_recommendYARNConfigurations_create_llap_queue_1(self):
 
@@ -913,6 +914,7 @@ class TestHDP25StackAdvisor(TestCase):
 
 
   # Test 4: (1). Only default queue exists in capacity-scheduler and capacity-scheduler is
passed-in as a dictionary
+  # and services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"]
is null
   # (2). enable_hive_interactive' is 'On' and 'llap_queue_capacity is 40.
   def test_recommendYARNConfigurations_create_llap_queue_2(self):
     services = {
@@ -977,7 +979,7 @@ class TestHDP25StackAdvisor(TestCase):
       "configurations": {
         "capacity-scheduler" : {
           "properties" : {
-            "capacity-scheduler" : "null",
+            "capacity-scheduler" : None,
             "yarn.scheduler.capacity.root.accessible-node-labels" : "*",
             "yarn.scheduler.capacity.maximum-am-resource-percent" : "1",
             "yarn.scheduler.capacity.root.acl_administer_queue" : "*",
@@ -4606,7 +4608,8 @@ class TestHDP25StackAdvisor(TestCase):
 
 
   # Test 28: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler,
and
-  #          'capacity-scheduler' configs are passed-in as dictionary  and
+  #          'capacity-scheduler' configs are passed-in as dictionary and
+  #          services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"]
is set to value "null"  and
   #          (2). enable_hive_interactive' is 'on' and (3). configuration change detected
for 'hive.server2.tez.sessions.per.default.queue'
   #         Expected : Configurations values recommended for llap related configs.
   def test_recommendYARNConfigurations_five_node_manager_llap_configs_updated_3(self):


Mime
View raw message