ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srima...@apache.org
Subject git commit: AMBARI-7464. Make stack_advisor code more human readable (srimanth)
Date Wed, 24 Sep 2014 02:11:05 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 6e73363bb -> 1ea56f9f8


AMBARI-7464. Make stack_advisor code more human readable (srimanth)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1ea56f9f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1ea56f9f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1ea56f9f

Branch: refs/heads/trunk
Commit: 1ea56f9f817cc1f588093004ab033695a2b25c2b
Parents: 6e73363
Author: Srimanth Gunturi <sgunturi@hortonworks.com>
Authored: Tue Sep 23 19:05:17 2014 -0700
Committer: Srimanth Gunturi <sgunturi@hortonworks.com>
Committed: Tue Sep 23 19:05:24 2014 -0700

----------------------------------------------------------------------
 .../src/main/resources/stacks/stack_advisor.py  | 270 ++++++++++++++++++-
 1 file changed, 266 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/1ea56f9f/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index dca2834..f2b5446 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -20,23 +20,285 @@ limitations under the License.
 import socket
 
 class StackAdvisor(object):
+  """
+  Abstract class implemented by all stack advisors. Stack advisors advise on stack specific
questions. 
+
+  Currently stack advisors provide following abilities:
+  - Recommend where services should be installed in cluster
+  - Recommend configurations based on host hardware
+  - Validate user selection of where services are installed on cluster
+  - Validate user configuration values 
+
+  Each of the above methods is passed in parameters about services and hosts involved as
described below.
+
+    @type services: dictionary
+    @param services: Dictionary containing all information about services selected by the
user. 
+      Example: {
+      "services": [
+        {
+          "StackServices": {
+            "service_name" : "HDFS",
+            "service_version" : "2.6.0.2.2",
+          },
+          "components" : [ 
+            {
+              "StackServiceComponents" : {
+                "cardinality" : "1+",
+                "component_category" : "SLAVE",
+                "component_name" : "DATANODE",
+                "display_name" : "DataNode",
+                "service_name" : "HDFS",
+                "hostnames" : []
+              },
+              "dependencies" : []
+            }, {
+              "StackServiceComponents" : {
+                "cardinality" : "1-2",
+                "component_category" : "MASTER",
+                "component_name" : "NAMENODE",
+                "display_name" : "NameNode",
+                "service_name" : "HDFS",
+                "hostnames" : []
+              },
+              "dependencies" : []
+            },
+            ...
+          ]
+        },
+        ...
+      ]
+    }
+  @type hosts: dictionary
+  @param hosts: Dictionary containing all information about hosts in this cluster
+    Example: {
+      "items": [
+        {
+          Hosts: {
+            "host_name": "c6401.ambari.apache.org",
+            "public_host_name" : "c6401.ambari.apache.org",
+            "ip": "192.168.1.101",
+            "cpu_count" : 1,
+            "disk_info" : [
+              {
+              "available" : "4564632",
+              "used" : "5230344",
+              "percent" : "54%",
+              "size" : "10319160",
+              "type" : "ext4",
+              "mountpoint" : "/"
+              },
+              {
+              "available" : "1832436",
+              "used" : "0",
+              "percent" : "0%",
+              "size" : "1832436",
+              "type" : "tmpfs",
+              "mountpoint" : "/dev/shm"
+              }
+            ],
+            "host_state" : "HEALTHY",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "total_mem" : 3664872
+          }
+        },
+        ...
+      ]
+    }
+
+    Each of the methods can either return recommendations or validations.
+
+    Recommendations are made in a Ambari Blueprints friendly format. 
+    Validations are an array of validation objects.
+  """
 
   def recommendComponentLayout(self, services, hosts):
-    """Returns Services object with hostnames array populated for components"""
+    """
+    Returns recommendation of which hosts various service components should be installed
on.
+
+    This function takes as input all details about services being installed, and hosts
+    they are being installed into, to generate hostname assignments to various components
+    of each service.
+
+    @type services: dictionary
+    @param services: Dictionary containing all information about services selected by the
user.
+    @type hosts: dictionary
+    @param hosts: Dictionary containing all information about hosts in this cluster
+    @rtype: dictionary
+    @return: Layout recommendation of service components on cluster hosts in Ambari Blueprints
friendly format. 
+        Example: {
+          "resources" : [
+            {
+              "hosts" : [
+                "c6402.ambari.apache.org",
+                "c6401.ambari.apache.org"
+              ],
+              "services" : [
+                "HDFS"
+              ],
+              "recommendations" : {
+                "blueprint" : {
+                  "host_groups" : [
+                    {
+                      "name" : "host-group-2",
+                      "components" : [
+                        { "name" : "JOURNALNODE" },
+                        { "name" : "ZKFC" },
+                        { "name" : "DATANODE" },
+                        { "name" : "SECONDARY_NAMENODE" }
+                      ]
+                    },
+                    {
+                      "name" : "host-group-1",
+                      "components" : [
+                        { "name" : "HDFS_CLIENT" },
+                        { "name" : "NAMENODE" },
+                        { "name" : "JOURNALNODE" },
+                        { "name" : "ZKFC" },
+                        { "name" : "DATANODE" }
+                      ]
+                    }
+                  ]
+                },
+                "blueprint_cluster_binding" : {
+                  "host_groups" : [
+                    {
+                      "name" : "host-group-1",
+                      "hosts" : [ { "fqdn" : "c6401.ambari.apache.org" } ]
+                    },
+                    {
+                      "name" : "host-group-2",
+                      "hosts" : [ { "fqdn" : "c6402.ambari.apache.org" } ]
+                    }
+                  ]
+                }
+              }
+            }
+          ]
+        }
+    """
     pass
 
   def validateComponentLayout(self, services, hosts):
-    """Returns array of Validation objects about issues with hostnames components assigned
to"""
+    """
+    Returns array of Validation issues with service component layout on hosts
+
+    This function takes as input all details about services being installed along with
+    hosts the components are being installed on (hostnames property is populated for 
+    each component).  
+
+    @type services: dictionary
+    @param services: Dictionary containing information about services and host layout selected
by the user.
+    @type hosts: dictionary
+    @param hosts: Dictionary containing all information about hosts in this cluster
+    @rtype: dictionary
+    @return: Dictionary containing array of validation items
+        Example: {
+          "items": [
+            {
+              "type" : "host-group",
+              "level" : "ERROR",
+              "message" : "NameNode and Secondary NameNode should not be hosted on the same
machine",
+              "component-name" : "NAMENODE",
+              "host" : "c6401.ambari.apache.org" 
+            },
+            ...
+          ]
+        }  
+    """
     pass
 
   def recommendConfigurations(self, services, hosts):
-    """Returns Services object with configurations object populated"""
+    """
+    Returns recommendation of service configurations based on host-specific layout of components.
+
+    This function takes as input all details about services being installed, and hosts
+    they are being installed into, to recommend host-specific configurations.
+
+    @type services: dictionary
+    @param services: Dictionary containing all information about services and component layout
selected by the user.
+    @type hosts: dictionary
+    @param hosts: Dictionary containing all information about hosts in this cluster
+    @rtype: dictionary
+    @return: Layout recommendation of service components on cluster hosts in Ambari Blueprints
friendly format. 
+        Example: {
+         "services": [
+          "HIVE", 
+          "TEZ", 
+          "YARN"
+         ], 
+         "recommendations": {
+          "blueprint": {
+           "host_groups": [], 
+           "configurations": {
+            "yarn-site": {
+             "properties": {
+              "yarn.scheduler.minimum-allocation-mb": "682", 
+              "yarn.scheduler.maximum-allocation-mb": "2048", 
+              "yarn.nodemanager.resource.memory-mb": "2048"
+             }
+            }, 
+            "tez-site": {
+             "properties": {
+              "tez.am.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA
-XX:+UseParallelGC", 
+              "tez.am.resource.memory.mb": "682"
+             }
+            }, 
+            "hive-site": {
+             "properties": {
+              "hive.tez.container.size": "682", 
+              "hive.tez.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8
-XX:+UseNUMA -XX:+UseParallelGC", 
+              "hive.auto.convert.join.noconditionaltask.size": "238026752"
+             }
+            }
+           }
+          }, 
+          "blueprint_cluster_binding": {
+           "host_groups": []
+          }
+         }, 
+         "hosts": [
+          "c6401.ambari.apache.org", 
+          "c6402.ambari.apache.org", 
+          "c6403.ambari.apache.org" 
+         ] 
+        }
+    """
     pass
 
   def validateConfigurations(self, services, hosts):
-    """Returns array of Validation objects about issues with configuration values provided
in services"""
+    """"
+    Returns array of Validation issues with configurations provided by user
+
+    This function takes as input all details about services being installed along with
+    configuration values entered by the user. These configurations can be validated against
+    service requirements, or host hardware to generate validation issues. 
+
+    @type services: dictionary
+    @param services: Dictionary containing information about services and user configurations.
+    @type hosts: dictionary
+    @param hosts: Dictionary containing all information about hosts in this cluster
+    @rtype: dictionary
+    @return: Dictionary containing array of validation items
+        Example: {
+         "items": [
+          {
+           "config-type": "yarn-site", 
+           "message": "Value is less than the recommended default of 682", 
+           "type": "configuration", 
+           "config-name": "yarn.scheduler.minimum-allocation-mb", 
+           "level": "WARN"
+          }
+         ]
+       }
+    """
     pass
 
+
+
+
+
+
 class DefaultStackAdvisor(StackAdvisor):
 
   def recommendComponentLayout(self, services, hosts):


Mime
View raw message