ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ja...@apache.org
Subject [10/10] ambari git commit: AMBARI-13442: stack advisor layout and validations for HAWQ service master components (adenissov via jaoki)
Date Mon, 26 Oct 2015 20:57:03 GMT
AMBARI-13442: stack advisor layout and validations for HAWQ service master components (adenissov
via jaoki)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/46a20019
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/46a20019
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/46a20019

Branch: refs/heads/trunk
Commit: 46a20019de96f02437720f93f1d442fc33898585
Parents: 7a0aa25
Author: Jun Aoki <jaoki@apache.org>
Authored: Mon Oct 26 13:56:44 2015 -0700
Committer: Jun Aoki <jaoki@apache.org>
Committed: Mon Oct 26 13:56:44 2015 -0700

----------------------------------------------------------------------
 .../stacks/HDP/2.3/services/stack_advisor.py    |   70 +
 .../python/stacks/2.3/common/hosts-1-host.json  |   93 +
 .../python/stacks/2.3/common/hosts-3-hosts.json |  269 ++
 .../stacks/2.3/common/services-hawq-1-host.json | 2575 ++++++++++++++++++
 .../2.3/common/services-hawq-3-hosts.json       | 2575 ++++++++++++++++++
 .../services-master_ambari_colo-3-hosts.json    | 2575 ++++++++++++++++++
 .../services-master_standby_colo-3-hosts.json   | 2575 ++++++++++++++++++
 .../2.3/common/services-nohawq-3-hosts.json     | 2214 +++++++++++++++
 .../common/services-normal-hawq-3-hosts.json    | 2575 ++++++++++++++++++
 .../common/services-normal-nohawq-3-hosts.json  | 2214 +++++++++++++++
 .../services-standby_ambari_colo-3-hosts.json   | 2575 ++++++++++++++++++
 .../stacks/2.3/common/test_stack_advisor.py     |  175 ++
 12 files changed, 20485 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/46a20019/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 464f9cc..2a26405 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -22,6 +22,76 @@ import socket
 
 class HDP23StackAdvisor(HDP22StackAdvisor):
 
+  def createComponentLayoutRecommendations(self, services, hosts):
+    parentComponentLayoutRecommendations = super(HDP23StackAdvisor, self).createComponentLayoutRecommendations(services,
hosts)
+
+    # remove HAWQSTANDBY on a single node
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    if len(hostsList) == 1:
+      servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+      if "HAWQ" in servicesList:
+        components = parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"]
+        components = [ component for component in components if component["name"] != 'HAWQSTANDBY'
]
+        parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"]
= components
+
+    return parentComponentLayoutRecommendations
+
+  def getComponentLayoutValidations(self, services, hosts):
+    parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services,
hosts)
+
+    if not "HAWQ" in [service["StackServices"]["service_name"] for service in services["services"]]:
+      return parentItems
+
+    childItems = []
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    hostsCount = len(hostsList)
+
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
+    hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
+
+    # single node case is not analyzed because HAWQ Standby Master will not be present in
single node topology due to logic in createComponentLayoutRecommendations()
+    if len(hawqMasterHosts) > 0 and len(hawqStandbyHosts) > 0:
+      commonHosts = [host for host in hawqMasterHosts[0] if host in hawqStandbyHosts[0]]
+      for host in commonHosts:
+        message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same
host."
+        childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'HAWQSTANDBY', "host": host } )
+
+    if len(hawqMasterHosts) > 0 and hostsCount > 1:
+      ambariServerHosts = [host for host in hawqMasterHosts[0] if self.isLocalHost(host)]
+      for host in ambariServerHosts:
+        message = "HAWQ Master and Ambari Server should not be deployed on the same host.
" \
+                  "If you leave them collocated, make sure to set HAWQ Master Port property
" \
+                  "to a value different from the port number used by Ambari Server database."
+        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQMASTER', "host": host } )
+
+    if len(hawqStandbyHosts) > 0 and hostsCount > 1:
+      ambariServerHosts = [host for host in hawqStandbyHosts[0] if self.isLocalHost(host)]
+      for host in ambariServerHosts:
+        message = "HAWQ Standby Master and Ambari Server should not be deployed on the same
host. " \
+                  "If you leave them collocated, make sure to set HAWQ Master Port property
" \
+                  "to a value different from the port number used by Ambari Server database."
+        childItems.append( { "type": 'host-component', "level": 'WARN', "message": message,
"component-name": 'HAWQSTANDBY', "host": host } )
+    
+    parentItems.extend(childItems)
+    return parentItems
+
+  def getNotPreferableOnServerComponents(self):
+    parentComponents = super(HDP23StackAdvisor, self).getNotPreferableOnServerComponents()
+    parentComponents.extend(['HAWQMASTER', 'HAWQSTANDBY'])
+    return parentComponents
+
+  def getComponentLayoutSchemes(self):
+    parentSchemes = super(HDP23StackAdvisor, self).getComponentLayoutSchemes()
+    # key is max number of cluster hosts + 1, value is index in host list where to put the
component
+    childSchemes = {
+        'HAWQMASTER' : {6: 2, 31: 1, "else": 5},
+        'HAWQSTANDBY': {6: 1, 31: 2, "else": 3}
+    }
+    parentSchemes.update(childSchemes)
+    return parentSchemes
+
   def getServiceConfigurationRecommenderDict(self):
     parentRecommendConfDict = super(HDP23StackAdvisor, self).getServiceConfigurationRecommenderDict()
     childRecommendConfDict = {

http://git-wip-us.apache.org/repos/asf/ambari/blob/46a20019/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json b/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json
new file mode 100644
index 0000000..5d6a5b6
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json
@@ -0,0 +1,93 @@
+{
+  "href" : "/api/v1/hosts?fields=Hosts/*&Hosts/host_name.in(c6401.ambari.apache.org)",
+  "items" : [
+    {
+      "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+      "Hosts" : {
+        "cpu_count" : 1,
+        "desired_configs" : null,
+        "disk_info" : [
+          {
+            "available" : "481199976",
+            "device" : "/dev/mapper/VolGroup-lv_root",
+            "used" : "5713880",
+            "percent" : "2%",
+            "size" : "512971376",
+            "type" : "ext4",
+            "mountpoint" : "/"
+          },
+          {
+            "available" : "1478456",
+            "device" : "tmpfs",
+            "used" : "0",
+            "percent" : "0%",
+            "size" : "1478456",
+            "type" : "tmpfs",
+            "mountpoint" : "/dev/shm"
+          },
+          {
+            "available" : "438284",
+            "device" : "/dev/sda1",
+            "used" : "31960",
+            "percent" : "7%",
+            "size" : "495844",
+            "type" : "ext4",
+            "mountpoint" : "/boot"
+          },
+          {
+            "available" : "191037148",
+            "device" : "vagrant",
+            "used" : "296051072",
+            "percent" : "61%",
+            "size" : "487088220",
+            "type" : "vboxsf",
+            "mountpoint" : "/vagrant"
+          }
+        ],
+        "host_health_report" : "",
+        "host_name" : "c6401.ambari.apache.org",
+        "host_state" : "HEALTHY",
+        "host_status" : "HEALTHY",
+        "ip" : "192.168.64.101",
+        "last_agent_env" : {
+          "stackFoldersAndFiles" : [ ],
+          "alternatives" : [ ],
+          "existingUsers" : [ ],
+          "existingRepos" : [ ],
+          "installedPackages" : [ ],
+          "hostHealth" : {
+            "activeJavaProcs" : [ ],
+            "agentTimeStampAtReporting" : 1445288922364,
+            "serverTimeStampAtReporting" : 1445288922425,
+            "liveServices" : [
+              {
+                "desc" : "",
+                "name" : "ntpd",
+                "status" : "Healthy"
+              }
+            ]
+          },
+          "umask" : 18,
+          "transparentHugePage" : "always",
+          "firewallRunning" : false,
+          "firewallName" : "iptables",
+          "reverseLookup" : true
+        },
+        "last_heartbeat_time" : 1445288922425,
+        "last_registration_time" : 1445288888619,
+        "os_arch" : "x86_64",
+        "os_family" : "redhat6",
+        "os_type" : "centos6",
+        "ph_cpu_count" : 1,
+        "public_host_name" : "c6401.ambari.apache.org",
+        "rack_info" : "/default-rack",
+        "recovery_report" : {
+          "summary" : "DISABLED",
+          "component_reports" : [ ]
+        },
+        "recovery_summary" : "DISABLED",
+        "total_mem" : 2956916
+      }
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/46a20019/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json b/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json
new file mode 100644
index 0000000..3c0511e
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json
@@ -0,0 +1,269 @@
+{
+  "href" : "/api/v1/hosts?fields=Hosts/*&Hosts/host_name.in(c6401.ambari.apache.org,c6402.ambari.apache.org,c6403.ambari.apache.org)",
+  "items" : [
+    {
+      "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+      "Hosts" : {
+        "cpu_count" : 1,
+        "desired_configs" : null,
+        "disk_info" : [
+          {
+            "available" : "481199416",
+            "device" : "/dev/mapper/VolGroup-lv_root",
+            "used" : "5714440",
+            "percent" : "2%",
+            "size" : "512971376",
+            "type" : "ext4",
+            "mountpoint" : "/"
+          },
+          {
+            "available" : "1478456",
+            "device" : "tmpfs",
+            "used" : "0",
+            "percent" : "0%",
+            "size" : "1478456",
+            "type" : "tmpfs",
+            "mountpoint" : "/dev/shm"
+          },
+          {
+            "available" : "438284",
+            "device" : "/dev/sda1",
+            "used" : "31960",
+            "percent" : "7%",
+            "size" : "495844",
+            "type" : "ext4",
+            "mountpoint" : "/boot"
+          },
+          {
+            "available" : "191426876",
+            "device" : "vagrant",
+            "used" : "295661344",
+            "percent" : "61%",
+            "size" : "487088220",
+            "type" : "vboxsf",
+            "mountpoint" : "/vagrant"
+          }
+        ],
+        "host_health_report" : "",
+        "host_name" : "c6401.ambari.apache.org",
+        "host_state" : "HEALTHY",
+        "host_status" : "HEALTHY",
+        "ip" : "192.168.64.101",
+        "last_agent_env" : {
+          "stackFoldersAndFiles" : [ ],
+          "alternatives" : [ ],
+          "existingUsers" : [ ],
+          "existingRepos" : [ ],
+          "installedPackages" : [ ],
+          "hostHealth" : {
+            "activeJavaProcs" : [ ],
+            "agentTimeStampAtReporting" : 1445290054919,
+            "serverTimeStampAtReporting" : 1445290054980,
+            "liveServices" : [
+              {
+                "desc" : "",
+                "name" : "ntpd",
+                "status" : "Healthy"
+              }
+            ]
+          },
+          "umask" : 18,
+          "transparentHugePage" : "always",
+          "firewallRunning" : false,
+          "firewallName" : "iptables",
+          "reverseLookup" : true
+        },
+        "last_heartbeat_time" : 1445290077648,
+        "last_registration_time" : 1445289898391,
+        "os_arch" : "x86_64",
+        "os_family" : "redhat6",
+        "os_type" : "centos6",
+        "ph_cpu_count" : 1,
+        "public_host_name" : "c6401.ambari.apache.org",
+        "rack_info" : "/default-rack",
+        "recovery_report" : {
+          "summary" : "DISABLED",
+          "component_reports" : [ ]
+        },
+        "recovery_summary" : "DISABLED",
+        "total_mem" : 2956916
+      }
+    },
+    {
+      "href" : "/api/v1/hosts/c6402.ambari.apache.org",
+      "Hosts" : {
+        "cpu_count" : 1,
+        "desired_configs" : null,
+        "disk_info" : [
+          {
+            "available" : "482615388",
+            "device" : "/dev/mapper/VolGroup-lv_root",
+            "used" : "4298468",
+            "percent" : "1%",
+            "size" : "512971376",
+            "type" : "ext4",
+            "mountpoint" : "/"
+          },
+          {
+            "available" : "1478456",
+            "device" : "tmpfs",
+            "used" : "0",
+            "percent" : "0%",
+            "size" : "1478456",
+            "type" : "tmpfs",
+            "mountpoint" : "/dev/shm"
+          },
+          {
+            "available" : "438284",
+            "device" : "/dev/sda1",
+            "used" : "31960",
+            "percent" : "7%",
+            "size" : "495844",
+            "type" : "ext4",
+            "mountpoint" : "/boot"
+          },
+          {
+            "available" : "191430912",
+            "device" : "vagrant",
+            "used" : "295657308",
+            "percent" : "61%",
+            "size" : "487088220",
+            "type" : "vboxsf",
+            "mountpoint" : "/vagrant"
+          }
+        ],
+        "host_health_report" : "",
+        "host_name" : "c6402.ambari.apache.org",
+        "host_state" : "HEALTHY",
+        "host_status" : "HEALTHY",
+        "ip" : "192.168.64.102",
+        "last_agent_env" : {
+          "stackFoldersAndFiles" : [ ],
+          "alternatives" : [ ],
+          "existingUsers" : [ ],
+          "existingRepos" : [ ],
+          "installedPackages" : [ ],
+          "hostHealth" : {
+            "activeJavaProcs" : [ ],
+            "agentTimeStampAtReporting" : 1445290076132,
+            "serverTimeStampAtReporting" : 1445290076343,
+            "liveServices" : [
+              {
+                "desc" : "",
+                "name" : "ntpd",
+                "status" : "Healthy"
+              }
+            ]
+          },
+          "umask" : 18,
+          "transparentHugePage" : "always",
+          "firewallRunning" : false,
+          "firewallName" : "iptables",
+          "reverseLookup" : true
+        },
+        "last_heartbeat_time" : 1445290086404,
+        "last_registration_time" : 1445290032489,
+        "os_arch" : "x86_64",
+        "os_family" : "redhat6",
+        "os_type" : "centos6",
+        "ph_cpu_count" : 1,
+        "public_host_name" : "c6402.ambari.apache.org",
+        "rack_info" : "/default-rack",
+        "recovery_report" : {
+          "summary" : "DISABLED",
+          "component_reports" : [ ]
+        },
+        "recovery_summary" : "DISABLED",
+        "total_mem" : 2956916
+      }
+    },
+    {
+      "href" : "/api/v1/hosts/c6403.ambari.apache.org",
+      "Hosts" : {
+        "cpu_count" : 1,
+        "desired_configs" : null,
+        "disk_info" : [
+          {
+            "available" : "482615388",
+            "device" : "/dev/mapper/VolGroup-lv_root",
+            "used" : "4298468",
+            "percent" : "1%",
+            "size" : "512971376",
+            "type" : "ext4",
+            "mountpoint" : "/"
+          },
+          {
+            "available" : "1478456",
+            "device" : "tmpfs",
+            "used" : "0",
+            "percent" : "0%",
+            "size" : "1478456",
+            "type" : "tmpfs",
+            "mountpoint" : "/dev/shm"
+          },
+          {
+            "available" : "438284",
+            "device" : "/dev/sda1",
+            "used" : "31960",
+            "percent" : "7%",
+            "size" : "495844",
+            "type" : "ext4",
+            "mountpoint" : "/boot"
+          },
+          {
+            "available" : "191430912",
+            "device" : "vagrant",
+            "used" : "295657308",
+            "percent" : "61%",
+            "size" : "487088220",
+            "type" : "vboxsf",
+            "mountpoint" : "/vagrant"
+          }
+        ],
+        "host_health_report" : "",
+        "host_name" : "c6403.ambari.apache.org",
+        "host_state" : "HEALTHY",
+        "host_status" : "HEALTHY",
+        "ip" : "192.168.64.103",
+        "last_agent_env" : {
+          "stackFoldersAndFiles" : [ ],
+          "alternatives" : [ ],
+          "existingUsers" : [ ],
+          "existingRepos" : [ ],
+          "installedPackages" : [ ],
+          "hostHealth" : {
+            "activeJavaProcs" : [ ],
+            "agentTimeStampAtReporting" : 1445290073223,
+            "serverTimeStampAtReporting" : 1445290073435,
+            "liveServices" : [
+              {
+                "desc" : "",
+                "name" : "ntpd",
+                "status" : "Healthy"
+              }
+            ]
+          },
+          "umask" : 18,
+          "transparentHugePage" : "always",
+          "firewallRunning" : false,
+          "firewallName" : "iptables",
+          "reverseLookup" : true
+        },
+        "last_heartbeat_time" : 1445290083492,
+        "last_registration_time" : 1445290030522,
+        "os_arch" : "x86_64",
+        "os_family" : "redhat6",
+        "os_type" : "centos6",
+        "ph_cpu_count" : 1,
+        "public_host_name" : "c6403.ambari.apache.org",
+        "rack_info" : "/default-rack",
+        "recovery_report" : {
+          "summary" : "DISABLED",
+          "component_reports" : [ ]
+        },
+        "recovery_summary" : "DISABLED",
+        "total_mem" : 2956916
+      }
+    }
+  ]
+}
\ No newline at end of file


Mime
View raw message