ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ja...@apache.org
Subject [01/11] ambari git commit: AMBARI-13665: Enable HAWQ/PXF services in Ambari 2.1.3
Date Thu, 19 Nov 2015 21:32:37 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 12f9d0764 -> 8857e63b8


http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 02753d4..99cfa6c 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -16,6 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+import json
 import os
 import socket
 from unittest import TestCase
@@ -51,6 +52,12 @@ class TestHDP23StackAdvisor(TestCase):
     self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid
     self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic
 
+  def load_json(self, filename):
+    file = os.path.join(self.testDirectory, filename)
+    with open(file, 'rb') as f:
+      data = json.load(f)
+    return data    
+
   @patch('__builtin__.open')
   @patch('os.path.exists')
   def get_system_min_uid_magic(self, exists_mock, open_mock):
@@ -71,6 +78,174 @@ class TestHDP23StackAdvisor(TestCase):
     open_mock.return_value = MagicFile()
     return self.get_system_min_uid_real()
 
+
+  def test_createComponentLayoutRecommendations_hawq_1_Host(self):
+    """ Test that HAWQSTANDBY is not recommended on a single node cluster """
+    
+    services = self.load_json("services-hawq-1-host.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    componentNames = [component["StackServiceComponents"]["component_name"] for component
in componentsList]
+    self.assertTrue('HAWQSTANDBY' in componentNames)
+    
+    hosts = self.load_json("hosts-1-host.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 1)
+
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+
+    recommendedComponentsListList = [hostgroup["components"] for hostgroup in recommendations["blueprint"]["host_groups"]]
+    recommendedComponents = [item["name"] for sublist in recommendedComponentsListList for
item in sublist]
+    self.assertTrue('HAWQMASTER' in recommendedComponents) 
+    self.assertFalse('HAWQSTANDBY' in recommendedComponents) 
+    self.assertTrue('HAWQSEGMENT' in recommendedComponents) 
+
+
+  def test_createComponentLayoutRecommendations_hawq_3_Hosts(self):
+    """ Test that HAWQSTANDBY is recommended on a 3-node cluster """
+
+    services = self.load_json("services-hawq-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    componentNames = [component["StackServiceComponents"]["component_name"] for component
in componentsList]
+    self.assertTrue('HAWQSTANDBY' in componentNames)
+    
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    
+    recommendedComponentsListList = [hostgroup["components"] for hostgroup in recommendations["blueprint"]["host_groups"]]
+    recommendedComponents = [item["name"] for sublist in recommendedComponentsListList for
item in sublist]
+    self.assertTrue('HAWQMASTER' in recommendedComponents) 
+    self.assertTrue('HAWQSTANDBY' in recommendedComponents) 
+    self.assertTrue('HAWQSEGMENT' in recommendedComponents)
+    
+    # make sure master components are not collocated
+    for sublist in recommendedComponentsListList:
+      hostComponents = [item["name"] for item in sublist]
+      self.assertFalse(set(['HAWQMASTER', 'HAWQSTANDBY']).issubset(hostComponents))
+
+
+  def test_createComponentLayoutRecommendations_no_hawq_3_Hosts(self):
+    """ Test no failures when there are no HAWQ components """
+
+    services = self.load_json("services-nohawq-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    componentNames = [component["StackServiceComponents"]["component_name"] for component
in componentsList]
+    self.assertFalse('HAWQMASTER' in componentNames) 
+    self.assertFalse('HAWQSTANDBY' in componentNames) 
+    self.assertFalse('HAWQSEGMENT' in componentNames)
+    
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+    
+    recommendedComponentsListList = [hostgroup["components"] for hostgroup in recommendations["blueprint"]["host_groups"]]
+    recommendedComponents = [item["name"] for sublist in recommendedComponentsListList for
item in sublist]
+    self.assertFalse('HAWQMASTER' in recommendedComponents) 
+    self.assertFalse('HAWQSTANDBY' in recommendedComponents) 
+    self.assertFalse('HAWQSEGMENT' in recommendedComponents)
+      
+
+  def fqdn_mock_result(value=None):
+      return 'c6401.ambari.apache.org' if value is None else value
+
+      
+  @patch('socket.getfqdn', side_effect=fqdn_mock_result)  
+  def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock):
+    """ Test layout validations for HAWQ components on a 3-node cluster """
+
+    # case-1: normal placement, no warnings
+    services = self.load_json("services-normal-hawq-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
+    hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
+    self.assertEquals(len(hawqMasterHosts[0]), 1)
+    self.assertEquals(len(hawqStandbyHosts[0]), 1)
+    self.assertNotEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0])
+    
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 0)
+    
+    # case-2: HAWQ masters are collocated
+    services = self.load_json("services-master_standby_colo-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
+    hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
+    self.assertEquals(len(hawqMasterHosts[0]), 1)
+    self.assertEquals(len(hawqStandbyHosts[0]), 1)
+    self.assertEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0])
+    
+    validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 1)
+    expected={'component-name': 'HAWQSTANDBY', 'message': 'HAWQ Standby Master and HAWQ Master
should not be deployed on the same host.', 'type': 'host-component', 'host': 'c6403.ambari.apache.org',
'level': 'ERROR'}
+    self.assertEquals(validations[0], expected)
+    
+    # case-3: HAWQ Master and Ambari Server are collocated
+    services = self.load_json("services-master_ambari_colo-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
+    hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
+    self.assertEquals(len(hawqMasterHosts[0]), 1)
+    self.assertEquals(len(hawqStandbyHosts[0]), 1)
+    self.assertNotEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0])
+    self.assertEquals(hawqMasterHosts[0][0], "c6401.ambari.apache.org")
+    
+    validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 1)
+    expected={'component-name': 'HAWQMASTER', 'message': 'HAWQ Master and Ambari Server should
not be deployed on the same host. If you leave them collocated, make sure to set HAWQ Master
Port property to a value different from the port number used by Ambari Server database.',
'type': 'host-component', 'host': 'c6401.ambari.apache.org', 'level': 'WARN'}
+    self.assertEquals(validations[0], expected)
+
+    # case-4: HAWQ Standby and Ambari Server are collocated
+    services = self.load_json("services-standby_ambari_colo-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
+    hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
+    self.assertEquals(len(hawqMasterHosts[0]), 1)
+    self.assertEquals(len(hawqStandbyHosts[0]), 1)
+    self.assertNotEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0])
+    self.assertEquals(hawqStandbyHosts[0][0], "c6401.ambari.apache.org")
+    
+    validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 1)
+    expected={'component-name': 'HAWQSTANDBY', 'message': 'HAWQ Standby Master and Ambari
Server should not be deployed on the same host. If you leave them collocated, make sure to
set HAWQ Master Port property to a value different from the port number used by Ambari Server
database.', 'type': 'host-component', 'host': 'c6401.ambari.apache.org', 'level': 'WARN'}
+    self.assertEquals(validations[0], expected)
+
+
+  @patch('socket.getfqdn', side_effect=fqdn_mock_result)  
+  def test_getComponentLayoutValidations_nohawq_3_Hosts(self, socket_mock):
+    """ Test no failures when there are no HAWQ components on a 3-node cluster """
+
+    # normal placement, no warnings
+    services = self.load_json("services-normal-nohawq-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"]
+    hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in
componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"]
+    self.assertEquals(len(hawqMasterHosts), 0)
+    self.assertEquals(len(hawqStandbyHosts), 0)
+    
+    hosts = self.load_json("hosts-3-hosts.json")
+    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+    self.assertEquals(len(hostsList), 3)
+
+    validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
+    self.assertEquals(len(validations), 0)
+
+                           
   def test_recommendHDFSConfigurations(self):
     configurations = {}
     clusterData = {

http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-web/app/data/HDP2.3/site_properties.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/HDP2.3/site_properties.js b/ambari-web/app/data/HDP2.3/site_properties.js
index 6bdc373..cdbd9c2 100644
--- a/ambari-web/app/data/HDP2.3/site_properties.js
+++ b/ambari-web/app/data/HDP2.3/site_properties.js
@@ -231,6 +231,84 @@ hdp23properties.push({
     "serviceName": "RANGER",
     "filename": "ranger-admin-site.xml",
     "category": "ADSettings"
+  },
+  /*********************************************** HAWQ **********************************************/
+  {
+    "name": "hawq_master_address_host",
+    "displayType": "masterHost",
+    "filename": "hawq-site.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 0
+  },
+  {
+    "name": "hawq_standby_address_host",
+    "displayType": "masterHost",
+    "filename": "hawq-site.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 1
+  },
+  {
+    "name": "hawq_master_address_port",
+    "displayType": "int",
+    "filename": "hawq-site.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 2
+  },
+  {
+    "name": "hawq_segment_address_port",
+    "displayType": "int",
+    "filename": "hawq-site.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 3
+  },
+  {
+    "name": "hawq_dfs_url",
+    "filename": "hawq-site.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 4
+  },
+  {
+    "name": "hawq_master_directory",
+    "displayType": "directory",
+    "filename": "hawq-site.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 5
+  },
+  {
+    "name": "hawq_master_temp_directory",
+    "displayType": "directory",
+    "filename": "hawq-site.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 6
+  },
+  {
+    "name": "hawq_segment_directory",
+    "displayType": "directory",
+    "filename": "hawq-site.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 7
+  },
+  {
+    "name": "hawq_segment_temp_directory",
+    "displayType": "directory",
+    "filename": "hawq-site.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 8
+  },
+  {
+    "name": "content",
+    "serviceName": "HAWQ",
+    "filename": "gpcheck-env.xml",
+    "category": "AdvancedGpcheck"
   }
 );
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-web/app/models/stack_service.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/stack_service.js b/ambari-web/app/models/stack_service.js
index b783bd9..da2d41a 100644
--- a/ambari-web/app/models/stack_service.js
+++ b/ambari-web/app/models/stack_service.js
@@ -175,7 +175,7 @@ App.StackService = DS.Model.extend({
     var configTypes = this.get('configTypes');
     var serviceComponents = this.get('serviceComponents');
     if (configTypes && Object.keys(configTypes).length) {
-      var pattern = ["General", "CapacityScheduler", "FaultTolerance", "Isolation", "Performance",
"HIVE_SERVER2", "KDC", "Kadmin","^Advanced", "Env$", "^Custom", "Falcon - Oozie integration",
"FalconStartupSite", "FalconRuntimeSite", "MetricCollector", "Settings$"];
+      var pattern = ["General", "CapacityScheduler", "FaultTolerance", "Isolation", "Performance",
"HIVE_SERVER2", "KDC", "Kadmin","^Advanced", "Env$", "^Custom", "Falcon - Oozie integration",
"FalconStartupSite", "FalconRuntimeSite", "MetricCollector", "Settings$", "AdvancedGpcheck"];
       configCategories = App.StackService.configCategories.call(this).filter(function (_configCategory)
{
         var serviceComponentName = _configCategory.get('name');
         var isServiceComponent = serviceComponents.someProperty('componentName', serviceComponentName);
@@ -202,8 +202,10 @@ App.StackService.displayOrder = [
   'TEZ',
   'GANGLIA',
   'HIVE',
+  'HAWQ',
   'HBASE',
   'PIG',
+  'PXF',
   'SQOOP',
   'OOZIE',
   'ZOOKEEPER',
@@ -360,6 +362,12 @@ App.StackService.configCategories = function () {
       break;
     case 'SQOOP':
       break;
+    case 'HAWQ':
+      serviceConfigCategories.pushObjects([
+        App.ServiceConfigCategory.create({ name: 'General', displayName: 'General'}),
+        App.ServiceConfigCategory.create({ name: 'AdvancedGpcheck', displayName: 'Advanced
gpcheck'})
+      ]);
+      break;
     default:
       serviceConfigCategories.pushObjects([
         App.ServiceConfigCategory.create({ name: 'General', displayName: 'General'})

http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-web/app/utils/configs/config_property_helper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/config_property_helper.js b/ambari-web/app/utils/configs/config_property_helper.js
index 2170590..cb1f66b 100644
--- a/ambari-web/app/utils/configs/config_property_helper.js
+++ b/ambari-web/app/utils/configs/config_property_helper.js
@@ -219,9 +219,35 @@ module.exports = {
           configProperty.set('isRequired', 'false');
         }
         break;
+      case 'hawq_master_address_host':
+        configProperty.set('value', masterComponentHostsInDB.findProperty('component', 'HAWQMASTER').hostName);
+        break;
+      case 'hawq_standby_address_host':
+        var hawqStandby = masterComponentHostsInDB.findProperty('component', 'HAWQSTANDBY');
+        if (hawqStandby) {
+          configProperty.set('value', hawqStandby.hostName);
+        }
+        break;
+      case 'hawq_dfs_url':
+        var nnHost =  masterComponentHostsInDB.findProperty('component', 'NAMENODE').hostName;
+        this.setRecommendedValue(configProperty, hostWithPort, nnHost);
+        break;
+      case 'hawq_rm_yarn_address':
+        var rmHost = masterComponentHostsInDB.findProperty('component', 'RESOURCEMANAGER');
+        if (rmHost) {
+          this.setRecommendedValue(configProperty, hostWithPort, rmHost.hostName);
+        }
+        break;
+      case 'hawq_rm_yarn_scheduler_address':
+        var rmHost = masterComponentHostsInDB.findProperty('component', 'RESOURCEMANAGER');
+        if (rmHost) {
+          this.setRecommendedValue(configProperty, hostWithPort, rmHost.hostName);
+        }
+        break;
     }
   },
 
+  
   /**
    * Get hive.metastore.uris initial value
    * @param hosts

http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-web/test/data/HDP2.3/site_properties_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/data/HDP2.3/site_properties_test.js b/ambari-web/test/data/HDP2.3/site_properties_test.js
index 1bc8a7f..48a1988 100644
--- a/ambari-web/test/data/HDP2.3/site_properties_test.js
+++ b/ambari-web/test/data/HDP2.3/site_properties_test.js
@@ -26,7 +26,7 @@ describe('hdp2SiteProperties', function () {
    * @stackProperties: All the properties that are derived from stack definition
    */
   var stackProperties = siteProperties.filter(function(item){
-    return  (!(item.isRequiredByAgent === false || item.filename === 'alert_notification'
|| item.category === 'Ambari Principals' || item.name === 'oozie_hostname'))
+    return  (!(item.isRequiredByAgent === false || item.filename === 'alert_notification'
|| item.category === 'Ambari Principals' || item.name === 'oozie_hostname'  || item.serviceName
=== 'HAWQ'))
   });
 
   stackProperties.forEach(function(siteProperty){
@@ -75,4 +75,4 @@ describe('hdp2SiteProperties', function () {
     });
   });
 
-});
\ No newline at end of file
+});


Mime
View raw message