ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ja...@apache.org
Subject ambari git commit: AMBARI-14840: Expose default_segment_num property for HAWQ (adenissov via jaoki)
Date Fri, 29 Jan 2016 21:32:03 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk ea60d2980 -> 69c834195


AMBARI-14840: Expose default_segment_num property for HAWQ (adenissov via jaoki)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/69c83419
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/69c83419
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/69c83419

Branch: refs/heads/trunk
Commit: 69c83419517a5d87f9c460fc2b1bb8f41e50961a
Parents: ea60d29
Author: Jun Aoki <jaoki@apache.org>
Authored: Fri Jan 29 13:31:46 2016 -0800
Committer: Jun Aoki <jaoki@apache.org>
Committed: Fri Jan 29 13:31:46 2016 -0800

----------------------------------------------------------------------
 .../HAWQ/2.0.0/configuration/hawq-site.xml      | 14 ++++++++
 .../stacks/HDP/2.3/services/stack_advisor.py    |  9 +++++-
 .../common/services-normal-hawq-3-hosts.json    | 11 +++++++
 .../stacks/2.3/common/test_stack_advisor.py     | 34 ++++++++++++++++++++
 4 files changed, 67 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/69c83419/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
index 90982e9..f034749 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
@@ -205,4 +205,18 @@
     </description>
   </property>
 
+  <property>
+    <name>default_segment_num</name>
+    <display-name>Default Number of Virtual Segments</display-name>
+    <value>24</value>
+    <description>
+      The default number of virtual segments to use when executing a query statement. When
the query is actually executed,
+      the number of virtual segments may differ from this number depending on the query's
needs.
+      When expanding the cluster, you should adjust this number to reflect the number of
nodes in the new cluster times the number of virtual segments per node.
+    </description>
+    <value-attributes>
+      <type>int</type>
+    </value-attributes>
+  </property>
+
 </configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/69c83419/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 0544f5a..3e60f05 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -671,7 +671,14 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     if self.isHawqMasterComponentOnAmbariServer(services):
       if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]:
         putHawqSiteProperty('hawq_master_address_port', '')
-          
+    # calculate optimal number of virtual segments
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item["StackServiceComponents"] for sublist in componentsListList for
item in sublist]
+    numSegments = len(self.__getHosts(componentsList, "HAWQSEGMENT"))
+    # update default if segments are deployed
+    if numSegments and "hawq-site" in services["configurations"] and "default_segment_num"
in services["configurations"]["hawq-site"]["properties"]:
+      factor = 6 if numSegments < 50 else 4
+      putHawqSiteProperty('default_segment_num', numSegments * factor)
           
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()

http://git-wip-us.apache.org/repos/asf/ambari/blob/69c83419/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
b/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
index a16ab32..d10a976 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
+++ b/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
@@ -358,6 +358,17 @@
         "type" : "hawq-site.xml"
       },
       "dependencies" : [ ]
+    }, {
+      "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/configurations/default_segment_num",
+      "StackConfigurations" : {
+        "property_depends_on" : [ ],
+        "property_name" : "default_segment_num",
+        "service_name" : "HAWQ",
+        "stack_name" : "HDP",
+        "stack_version" : "2.3",
+        "type" : "hawq-site.xml"
+      },
+      "dependencies" : [ ]
     } ]
   }, {
     "href" : "/api/v1/stacks/HDP/versions/2.3/services/HDFS",

http://git-wip-us.apache.org/repos/asf/ambari/blob/69c83419/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 362a7cd..e345e2a 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1316,6 +1316,40 @@ class TestHDP23StackAdvisor(TestCase):
     self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
+  def test_recommendHAWQConfigurations(self):
+
+    # original cluster data with 3 segments
+    services = self.load_json("services-normal-hawq-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    hawqSegmentComponent = [component["StackServiceComponents"] for component in componentsList
if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
+    services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24"}}
+
+    configurations = {}
+    clusterData = {}
+
+    # Test 1 - with 3 segments
+    self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3)
+    self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(3
* 6))
+
+    # Test 2 - with 49 segments
+    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(49)]
+    self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(49
* 6))
+
+    # Test 3 - with 50 segments (threshold for new factor)
+    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(50)]
+    self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(50
* 4))
+
+    # Test 4 - with no segments
+    configurations = {}
+    hawqSegmentComponent["hostnames"] = []
+    self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
+    self.assertEquals(configurations, {'hawq-site': {'properties': {}}})
+
+
   def test_validateHiveConfigurations(self):
     properties = {"hive_security_authorization": "None",
                   "hive.exec.orc.default.stripe.size": "8388608",


Mime
View raw message