Return-Path: 
 <commits-return-24421-apmail-ambari-commits-archive=ambari.apache.org@ambari.apache.org>
X-Original-To: apmail-ambari-commits-archive@www.apache.org
Delivered-To: apmail-ambari-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id 105D41881E
	for <apmail-ambari-commits-archive@www.apache.org>;
 Fri, 29 Jan 2016 21:36:15 +0000 (UTC)
Received: (qmail 53879 invoked by uid 500); 29 Jan 2016 21:35:16 -0000
Delivered-To: apmail-ambari-commits-archive@ambari.apache.org
Received: (qmail 53851 invoked by uid 500); 29 Jan 2016 21:35:16 -0000
Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@ambari.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@ambari.apache.org>
List-Post: <mailto:commits@ambari.apache.org>
List-Id: <commits.ambari.apache.org>
Reply-To: ambari-dev@ambari.apache.org
Delivered-To: mailing list commits@ambari.apache.org
Received: (qmail 53842 invoked by uid 99); 29 Jan 2016 21:35:16 -0000
Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org)
 (140.211.11.23)
    by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 29 Jan 2016 21:35:16 +0000
Received: by git1-us-west.apache.org (ASF Mail Server at
 git1-us-west.apache.org, from userid 33)
	id 8251CE01C1; Fri, 29 Jan 2016 21:35:16 +0000 (UTC)
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: jaoki@apache.org
To: commits@ambari.apache.org
Message-Id: <cf75555c55e645cd82948a63f2e22b74@git.apache.org>
X-Mailer: ASF-Git Admin Mailer
Subject: ambari git commit: AMBARI-14840: Expose default_segment_num property
 for HAWQ (adenissov via jaoki)
Date: Fri, 29 Jan 2016 21:35:16 +0000 (UTC)

Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 801e00c2c -> 6460f686c


AMBARI-14840: Expose default_segment_num property for HAWQ (adenissov via jaoki)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6460f686
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6460f686
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6460f686

Branch: refs/heads/branch-2.2
Commit: 6460f686c54bb3d784d1dbbd2d692a1ec3dbfa20
Parents: 801e00c
Author: Jun Aoki <jaoki@apache.org>
Authored: Fri Jan 29 13:35:07 2016 -0800
Committer: Jun Aoki <jaoki@apache.org>
Committed: Fri Jan 29 13:35:07 2016 -0800

----------------------------------------------------------------------
 .../HAWQ/2.0.0/configuration/hawq-site.xml      | 14 ++++++++
 .../stacks/HDP/2.3/services/stack_advisor.py    |  9 +++++-
 .../common/services-normal-hawq-3-hosts.json    | 11 +++++++
 .../stacks/2.3/common/test_stack_advisor.py     | 34 ++++++++++++++++++++
 4 files changed, 67 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6460f686/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
index 90982e9..f034749 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
@@ -205,4 +205,18 @@
     </description>
   </property>
 
+  <property>
+    <name>default_segment_num</name>
+    <display-name>Default Number of Virtual Segments</display-name>
+    <value>24</value>
+    <description>
+      The default number of virtual segments to use when executing a query statement. When the query is actually executed,
+      the number of virtual segments may differ from this number depending on the query's needs.
+      When expanding the cluster, you should adjust this number to reflect the number of nodes in the new cluster times the number of virtual segments per node.
+    </description>
+    <value-attributes>
+      <type>int</type>
+    </value-attributes>
+  </property>
+
 </configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/6460f686/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 0efed8e..b9bb8af 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -672,7 +672,14 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     if self.isHawqMasterComponentOnAmbariServer(services):
       if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]:
         putHawqSiteProperty('hawq_master_address_port', '')
-          
+    # calculate optimal number of virtual segments
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+    numSegments = len(self.__getHosts(componentsList, "HAWQSEGMENT"))
+    # update default if segments are deployed
+    if numSegments and "hawq-site" in services["configurations"] and "default_segment_num" in services["configurations"]["hawq-site"]["properties"]:
+      factor = 6 if numSegments < 50 else 4
+      putHawqSiteProperty('default_segment_num', numSegments * factor)
           
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()

http://git-wip-us.apache.org/repos/asf/ambari/blob/6460f686/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json b/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
index 9fab56d..7a5a7b7 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
+++ b/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
@@ -358,6 +358,17 @@
         "type" : "hawq-site.xml"
       },
       "dependencies" : [ ]
+    }, {
+      "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/configurations/default_segment_num",
+      "StackConfigurations" : {
+        "property_depends_on" : [ ],
+        "property_name" : "default_segment_num",
+        "service_name" : "HAWQ",
+        "stack_name" : "HDP",
+        "stack_version" : "2.3",
+        "type" : "hawq-site.xml"
+      },
+      "dependencies" : [ ]
     } ]
   }, {
     "href" : "/api/v1/stacks/HDP/versions/2.3/services/HDFS",

http://git-wip-us.apache.org/repos/asf/ambari/blob/6460f686/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 5267fac..cc397c3 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1304,6 +1304,40 @@ class TestHDP23StackAdvisor(TestCase):
     self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
+  def test_recommendHAWQConfigurations(self):
+
+    # original cluster data with 3 segments
+    services = self.load_json("services-normal-hawq-3-hosts.json")
+    componentsListList = [service["components"] for service in services["services"]]
+    componentsList = [item for sublist in componentsListList for item in sublist]
+    hawqSegmentComponent = [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
+    services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24"}}
+
+    configurations = {}
+    clusterData = {}
+
+    # Test 1 - with 3 segments
+    self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3)
+    self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(3 * 6))
+
+    # Test 2 - with 49 segments
+    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(49)]
+    self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(49 * 6))
+
+    # Test 3 - with 50 segments (threshold for new factor)
+    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(50)]
+    self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(50 * 4))
+
+    # Test 4 - with no segments
+    configurations = {}
+    hawqSegmentComponent["hostnames"] = []
+    self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
+    self.assertEquals(configurations, {'hawq-site': {'properties': {}}})
+
+
   def test_validateHiveConfigurations(self):
     properties = {"hive_security_authorization": "None",
                   "hive.exec.orc.default.stripe.size": "8388608",