ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ja...@apache.org
Subject ambari git commit: AMBARI-15435: Update Hawq GUCS: Refactor resource related GUC (goutamtadi via jaoki)
Date Thu, 17 Mar 2016 23:43:06 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 1bf667ead -> b663dbe1b


AMBARI-15435: Update Hawq GUCS: Refactor resource related GUC (goutamtadi via jaoki)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b663dbe1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b663dbe1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b663dbe1

Branch: refs/heads/trunk
Commit: b663dbe1be5291d8d32de7ec2b4434612d0861c0
Parents: 1bf667e
Author: Jun Aoki <jaoki@apache.org>
Authored: Thu Mar 17 16:42:56 2016 -0700
Committer: Jun Aoki <jaoki@apache.org>
Committed: Thu Mar 17 16:42:56 2016 -0700

----------------------------------------------------------------------
 .../HAWQ/2.0.0/configuration/hawq-site.xml      |  66 ++++++++---
 .../HAWQ/2.0.0/themes/theme.json                | 113 ++++++++++++++++---
 .../stacks/HDP/2.3/services/stack_advisor.py    |   5 +-
 .../stacks/2.3/common/test_stack_advisor.py     |  15 +--
 4 files changed, 157 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b663dbe1/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
index d187017..93ad3fe 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
@@ -114,7 +114,7 @@
 
   <property>
     <name>hawq_master_temp_directory</name>
-    <display-name>HAWQ Master Temp Directory</display-name>
+    <display-name>HAWQ Master Temp Directories</display-name>
     <value>/tmp/hawq/master</value>
     <description>
       Used as temporary filespace to store workfile cache, sort/join temp files etc.
@@ -128,7 +128,7 @@
 
   <property>
     <name>hawq_segment_temp_directory</name>
-    <display-name>HAWQ Segment Temp Directory</display-name>
+    <display-name>HAWQ Segment Temp Directories</display-name>
     <value>/tmp/hawq/segment</value>
     <description>
       Used as temporary filespace to store workfile cache, sort/join temp files etc.
@@ -146,17 +146,17 @@
     <display-name>Resource Manager</display-name>
     <value>none</value>
     <description>
-      HAWQ global resource manager type. Valid values are YARN ('yarn') and Native ('none').
-      Native('none') indicates that the HAWQ resource manager manages its own resources exclusively
for the entire cluster.
+      HAWQ global resource manager type. Valid values are YARN ('yarn') and Standalone ('none').
+      Standalone('none') indicates that the HAWQ resource manager manages its own resources
exclusively for the entire cluster.
       YARN ('yarn') means that HAWQ will negotiate with YARN's resource manager for resources.
-      The default is Native ('none').
+      The default is Standalone ('none').
     </description>
     <value-attributes>
       <type>value-list</type>
       <entries>
         <entry>
           <value>none</value>
-          <label>Native</label>
+          <label>Standalone</label>
         </entry>
         <entry>
           <value>yarn</value>
@@ -172,7 +172,7 @@
     <display-name>Segment Memory Usage Limit</display-name>
     <value>64GB</value>
     <description>
-      The maximum memory that can be used by a HAWQ segment when Resource Manager ('hawq_global_rm_type')
is set to Native ('none').
+      The maximum memory that can be used by a HAWQ segment when Resource Manager ('hawq_global_rm_type')
is set to Standalone ('none').
       The default is 64GB.
     </description>
     <value-attributes>
@@ -185,7 +185,7 @@
     <display-name>Segment Virtual Core Usage Limit</display-name>
     <value>16</value>
     <description>
-      The maximum number of virtual cores that can be used for query execution in a HAWQ
segment when Resource Manager ('hawq_global_rm_type') is set to Native ('none').
+      The maximum number of virtual cores that can be used for query execution in a HAWQ
segment when Resource Manager ('hawq_global_rm_type') is set to Standalone ('none').
       The default is 16.
     </description>
     <value-attributes>
@@ -299,18 +299,52 @@
   </property>
 
   <property>
-    <name>default_segment_num</name>
-    <display-name>Default Number of Virtual Segments</display-name>
-    <value>24</value>
+    <name>default_hash_table_bucket_number</name>
+    <display-name>Default buckets for Hash Distributed tables</display-name>
+    <value>6</value>
     <description>
-      The default number of virtual segments to use when executing a query statement.
-      When the query is actually executed, the number of virtual segments may differ from
this number depending on the query's needs.
-      When expanding the cluster, you should adjust this number to reflect the number of
nodes in the new cluster times the number of virtual segments per node.
-      The default is 24.
+     The default number of virtual segments per query for hash distributed tables that can
be used for query execution in a HAWQ segment when Resource Manager ('hawq_global_rm_type')
is set to Standalone ('none').
+      The default is 6.
     </description>
     <value-attributes>
       <type>int</type>
-      <minimum>0</minimum>
+      <minimum>1</minimum>
+      <maximum>10000</maximum>
+      <increment-step>100</increment-step>
+    </value-attributes>
+  </property>
+
+    <property>
+    <name>hawq_rm_nvseg_perquery_perseg_limit</name>
+    <display-name>Virtual Segments Limit per Query (per Segment Node)</display-name>
+    <value>6</value>
+    <description>
+      Not recommended to change this parameter.
+       The maximum  number of virtual segments per query per Segment Node that can be used
for query execution in a HAWQ segment when Resource Manager ('hawq_global_rm_type') is set
to Standalone ('none').
+      The default is 6.
+    </description>
+    <value-attributes>
+      <type>int</type>
+      <minimum>1</minimum>
+      <maximum>16</maximum>
+       <increment-step>1</increment-step>
+    </value-attributes>
+  </property>
+
+   <property>
+    <name>hawq_rm_nvseg_perquery_limit</name>
+    <display-name>Virtual Segments Limit per Query (Total)</display-name>
+    <value>512</value>
+    <description>
+      Not recommended to change this parameter.
+      The maximum total number of virtual segments per query that can be used for query execution
in a HAWQ segment when Resource Manager ('hawq_global_rm_type') is set to Standalone ('none').
+      The default is 512.
+    </description>
+    <value-attributes>
+      <type>int</type>
+      <minimum>1</minimum>
+      <maximum>10000</maximum>
+       <increment-step>100</increment-step>
     </value-attributes>
   </property>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/b663dbe1/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
index aa35311..a23afe6 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
@@ -10,8 +10,8 @@
                   "name":"settings",
                   "display-name":"Settings",
                   "layout":{
-                     "tab-columns":"2",
-                     "tab-rows":"1",
+                     "tab-columns":"1",
+                     "tab-rows":"2",
                      "sections":[
                         {
                            "name":"general",
@@ -20,26 +20,35 @@
                            "column-index":"0",
                            "row-span":"1",
                            "column-span":"1",
-                           "section-columns":"1",
+                           "section-columns":"2",
                            "section-rows":"1",
                            "subsections":[
                               {
                                  "name":"general-col1",
+                                 "display-name":"HAWQ Masters",
                                  "row-index":"0",
                                  "column-index":"0",
                                  "row-span":"1",
                                  "column-span":"1"
+                              },
+                              {
+                                 "name":"general-col2",
+                                 "display-name":"HAWQ Segments",
+                                 "row-index":"0",
+                                 "column-index":"1",
+                                 "row-span":"1",
+                                 "column-span":"1"
                               }
                            ]
                         },
                         {
                            "name":"resource-management",
                            "display-name":"Resource Management",
-                           "row-index":"0",
-                           "column-index":"1",
+                           "row-index":"1",
+                           "column-index":"0",
                            "row-span":"1",
                            "column-span":"1",
-                           "section-columns":"1",
+                           "section-columns":"2",
                            "section-rows":"1",
                            "subsections":[
                               {
@@ -48,6 +57,13 @@
                                  "column-index":"0",
                                  "row-span":"1",
                                  "column-span":"1"
+                              },
+                              {
+                                 "name":"resource-management-col2",
+                                 "row-index":"0",
+                                 "column-index":"1",
+                                 "row-span":"1",
+                                 "column-span":"1"
                               }
                            ]
                         }
@@ -64,26 +80,52 @@
                "config":"hawq-site/hawq_master_directory",
                "subsection-name":"general-col1"
             },
+
             {
                "config":"hawq-site/hawq_segment_directory",
-               "subsection-name":"general-col1"
+               "subsection-name":"general-col2"
             },
+
             {
-               "config":"hawq-site/default_segment_num",
+               "config":"hawq-site/hawq_master_temp_directory",
                "subsection-name":"general-col1"
             },
+
             {
-               "config":"hawq-sysctl-env/vm.overcommit_memory",
-               "subsection-name":"general-col1"
+               "config":"hawq-site/hawq_segment_temp_directory",
+               "subsection-name":"general-col2"
+            },
+
+            {
+               "config":"hawq-site/default_hash_table_bucket_number",
+               "subsection-name":"resource-management-col2"
+            },
+
+            {
+               "config":"hawq-site/hawq_rm_nvseg_perquery_perseg_limit",
+               "subsection-name":"resource-management-col2"
+            },
+
+            {
+               "config":"hawq-site/hawq_rm_nvseg_perquery_limit",
+               "subsection-name":"resource-management-col2"
             },
+
             {
                "config":"hawq-site/hawq_global_rm_type",
                "subsection-name":"resource-management-col1"
             },
+
+            {
+               "config":"hawq-sysctl-env/vm.overcommit_memory",
+               "subsection-name":"resource-management-col1"
+            },
+
             {
                "config":"hawq-site/hawq_rm_memory_limit_perseg",
                "subsection-name":"resource-management-col1"
             },
+
             {
                "config":"hawq-site/hawq_rm_nvcore_limit_perseg",
                "subsection-name":"resource-management-col1"
@@ -104,12 +146,57 @@
             }
          },
          {
-            "config":"hawq-site/default_segment_num",
+            "config":"hawq-site/hawq_master_temp_directory",
             "widget":{
-               "type":"text-field"
+               "type":"directories"
             }
          },
          {
+            "config":"hawq-site/hawq_segment_temp_directory",
+            "widget":{
+               "type":"directories"
+            }
+         },
+         {
+            "config":"hawq-site/default_hash_table_bucket_number",
+            "widget":{
+               "type":"slider",
+               "units": [
+                  {
+                    "unit-name":"int"
+                  }
+               ]
+            }
+         },
+         {
+            "config":"hawq-site/hawq_rm_nvseg_perquery_perseg_limit",
+            "widget":{
+               "type":"slider",
+               "units": [
+                  {
+                    "unit-name":"int"
+                  }
+               ]
+            }
+         },
+         {
+            "config":"hawq-site/hawq_rm_nvseg_perquery_limit",
+            "widget":{
+               "type":"slider",
+               "units": [
+                  {
+                    "unit-name":"int"
+                  }
+               ]
+            }
+         },
+         {
+            "config":"hawq-sysctl-env/vm.overcommit_memory",
+            "widget":{
+               "type":"combo"
+            }
+         },
+          {
             "config":"hawq-sysctl-env/vm.overcommit_memory",
             "widget":{
                "type":"combo"
@@ -140,4 +227,4 @@
          }
       ]
    }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/b663dbe1/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 8e1aff2..6359a49 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -687,9 +687,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     # calculate optimal number of virtual segments
     numSegments = len(self.__getHosts(componentsList, "HAWQSEGMENT"))
     # update default if segments are deployed
-    if numSegments and "default_segment_num" in hawq_site:
-      factor = 6 if numSegments < 50 else 4
-      putHawqSiteProperty('default_segment_num', numSegments * factor)
+    if numSegments and "default_hash_table_bucket_number" in hawq_site:
+      putHawqSiteProperty('default_hash_table_bucket_number', numSegments * 6)
 
     # update YARN RM urls with the values from yarn-site if YARN is installed
     if "YARN" in servicesList and "yarn-site" in services["configurations"]:

http://git-wip-us.apache.org/repos/asf/ambari/blob/b663dbe1/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 7a2a37d..0799f9f 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1471,7 +1471,7 @@ class TestHDP23StackAdvisor(TestCase):
     hawqSegmentComponent = [component["StackServiceComponents"] for component in componentsList
if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
 
     # setup default configuration values
-    services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24",
+    services["configurations"]["hawq-site"] = {"properties": {"default_hash_table_bucket_number":
"24",
                                                               "hawq_rm_yarn_address": "localhost:8032",
                                                               "hawq_rm_yarn_scheduler_address":
"localhost:8030"}}
     services["configurations"]["yarn-site"] = {"properties": {"yarn.resourcemanager.address":
"host1:8050",
@@ -1483,21 +1483,16 @@ class TestHDP23StackAdvisor(TestCase):
     # Test 1 - with 3 segments
     self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3)
     self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
-    self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(3
* 6))
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"],
str(3 * 6))
 
     # check derived properties
     self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_address"],
"host1:8050")
     self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_scheduler_address"],
"host1:8030")
 
-    # Test 2 - with 49 segments
-    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(49)]
+    # Test 2 - with 100 segments
+    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(100)]
     self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
-    self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(49
* 6))
-
-    # Test 3 - with 50 segments (threshold for new factor)
-    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(50)]
-    self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services,
None)
-    self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(50
* 4))
+    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"],
str(100 * 6))
 
     # Test 4 - with no segments
     configurations = {}


Mime
View raw message