ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mithm...@apache.org
Subject ambari git commit: AMBARI-17473: Add additional parameters to hawq-site.xml (bhuvnesh2703, mithmatt)
Date Thu, 30 Jun 2016 07:45:31 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk b618d22bd -> 4c0b1f430


AMBARI-17473: Add additional parameters to hawq-site.xml (bhuvnesh2703, mithmatt)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4c0b1f43
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4c0b1f43
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4c0b1f43

Branch: refs/heads/trunk
Commit: 4c0b1f430983efdf1b1d4bd886e8558144752ff4
Parents: b618d22
Author: Matt <mmathew@pivotal.io>
Authored: Thu Jun 30 00:44:52 2016 -0700
Committer: Matt <mmathew@pivotal.io>
Committed: Thu Jun 30 00:44:52 2016 -0700

----------------------------------------------------------------------
 .../HAWQ/2.0.0/configuration/hawq-site.xml      | 238 +++++++++++++++++++
 .../2.0.0/configuration/hawq-sysctl-env.xml     |   9 +
 .../HAWQ/2.0.0/service_advisor.py               |   4 +
 .../HAWQ/2.0.0/themes/theme.json                |  86 ++++++-
 .../stacks/2.3/HAWQ/test_service_advisor.py     |  17 ++
 5 files changed, 341 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4c0b1f43/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
index 6e21d84..150b2c6 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
@@ -352,4 +352,242 @@
     </value-attributes>
     <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>gp_workfile_compress_algorithm</name>
+    <display-name>Spill Files Compression Algorithm</display-name>
+    <value>none</value>
+    <description>
+      Compression algorithm to use on the spill files when a hash aggregation or hash join
operation spills to disk during query processing.
+      If using zlib, it must be in your $PATH on all segments. The default is none.
+    </description>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>none</value>
+          <label>None</label>
+        </entry>
+        <entry>
+          <value>zlib</value>
+          <label>zlib</label>
+        </entry>
+      </entries>
+      <selection-cardinality>1</selection-cardinality>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>gp_workfile_limit_files_per_query</name>
+    <display-name>Max. No. of Temporary Spill Files allowed per Query per Segment</display-name>
+    <value>3000000</value>
+    <description>
+      Sets the maximum number of temporary spill files (also known as workfiles) allowed
per query per segment.
+      Spill files are created when executing a query that requires more memory than it is
allocated.
+      The current query is terminated when the limit is exceeded. The default is 3000000.
+    </description>
+    <value-attributes>
+      <type>int</type>
+      <minimum>0</minimum>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>gp_workfile_limit_per_query</name>
+    <display-name>Max. Disk Size Allowed for Individual Query for creating Temporary
Spill Files per Segment</display-name>
+    <value>0</value>
+    <description>
+      Sets the maximum disk size an individual query is allowed to use for creating temporary
spill files at each segment.
+      The default value is 0, which means a limit is not enforced.
+    </description>
+    <value-attributes>
+      <type>int</type>
+      <minimum>0</minimum>
+      <unit>kB</unit>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>gp_workfile_limit_per_segment</name>
+    <display-name>Max. Disk Size Allowed for All Queries for creating Temporary Spill
Files per Segment</display-name>
+    <value>0</value>
+    <description>
+      Sets the maximum total disk size that all running queries are allowed to use for creating
temporary spill files at each segment.
+      The default value is 0, which means a limit is not enforced.
+    </description>
+    <value-attributes>
+      <type>int</type>
+      <minimum>0</minimum>
+      <unit>kB</unit>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>gp_autostats_mode</name>
+    <value>on_no_stats</value>
+    <description>
+      Specifies the mode for triggering automatic statistics collection with ANALYZE.
+      The on_no_stats option triggers statistics collection for CREATE TABLE AS SELECT, INSERT,
or COPY operations on any table that has no existing statistics.
+      The on_change option triggers statistics collection only when the number of rows affected
exceeds the threshold defined by gp_autostats_on_change_threshold.
+      Operations that can trigger automatic statistics collection with on_change are CREATE
TABLE AS SELECT, UPDATE, DELETE, INSERT and COPY.
+      The default is on_no_stats.
+    </description>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>none</value>
+        </entry>
+        <entry>
+          <value>on_change</value>
+        </entry>
+        <entry>
+          <value>on_no_stats</value>
+        </entry>
+      </entries>
+      <selection-cardinality>1</selection-cardinality>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>gp_autostats_on_change_threshold</name>
+    <value>2147483647</value>
+    <description>
+      Specifies the threshold for automatic statistics collection when gp_autostats_mode
is set to on_change.
+      When a triggering table operation affects a number of rows exceeding this threshold,
ANALYZE is added and statistics are collected for the table.
+      The default is 2147483647.
+    </description>
+    <value-attributes>
+      <type>int</type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>gp_external_max_segs</name>
+    <value>64</value>
+    <description>
+      Sets the number of segments that will scan external table data during an external table
operation, the purpose being not to overload the system with scanning data and take away resources
from other concurrent operations.
+      This only applies to external tables that use the gpfdist://protocol to access external
table data. The default is 64.
+    </description>
+    <value-attributes>
+      <type>int</type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>hawq_rm_resource_idle_timeout</name>
+    <value>300</value>
+    <description>
+      Amount of time, in seconds, before idle resources are returned to YARN. The default
is 300.
+    </description>
+    <value-attributes>
+      <type>int</type>
+      <minimum>-1</minimum>
+      <maximum>65535</maximum>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>log_duration</name>
+    <value>off</value>
+    <description>
+      Causes the duration of every completed statement which satisfies log_statement to be
logged. The default is off.
+    </description>
+    <value-attributes>
+      <type>string</type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>log_rotation_age</name>
+    <value>1d</value>
+    <description>
+      Determines the maximum lifetime of an individual log file. After this time has elapsed,
a new log file will be created.
+      Set to zero to disable time-based creation of new log files. The default is 1d.
+    </description>
+    <value-attributes>
+      <type>string</type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>log_statement</name>
+    <value>ALL</value>
+    <description>
+      Controls which SQL statements are logged. DDL logs all data definition commands like
CREATE, ALTER, and DROP commands.
+      MOD logs all DDL statements, plus INSERT, UPDATE, DELETE, TRUNCATE, and COPY FROM.
+      PREPARE and EXPLAIN ANALYZE statements are also logged if their contained command is
of an appropriate type.
+      The default is ALL.
+    </description>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>NONE</value>
+        </entry>
+        <entry>
+          <value>DDL</value>
+        </entry>
+        <entry>
+          <value>MOD</value>
+        </entry>
+        <entry>
+          <value>ALL</value>
+        </entry>
+      </entries>
+      <selection-cardinality>1</selection-cardinality>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>log_statement_stats</name>
+    <value>off</value>
+    <description>
+      For each query, write total performance statistics of the query parser, planner, and
executor to the server log.
+      This is a crude profiling instrument. The default is off.
+    </description>
+    <value-attributes>
+      <type>string</type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>log_truncate_on_rotation</name>
+    <value>off</value>
+    <description>
+      Truncates (overwrites), rather than appends to, any existing log file of the same name.
+      Truncation will occur only when a new file is being opened due to time-based rotation.
+      For example, using this setting in combination with a log_filename such as hawq-%H.log
would result in generating twenty-four hourly log files and then cyclically overwriting them.
+      When off, pre-existing files will be appended to in all cases. The default is off.
+    </description>
+    <value-attributes>
+      <type>string</type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>runaway_detector_activation_percent</name>
+    <value>95</value>
+    <description>
+      Sets the percentage of the virtual memory quota that will trigger the termination of
queries.
+      If the amount of virtual memory utilized by a physical segment exceeds the calculated
threshold, then HAWQ begins terminating queries based on memory usage, starting with the query
that is consuming the largest amount of memory.
+      Queries are terminated until the percentage of utilized virtual memory is below the
specified percentage. The default is 95.
+    </description>
+    <value-attributes>
+      <type>int</type>
+      <minimum>0</minimum>
+      <maximum>100</maximum>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>statement_timeout</name>
+    <value>0</value>
+    <description>
+      Abort any statement that takes over the specified number of milliseconds. 0 turns off
the limitation. The default is 0.
+    </description>
+    <value-attributes>
+      <type>int</type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c0b1f43/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml
index 1d27052..290239e 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml
@@ -343,8 +343,17 @@
       When overcommit_memory is set to 2, the committed address space is not permitted to
exceed swap plus this percentage of physical RAM.
       Default is 50.
     </description>
+    <depends-on>
+      <property>
+        <type>hawq-sysctl-env</type>
+        <name>vm.overcommit_memory</name>
+      </property>
+    </depends-on>
     <value-attributes>
       <type>int</type>
+      <minimum>0</minimum>
+      <maximum>200</maximum>
+      <increment-step>25</increment-step>
     </value-attributes>
     <on-ambari-upgrade add="true"/>
   </property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c0b1f43/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
index f0e8b33..e254094 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py
@@ -146,6 +146,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
     putHawqSitePropertyAttribute = self.putPropertyAttribute(configurations, "hawq-site")
     hawq_sysctl_env = services["configurations"]["hawq-sysctl-env"]["properties"]
     putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services)
+    putHawqSysctlEnvPropertyAttribute = self.putPropertyAttribute(configurations, "hawq-sysctl-env")
 
     # remove master port when master is colocated with Ambari server
     if self.isHawqMasterComponentOnAmbariServer(services) and "hawq_master_address_port"
in hawq_site:
@@ -188,6 +189,9 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
         # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater
than 32GB
         vm_overcommit_mem_value = int(hawq_sysctl_env["vm.overcommit_memory"])
       putHawqSysctlEnvProperty("vm.overcommit_ratio", vm_overcommit_ratio)
+      # Show vm.overcommit_ratio on theme only if vm.overcommit_memory is set to 2
+      overcommit_ratio_visibility = "true" if vm_overcommit_mem_value == 2 else "false"
+      putHawqSysctlEnvPropertyAttribute("vm.overcommit_ratio", "visible", overcommit_ratio_visibility)
       putHawqSysctlEnvProperty("vm.overcommit_memory", vm_overcommit_mem_value)
       host_ram_kb = minHawqHostsMemory * vm_overcommit_ratio / 100 if vm_overcommit_mem_value
== 2 else minHawqHostsMemory
       host_ram_gb = float(host_ram_kb) / (1024 * 1024)

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c0b1f43/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
index d7940a1..3c8a897 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json
@@ -11,7 +11,7 @@
                   "display-name":"Settings",
                   "layout":{
                      "tab-columns":"1",
-                     "tab-rows":"2",
+                     "tab-rows":"3",
                      "sections":[
                         {
                            "name":"general",
@@ -66,6 +66,32 @@
                                  "column-span":"1"
                               }
                            ]
+                        },
+                        {
+                           "name":"workfile-limits",
+                           "display-name":"Workfile Limits",
+                           "row-index":"2",
+                           "column-index":"0",
+                           "row-span":"1",
+                           "column-span":"1",
+                           "section-columns":"2",
+                           "section-rows":"1",
+                           "subsections":[
+                              {
+                                 "name":"workfile-limits-col1",
+                                 "row-index":"0",
+                                 "column-index":"0",
+                                 "row-span":"1",
+                                 "column-span":"1"
+                              },
+                              {
+                                 "name":"workfile-limits-col2",
+                                 "row-index":"0",
+                                 "column-index":"1",
+                                 "row-span":"1",
+                                 "column-span":"1"
+                              }
+                           ]
                         }
                      ]
                   }
@@ -80,60 +106,65 @@
                "config":"hawq-site/hawq_master_directory",
                "subsection-name":"general-col1"
             },
-
             {
                "config":"hawq-site/hawq_segment_directory",
                "subsection-name":"general-col2"
             },
-
             {
                "config":"hawq-site/hawq_master_temp_directory",
                "subsection-name":"general-col1"
             },
-
             {
                "config":"hawq-site/hawq_segment_temp_directory",
                "subsection-name":"general-col2"
             },
-
             {
                "config":"hawq-site/default_hash_table_bucket_number",
                "subsection-name":"resource-management-col2"
             },
-
             {
                "config":"hawq-site/hawq_rm_nvseg_perquery_perseg_limit",
                "subsection-name":"resource-management-col2"
             },
-
             {
                "config":"hawq-site/hawq_rm_nvseg_perquery_limit",
                "subsection-name":"resource-management-col2"
             },
-
             {
                "config":"hawq-site/hawq_global_rm_type",
                "subsection-name":"resource-management-col1"
             },
-
             {
                "config":"hawq-sysctl-env/vm.overcommit_memory",
                "subsection-name":"resource-management-col1"
             },
-
             {
                "config":"hawq-sysctl-env/vm.overcommit_ratio",
                "subsection-name":"resource-management-col1"
             },
-
             {
                "config":"hawq-site/hawq_rm_memory_limit_perseg",
                "subsection-name":"resource-management-col1"
             },
-
             {
                "config":"hawq-site/hawq_rm_nvcore_limit_perseg",
                "subsection-name":"resource-management-col2"
+            },
+            {
+               "config":"hawq-site/gp_workfile_compress_algorithm",
+               "subsection-name":"workfile-limits-col1"
+            },
+            {
+               "config":"hawq-site/gp_workfile_limit_files_per_query",
+               "subsection-name":"workfile-limits-col1"
+            },
+            {
+               "config":"hawq-site/gp_workfile_limit_per_query",
+               "subsection-name":"workfile-limits-col2"
+            },
+            {
+               "config":"hawq-site/gp_workfile_limit_per_segment",
+               "subsection-name":"workfile-limits-col2"
             }
          ]
       },
@@ -198,7 +229,12 @@
          {
             "config":"hawq-sysctl-env/vm.overcommit_ratio",
             "widget":{
-               "type":"text-field"
+               "type":"slider",
+               "units": [
+                  {
+                    "unit-name":"int"
+                  }
+               ]
             }
          },
           {
@@ -234,6 +270,30 @@
                   }
                ]
             }
+         },
+         {
+            "config":"hawq-site/gp_workfile_compress_algorithm",
+            "widget":{
+               "type":"combo"
+            }
+         },
+         {
+            "config":"hawq-site/gp_workfile_limit_files_per_query",
+            "widget":{
+               "type":"text-field"
+            }
+         },
+         {
+            "config":"hawq-site/gp_workfile_limit_per_query",
+            "widget":{
+               "type":"text-field"
+            }
+         },
+         {
+            "config":"hawq-site/gp_workfile_limit_per_segment",
+            "widget":{
+               "type":"text-field"
+            }
          }
       ]
    }

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c0b1f43/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
index 12f4fa1..8d97baa 100644
--- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py
@@ -299,3 +299,20 @@ class TestHAWQ200ServiceAdvisor(TestCase):
     self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services,
hosts)
     for property, status in properties_visible_status.iteritems():
       self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"],
status)
+
+    ## Test if vm.overcommit_ratio is set to visible / invisible based on the value of vm.overcommit_memory
+
+    # Case 1: vm.overcommit_ratio should be invisible when overcommit_memory is set as 0
+    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] =
0
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services,
hosts)
+    self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"],
"false")
+
+    # Case 2: vm.overcommit_ratio should be invisible when overcommit_memory is set as 1
+    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] =
1
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services,
hosts)
+    self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"],
"false")
+
+    # Case 3: vm.overcommit_ratio should be visible when overcommit_memory is set as 2
+    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] =
2
+    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services,
hosts)
+    self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"],
"true")
\ No newline at end of file


Mime
View raw message