ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject [2/2] ambari git commit: AMBARI-18901. Use 'Number of LLAP Nodes' selected as the driver for LLAP config calculations.
Date Mon, 21 Nov 2016 20:13:10 GMT
AMBARI-18901. Use 'Number of LLAP Nodes' selected as the driver for LLAP config calculations.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8be71baf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8be71baf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8be71baf

Branch: refs/heads/branch-feature-AMBARI-18901
Commit: 8be71baf4798d7a3abbc9e56b1b80d763bd5864e
Parents: 0c5850a
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Mon Nov 21 12:11:51 2016 -0800
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Mon Nov 21 12:11:51 2016 -0800

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog250.java       |  71 +-
 .../HIVE/configuration/hive-interactive-env.xml |  42 +-
 .../configuration/hive-interactive-site.xml     |  68 +-
 .../HIVE/configuration/tez-interactive-site.xml |  71 +-
 .../HDP/2.5/services/HIVE/themes/theme.json     |  27 +-
 .../YARN/configuration/capacity-scheduler.xml   |   2 +-
 .../stacks/HDP/2.5/services/stack_advisor.py    | 665 +++++++++++--------
 .../server/upgrade/UpgradeCatalog250Test.java   | 287 +++++++-
 8 files changed, 854 insertions(+), 379 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8be71baf/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index bc3c120..6de48c1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -17,13 +17,8 @@
  */
 package org.apache.ambari.server.upgrade;
 
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
+import com.google.inject.Inject;
+import com.google.inject.Injector;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.CommandExecutionType;
 import org.apache.ambari.server.controller.AmbariManagementController;
@@ -37,8 +32,12 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.inject.Inject;
-import com.google.inject.Injector;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Upgrade catalog for version 2.5.0.
@@ -134,6 +133,8 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
     addNewConfigurationsFromXml();
     updateAMSConfigs();
     updateKafkaConfigs();
+    updateHIVEInteractiveConfigs();
+    updateTEZInteractiveConfigs();
   }
 
   protected void updateHostVersionTable() throws SQLException {
@@ -269,5 +270,57 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
     dbAccessor.addColumn(SERVICE_DESIRED_STATE_TABLE,
       new DBColumnInfo(CREDENTIAL_STORE_ENABLED_COL, Short.class, null, 0, false));
   }
+
+  /**
+   * Updates Hive Interactive's config in hive-interactive-site.
+   *
+   * @throws AmbariException
+   */
+  protected void updateHIVEInteractiveConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = clusters.getClusters();
+
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+          Config hiveInteractiveSite = cluster.getDesiredConfigByType("hive-interactive-site");
+          if (hiveInteractiveSite != null) {
+            updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.tez.container.size",
+                "SET_ON_FIRST_INVOCATION"), true, true);
+
+            updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size",
+                "1000000000"), true, true);
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Updates Tez for Hive2 Interactive's config in tez-interactive-site.
+   *
+   * @throws AmbariException
+   */
+  protected void updateTEZInteractiveConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = clusters.getClusters();
+
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+          Config tezInteractiveSite = cluster.getDesiredConfigByType("tez-interactive-site");
+          if (tezInteractiveSite != null) {
+
+            updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.io.sort.mb",
"512"), true, true);
+
+            updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.unordered.output.buffer.size-mb",
+                "100"), true, true);
+          }
+        }
+      }
+    }
+  }
 }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8be71baf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index 1fd72eb..5a186b1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -53,40 +53,14 @@
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
-    <name>llap_queue_capacity</name>
-    <value>0</value>
-    <description>Percentage of the cluster dedicated to interactive query.</description>
-    <display-name>% of Cluster Capacity</display-name>
-    <value-attributes>
-      <type>int</type>
-      <minimum>20</minimum>
-      <maximum>100</maximum>
-      <increment-step>1</increment-step>
-    </value-attributes>
-    <depends-on>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>enable_hive_interactive</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.llap.daemon.queue.name</name>
-      </property>
-      <property>
-        <type>capacity-scheduler</type>
-        <name>yarn.scheduler.capacity.root.queues</name>
-      </property>
-    </depends-on>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
     <name>num_llap_nodes</name>
     <value>1</value>
     <description>The number of Hive LLAP daemons to run.</description>
-    <display-name>Number of LLAP Daemons</display-name>
+    <display-name>Number of LLAP Nodes</display-name>
     <value-attributes>
       <type>int</type>
       <minimum>1</minimum>
+      <increment-step>1</increment-step>
     </value-attributes>
     <depends-on>
       <property>
@@ -94,10 +68,6 @@
         <name>enable_hive_interactive</name>
       </property>
       <property>
-        <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
-      </property>
-      <property>
         <type>hive-interactive-site</type>
         <name>hive.llap.daemon.queue.name</name>
       </property>
@@ -105,10 +75,6 @@
         <type>capacity-scheduler</type>
         <name>yarn.scheduler.capacity.root.queues</name>
       </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.server2.tez.sessions.per.default.queue</name>
-      </property>
     </depends-on>
     <on-ambari-upgrade add="true"/>
   </property>
@@ -140,7 +106,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -177,7 +143,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/8be71baf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 0207e49..2fb1553 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -67,7 +67,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -411,7 +411,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -447,7 +447,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -478,7 +478,7 @@ limitations under the License.
     <name>hive.llap.daemon.yarn.container.mb</name>
     <display-name>Memory per daemon</display-name>
     <description>Total memory used by individual LLAP daemons. This includes memory
for the cache as well as for the query execution.</description>
-    <value>341</value>
+    <value>0</value>
     <value-attributes>
       <type>int</type>
       <unit>MB</unit>
@@ -491,7 +491,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -646,4 +646,62 @@ limitations under the License.
     </value-attributes>
     <on-ambari-upgrade add="false"/>
   </property>
+  <property>
+    <name>hive.tez.container.size</name>
+    <value>SET_ON_FIRST_INVOCATION</value>
+    <depends-on>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>hive.auto.convert.join.noconditionaltask.size</name>
+    <value>1000000000</value>
+    <description>If hive.auto.convert.join.noconditionaltask is off, this parameter
does not take affect. However, if it
+      is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller
than this size, the join is directly
+      converted to a mapjoin(there is no conditional task). The default is 10MB.
+    </description>
+    <depends-on>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/8be71baf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
index 9e588e9..b331736 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
@@ -83,11 +83,8 @@
   </property>
   <property>
     <name>tez.am.resource.memory.mb</name>
-    <value>1536</value>
+    <value>0</value>
     <description>The amount of memory to be used by the AppMaster</description>
-    <value-attributes>
-      <type>int</type>
-    </value-attributes>
     <depends-on>
       <property>
         <type>hive-interactive-env</type>
@@ -95,7 +92,71 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.runtime.io.sort.mb</name>
+    <value>512</value>
+    <description>The size of the sort buffer when output needs to be sorted</description>
+    <depends-on>
+      <property>
+        <type>tez-site</type>
+        <name>tez.task.resource.memory.mb</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.runtime.unordered.output.buffer.size-mb</name>
+    <value>100</value>
+    <description>The size of the buffer when output does not require to be sorted</description>
+    <depends-on>
+      <property>
+        <type>tez-site</type>
+        <name>tez.task.resource.memory.mb</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/8be71baf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
index 452537d..1d50b6a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
@@ -63,11 +63,7 @@
           ]
         },
         {
-          "config": "hive-interactive-env/llap_queue_capacity",
-          "subsection-name": "interactive-query-row1-col1"
-        },
-        {
-          "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
+          "config": "hive-interactive-env/num_llap_nodes",
           "subsection-name": "interactive-query-row1-col1",
           "depends-on": [
             {
@@ -89,12 +85,8 @@
           ]
         },
         {
-          "config": "hive-interactive-env/copy_num_llap_nodes",
+          "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
           "subsection-name": "interactive-query-row1-col1",
-          "property_value_attributes": {
-            "ui_only_property": true,
-            "copy": "hive-interactive-env/num_llap_nodes"
-          },
           "depends-on": [
             {
               "configs":[
@@ -226,23 +218,12 @@
         }
       },
       {
-        "config": "hive-interactive-env/copy_num_llap_nodes",
-        "widget": {
-          "type": "label",
-          "units": [
-            {
-              "unit-name": "int"
-            }
-          ]
-        }
-      },
-      {
-        "config": "hive-interactive-env/llap_queue_capacity",
+        "config": "hive-interactive-env/num_llap_nodes",
         "widget": {
           "type": "slider",
           "units": [
             {
-              "unit-name": "percent"
+              "unit-name": "int"
             }
           ]
         }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8be71baf/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
index 9ff8484..bc0ecc4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
@@ -26,7 +26,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
     </depends-on>
     <on-ambari-upgrade add="false"/>


Mime
View raw message