ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject [2/2] ambari git commit: Revert "AMBARI-18954. Use 'Number of LLAP Nodes' selected as the driver for LLAP config calculations."
Date Mon, 21 Nov 2016 20:32:41 GMT
Revert "AMBARI-18954. Use 'Number of LLAP Nodes' selected as the driver for LLAP config calculations."

This reverts commit 1162778e36f5b4e5a0f0d912a431559b049684e1.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4fcc4938
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4fcc4938
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4fcc4938

Branch: refs/heads/branch-feature-AMBARI-18901
Commit: 4fcc493834d2cdf4ff7edbf7e402e856a8447e93
Parents: 8be71ba
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Mon Nov 21 12:28:44 2016 -0800
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Mon Nov 21 12:32:31 2016 -0800

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog250.java       |  71 +-
 .../HIVE/configuration/hive-interactive-env.xml |  42 +-
 .../configuration/hive-interactive-site.xml     |  68 +-
 .../HIVE/configuration/tez-interactive-site.xml |  71 +-
 .../HDP/2.5/services/HIVE/themes/theme.json     |  27 +-
 .../YARN/configuration/capacity-scheduler.xml   |   2 +-
 .../stacks/HDP/2.5/services/stack_advisor.py    | 665 ++++++++-----------
 .../server/upgrade/UpgradeCatalog250Test.java   | 287 +-------
 8 files changed, 379 insertions(+), 854 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index 6de48c1..bc3c120 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -17,8 +17,13 @@
  */
 package org.apache.ambari.server.upgrade;
 
-import com.google.inject.Inject;
-import com.google.inject.Injector;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.CommandExecutionType;
 import org.apache.ambari.server.controller.AmbariManagementController;
@@ -32,12 +37,8 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
 
 /**
  * Upgrade catalog for version 2.5.0.
@@ -133,8 +134,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
     addNewConfigurationsFromXml();
     updateAMSConfigs();
     updateKafkaConfigs();
-    updateHIVEInteractiveConfigs();
-    updateTEZInteractiveConfigs();
   }
 
   protected void updateHostVersionTable() throws SQLException {
@@ -270,57 +269,5 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
     dbAccessor.addColumn(SERVICE_DESIRED_STATE_TABLE,
       new DBColumnInfo(CREDENTIAL_STORE_ENABLED_COL, Short.class, null, 0, false));
   }
-
-  /**
-   * Updates Hive Interactive's config in hive-interactive-site.
-   *
-   * @throws AmbariException
-   */
-  protected void updateHIVEInteractiveConfigs() throws AmbariException {
-    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
-    Clusters clusters = ambariManagementController.getClusters();
-    if (clusters != null) {
-      Map<String, Cluster> clusterMap = clusters.getClusters();
-
-      if (clusterMap != null && !clusterMap.isEmpty()) {
-        for (final Cluster cluster : clusterMap.values()) {
-          Config hiveInteractiveSite = cluster.getDesiredConfigByType("hive-interactive-site");
-          if (hiveInteractiveSite != null) {
-            updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.tez.container.size",
-                "SET_ON_FIRST_INVOCATION"), true, true);
-
-            updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size",
-                "1000000000"), true, true);
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * Updates Tez for Hive2 Interactive's config in tez-interactive-site.
-   *
-   * @throws AmbariException
-   */
-  protected void updateTEZInteractiveConfigs() throws AmbariException {
-    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
-    Clusters clusters = ambariManagementController.getClusters();
-    if (clusters != null) {
-      Map<String, Cluster> clusterMap = clusters.getClusters();
-
-      if (clusterMap != null && !clusterMap.isEmpty()) {
-        for (final Cluster cluster : clusterMap.values()) {
-          Config tezInteractiveSite = cluster.getDesiredConfigByType("tez-interactive-site");
-          if (tezInteractiveSite != null) {
-
-            updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.io.sort.mb",
"512"), true, true);
-
-            updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.unordered.output.buffer.size-mb",
-                "100"), true, true);
-          }
-        }
-      }
-    }
-  }
 }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index 5a186b1..1fd72eb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -53,14 +53,40 @@
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
+    <name>llap_queue_capacity</name>
+    <value>0</value>
+    <description>Percentage of the cluster dedicated to interactive query.</description>
+    <display-name>% of Cluster Capacity</display-name>
+    <value-attributes>
+      <type>int</type>
+      <minimum>20</minimum>
+      <maximum>100</maximum>
+      <increment-step>1</increment-step>
+    </value-attributes>
+    <depends-on>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
     <name>num_llap_nodes</name>
     <value>1</value>
     <description>The number of Hive LLAP daemons to run.</description>
-    <display-name>Number of LLAP Nodes</display-name>
+    <display-name>Number of LLAP Daemons</display-name>
     <value-attributes>
       <type>int</type>
       <minimum>1</minimum>
-      <increment-step>1</increment-step>
     </value-attributes>
     <depends-on>
       <property>
@@ -68,6 +94,10 @@
         <name>enable_hive_interactive</name>
       </property>
       <property>
+        <type>hive-interactive-env</type>
+        <name>llap_queue_capacity</name>
+      </property>
+      <property>
         <type>hive-interactive-site</type>
         <name>hive.llap.daemon.queue.name</name>
       </property>
@@ -75,6 +105,10 @@
         <type>capacity-scheduler</type>
         <name>yarn.scheduler.capacity.root.queues</name>
       </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
     </depends-on>
     <on-ambari-upgrade add="true"/>
   </property>
@@ -106,7 +140,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
+        <name>llap_queue_capacity</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -143,7 +177,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
+        <name>llap_queue_capacity</name>
       </property>
       <property>
         <type>hive-interactive-site</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 2fb1553..0207e49 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -67,7 +67,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
+        <name>llap_queue_capacity</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -411,7 +411,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
+        <name>llap_queue_capacity</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -447,7 +447,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
+        <name>llap_queue_capacity</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -478,7 +478,7 @@ limitations under the License.
     <name>hive.llap.daemon.yarn.container.mb</name>
     <display-name>Memory per daemon</display-name>
     <description>Total memory used by individual LLAP daemons. This includes memory
for the cache as well as for the query execution.</description>
-    <value>0</value>
+    <value>341</value>
     <value-attributes>
       <type>int</type>
       <unit>MB</unit>
@@ -491,7 +491,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
+        <name>llap_queue_capacity</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -646,62 +646,4 @@ limitations under the License.
     </value-attributes>
     <on-ambari-upgrade add="false"/>
   </property>
-  <property>
-    <name>hive.tez.container.size</name>
-    <value>SET_ON_FIRST_INVOCATION</value>
-    <depends-on>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>enable_hive_interactive</name>
-      </property>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.llap.daemon.queue.name</name>
-      </property>
-      <property>
-        <type>capacity-scheduler</type>
-        <name>yarn.scheduler.capacity.root.queues</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.server2.tez.sessions.per.default.queue</name>
-      </property>
-    </depends-on>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
-    <name>hive.auto.convert.join.noconditionaltask.size</name>
-    <value>1000000000</value>
-    <description>If hive.auto.convert.join.noconditionaltask is off, this parameter
does not take affect. However, if it
-      is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller
than this size, the join is directly
-      converted to a mapjoin(there is no conditional task). The default is 10MB.
-    </description>
-    <depends-on>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>enable_hive_interactive</name>
-      </property>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.llap.daemon.queue.name</name>
-      </property>
-      <property>
-        <type>capacity-scheduler</type>
-        <name>yarn.scheduler.capacity.root.queues</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.server2.tez.sessions.per.default.queue</name>
-      </property>
-    </depends-on>
-    <on-ambari-upgrade add="true"/>
-  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
index b331736..9e588e9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
@@ -83,8 +83,11 @@
   </property>
   <property>
     <name>tez.am.resource.memory.mb</name>
-    <value>0</value>
+    <value>1536</value>
     <description>The amount of memory to be used by the AppMaster</description>
+    <value-attributes>
+      <type>int</type>
+    </value-attributes>
     <depends-on>
       <property>
         <type>hive-interactive-env</type>
@@ -92,71 +95,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.llap.daemon.queue.name</name>
-      </property>
-      <property>
-        <type>capacity-scheduler</type>
-        <name>yarn.scheduler.capacity.root.queues</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.server2.tez.sessions.per.default.queue</name>
-      </property>
-    </depends-on>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
-    <name>tez.runtime.io.sort.mb</name>
-    <value>512</value>
-    <description>The size of the sort buffer when output needs to be sorted</description>
-    <depends-on>
-      <property>
-        <type>tez-site</type>
-        <name>tez.task.resource.memory.mb</name>
-      </property>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>enable_hive_interactive</name>
-      </property>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.llap.daemon.queue.name</name>
-      </property>
-      <property>
-        <type>capacity-scheduler</type>
-        <name>yarn.scheduler.capacity.root.queues</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.server2.tez.sessions.per.default.queue</name>
-      </property>
-    </depends-on>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
-    <name>tez.runtime.unordered.output.buffer.size-mb</name>
-    <value>100</value>
-    <description>The size of the buffer when output does not require to be sorted</description>
-    <depends-on>
-      <property>
-        <type>tez-site</type>
-        <name>tez.task.resource.memory.mb</name>
-      </property>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>enable_hive_interactive</name>
-      </property>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
+        <name>llap_queue_capacity</name>
       </property>
       <property>
         <type>hive-interactive-site</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
index 1d50b6a..452537d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
@@ -63,7 +63,11 @@
           ]
         },
         {
-          "config": "hive-interactive-env/num_llap_nodes",
+          "config": "hive-interactive-env/llap_queue_capacity",
+          "subsection-name": "interactive-query-row1-col1"
+        },
+        {
+          "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
           "subsection-name": "interactive-query-row1-col1",
           "depends-on": [
             {
@@ -85,8 +89,12 @@
           ]
         },
         {
-          "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
+          "config": "hive-interactive-env/copy_num_llap_nodes",
           "subsection-name": "interactive-query-row1-col1",
+          "property_value_attributes": {
+            "ui_only_property": true,
+            "copy": "hive-interactive-env/num_llap_nodes"
+          },
           "depends-on": [
             {
               "configs":[
@@ -218,9 +226,9 @@
         }
       },
       {
-        "config": "hive-interactive-env/num_llap_nodes",
+        "config": "hive-interactive-env/copy_num_llap_nodes",
         "widget": {
-          "type": "slider",
+          "type": "label",
           "units": [
             {
               "unit-name": "int"
@@ -229,6 +237,17 @@
         }
       },
       {
+        "config": "hive-interactive-env/llap_queue_capacity",
+        "widget": {
+          "type": "slider",
+          "units": [
+            {
+              "unit-name": "percent"
+            }
+          ]
+        }
+      },
+      {
         "config": "hive-interactive-site/ambari.copy.hive.llap.daemon.yarn.container.mb",
         "widget": {
           "type": "label",

http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
index bc0ecc4..9ff8484 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
@@ -26,7 +26,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>num_llap_nodes</name>
+        <name>llap_queue_capacity</name>
       </property>
     </depends-on>
     <on-ambari-upgrade add="false"/>


Mime
View raw message