ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [18/50] [abbrv] ambari git commit: AMBARI-18792. Update som configuration properties for hive interactive for the HDP 2.6 stack (sseth via Swapan Shridhar)
Date Mon, 14 Nov 2016 18:27:31 GMT
AMBARI-18792. Update som configuration properties for hive interactive for the HDP 2.6 stack
(sseth via Swapan Shridhar)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/03f6398f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/03f6398f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/03f6398f

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 03f6398fcfa7d2e8accd72013d834815ba8755c5
Parents: 4264688
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Thu Nov 10 02:06:37 2016 -0800
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Thu Nov 10 02:06:37 2016 -0800

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog250.java       | 67 +++++++++++++++++---
 .../configuration/hive-interactive-site.xml     | 57 +++++++++++++++++
 .../HIVE/configuration/tez-interactive-site.xml | 47 ++++++++++++++
 3 files changed, 161 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/03f6398f/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index 3425dd7..e679ada 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -17,13 +17,8 @@
  */
 package org.apache.ambari.server.upgrade;
 
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
+import com.google.inject.Inject;
+import com.google.inject.Injector;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.CommandExecutionType;
 import org.apache.ambari.server.controller.AmbariManagementController;
@@ -37,8 +32,13 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.inject.Inject;
-import com.google.inject.Injector;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * Upgrade catalog for version 2.5.0.
@@ -120,7 +120,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
    */
   @Override
   protected void executePreDMLUpdates() throws AmbariException, SQLException {
-
+    updateHiveLlapConfigs();
   }
 
   /**
@@ -214,6 +214,53 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
     }
   }
 
+  protected void updateHiveLlapConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = clusters.getClusters();
+
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+          Set<String> installedServices = cluster.getServices().keySet();
+
+          if (installedServices.contains("HIVE")) {
+            Config hiveSite = cluster.getDesiredConfigByType("hive-interactive-site");
+            if (hiveSite != null) {
+              Map<String, String> hiveSiteProperties = hiveSite.getProperties();
+              String schedulerDelay = hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay");
+              if (schedulerDelay != null) {
+                // Property exists. Change to new default if set to -1.
+                if (schedulerDelay.length() != 0) {
+                  try {
+                    int schedulerDelayInt = Integer.parseInt(schedulerDelay);
+                    if (schedulerDelayInt == -1) {
+                      // Old default. Set to new default.
+                      updateConfigurationProperties("hive-interactive-site", Collections
+                              .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"),
true,
+                          false);
+                    }
+                  } catch (NumberFormatException e) {
+                    // Invalid existing value. Set to new default.
+                    updateConfigurationProperties("hive-interactive-site", Collections
+                            .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"),
true,
+                        false);
+                  }
+                }
+              }
+              updateConfigurationProperties("hive-interactive-site",
+                  Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"),
true,
+                  false);
+              // Explicitly skipping hive.llap.allow.permanent.fns during upgrades, since
it's related to security,
+              // and we don't know if the value is set by the user or as a result of the
previous default.
+            }
+          }
+        }
+      }
+    }
+  }
+
   /**
    * Creates the servicecomponent_version table
    * @throws SQLException

http://git-wip-us.apache.org/repos/asf/ambari/blob/03f6398f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
new file mode 100644
index 0000000..2c18bcc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration supports_final="true">
+  <!--
+  Note: This file includes only those configs which are supposed to have different value
from the parent hive/hive-site.
+       It inherits the other required configs from hive/hive-site.
+       The inheritance logic in order to get hive2/hive-site goes like this :
+
+       1. We read the hive/hive-site which includes the stack defaults and Stack Advisor
recommended values.
+       2. We take the read hive/hive-site (step 1), and on top of it apply the hive-interactive-site
to get
+          hive2/hive-site.
+
+       Therefore, any config defined here will override the config value read from hive2/hive-site
(step 1).
+  -->
+  <property>
+    <name>hive.llap.task.scheduler.locality.delay</name>
+    <value>8000</value>
+    <description>
+      Amount of time to wait (in ms) before allocating a request which contains location
information,
+      to a location other than the ones requested. Set to -1 for an infinite delay, 0
+      for no delay.
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+  <property>
+    <name>hive.mapjoin.hybridgrace.hashtable</name>
+    <value>true</value>
+    <description>Whether to use hybrid grace hash join as the join method for mapjoin.
Tez only.</description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+  <property>
+    <name>hive.llap.enable.grace.join.in.llap</name>
+    <value>false</value>
+    <description>Override if grace join should be allowed to run in llap for regular
map joins.
+      Dynamic partitioned joins will honor the hive.mapjoin.hybridgrace.hashtable property
in LLAP
+    </description>
+  </property>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/03f6398f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
new file mode 100644
index 0000000..2b80980
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<configuration supports_final="true">
+  <!--
+  Note: This file includes only those configs which are supposed to have different value
from the parent tez/tez-site.
+        It inherits the other required configs from tez/tez-site.
+        The inheritance logic in order to get tez_hive2/tez-site goes like this :
+
+        1. We read the tez/tez-site which includes the stack defaults and Stack Advisor recommended
values.
+        2. We take the read tez/tez-site (step 1), and on top of it apply the tez-interactive-site
to get tez_hive2/tez-site.
+
+        Therefore, any config defined here will override the config value read from tez/tez-site
(step 1).
+  -->
+  <property>
+    <name>tez.task.heartbeat.timeout.check-ms</name>
+    <value>11000</value>
+    <description>The time interval, in milliseconds, at which the AM will check
+      for timed out tasks
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.task.timeout-ms</name>
+    <value>20000</value>
+    <description>Amount of time the Tez AM waits before marking a task which
+      has not sent in a heartbeat, as timed out
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+</configuration>


Mime
View raw message