ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-19933. HSI is not started with configured Hiveserver2 heap size, upgrade changes (smohanty)
Date Thu, 09 Feb 2017 19:39:57 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 259f31ae1 -> 6addaf50e


AMBARI-19933. HSI is not started with configured Hiveserver2 heap size, upgrade changes (smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6addaf50
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6addaf50
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6addaf50

Branch: refs/heads/trunk
Commit: 6addaf50e27d42d0da119cd006ae28f3dd1e678f
Parents: 259f31a
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Thu Feb 9 10:55:18 2017 -0800
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Thu Feb 9 11:39:43 2017 -0800

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog250.java       | 28 ++++++--
 .../server/upgrade/UpgradeCatalog250Test.java   | 69 ++++++++++++++++++++
 2 files changed, 90 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6addaf50/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index b617dc4..2082048 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -686,6 +686,8 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
    * @throws AmbariException
    */
   private static final String HIVE_INTERACTIVE_SITE = "hive-interactive-site";
+  private static final String HIVE_INTERACTIVE_ENV = "hive-interactive-env";
+  private static final String HIVE_ENV = "hive-env";
   protected void updateHIVEInteractiveConfigs() throws AmbariException {
     AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
     Clusters clusters = ambariManagementController.getClusters();
@@ -696,24 +698,36 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
         for (final Cluster cluster : clusterMap.values()) {
           Config hiveInteractiveSite = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_SITE);
           if (hiveInteractiveSite != null) {
-            updateConfigurationProperties(HIVE_INTERACTIVE_SITE, Collections.singletonMap("hive.tez.container.size",
-                "SET_ON_FIRST_INVOCATION"), true, true);
+            Map<String, String> newProperties = new HashMap<>();
+            newProperties.put("hive.auto.convert.join.noconditionaltask.size", "1000000000");
 
-            updateConfigurationProperties(HIVE_INTERACTIVE_SITE, Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size",
-                "1000000000"), true, true);
             String llapRpcPortString = hiveInteractiveSite.getProperties().get("hive.llap.daemon.rpc.port");
             if (StringUtils.isNotBlank(llapRpcPortString)) {
               try {
                 int llapRpcPort = Integer.parseInt(llapRpcPortString);
                 if (llapRpcPort == 15001) {
-                  updateConfigurationProperties(HIVE_INTERACTIVE_SITE,
-                      Collections.singletonMap("hive.llap.daemon.rpc.port", "0"),
-                      true, true);
+                  newProperties.put("hive.llap.daemon.rpc.port", "0");
+                  LOG.info("Updating HSI hive.llap.daemon.rpc.port to: 0");
                 }
               } catch (NumberFormatException e) {
                 LOG.warn("Unable to parse llap.rpc.port as integer: " + llapRpcPortString);
               }
             }
+            updateConfigurationProperties(HIVE_INTERACTIVE_SITE, newProperties, true, true);
+          }
+
+          Config hiveInteractiveEnv = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_ENV);
+          Config hiveEnv = cluster.getDesiredConfigByType(HIVE_ENV);
+          if (hiveInteractiveEnv != null) {
+            String hsiHeapSize = "512";
+            if (hiveEnv != null) {
+              if (hiveEnv.getProperties().containsKey("hive.heapsize")) {
+                hsiHeapSize = hiveEnv.getProperties().get("hive.heapsize");
+                LOG.info("Updating HSI heap size to: " + hsiHeapSize);
+              }
+            }
+            updateConfigurationProperties(HIVE_INTERACTIVE_ENV, Collections.singletonMap("hive_heapsize",
+                                                                                        
hsiHeapSize), true, true);
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/ambari/blob/6addaf50/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
index 7084769..09d56dc 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
@@ -1428,6 +1428,75 @@ public class UpgradeCatalog250Test {
   }
 
   @Test
+  public void testUpdateHiveConfigs() throws Exception {
+    reset(clusters, cluster);
+    expect(clusters.getClusters()).andReturn(ImmutableMap.of("normal", cluster)).anyTimes();
+
+    EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+    Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
+    AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+        .addMockedMethod("createConfiguration")
+        .addMockedMethod("getClusters", new Class[] {})
+        .addMockedMethod("createConfig")
+        .withConstructor(actionManager, clusters, injector)
+        .createNiceMock();
+
+    expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+    expect(controller.getClusters()).andReturn(clusters).anyTimes();
+
+    Map<String, String> oldHsiEnv = ImmutableMap.of(
+        "llap_app_name", "llap0");
+
+    Map<String, String> expectedHsiEnv = ImmutableMap.of(
+        "llap_app_name", "llap0",
+        "hive_heapsize", "1082");
+
+    Map<String, String> oldHiveEnv = ImmutableMap.of(
+        "hive.client.heapsize", "1024",
+        "hive.heapsize", "1082",
+        "hive.metastore.heapsize", "512",
+        "hive_ambari_database", "MySQL");
+
+    Map<String, String> oldHiveIntSite = ImmutableMap.of(
+        "hive.llap.daemon.rpc.port","15001");
+
+    Map<String, String> expectedHiveIntSite = ImmutableMap.of(
+        "hive.llap.daemon.rpc.port","0",
+        "hive.auto.convert.join.noconditionaltask.size", "1000000000");
+
+    Config mockHsiSite = easyMockSupport.createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("hive-interactive-site")).andReturn(mockHsiSite).atLeastOnce();
+    expect(mockHsiSite.getProperties()).andReturn(oldHiveIntSite).anyTimes();
+    Capture<Map<String, String>> hsiSiteCapture = EasyMock.newCapture();
+    expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(hsiSiteCapture),
anyString(),
+                                   EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
+
+    Config mockHiveEnv = easyMockSupport.createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("hive-env")).andReturn(mockHiveEnv).atLeastOnce();
+    expect(mockHiveEnv.getProperties()).andReturn(oldHiveEnv).anyTimes();
+
+    Config mockHsiEnv = easyMockSupport.createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("hive-interactive-env")).andReturn(mockHsiEnv).atLeastOnce();
+    expect(mockHsiEnv.getProperties()).andReturn(oldHsiEnv).anyTimes();
+    Capture<Map<String, String>> hsiEnvCapture = EasyMock.newCapture();
+    expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(hsiEnvCapture),
anyString(),
+                                   EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
+
+    replay(clusters, cluster);
+    replay(controller, injector2);
+    replay(mockHsiEnv, mockHiveEnv, mockHsiSite);
+    new UpgradeCatalog250(injector2).updateHIVEInteractiveConfigs();
+    easyMockSupport.verifyAll();
+
+    Map<String, String> updatedHsiSite = hsiSiteCapture.getValue();
+    assertTrue(Maps.difference(expectedHiveIntSite, updatedHsiSite).areEqual());
+
+    Map<String, String> updatedHsiEnv = hsiEnvCapture.getValue();
+    assertTrue(Maps.difference(expectedHsiEnv, updatedHsiEnv).areEqual());
+  }
+
+  @Test
   public void testUpdateAtlasConfigs() throws Exception {
 
     Map<String, String> oldHiveProperties = new HashMap<String, String>();


Mime
View raw message