ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-13817 HDP-2.3 does not have the ability to set heapsize for hive components (dsen)
Date Wed, 11 Nov 2015 14:10:40 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 26669facf -> ee1751323


AMBARI-13817 HDP-2.3 does not have the ability to set heapsize for hive components (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ee175132
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ee175132
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ee175132

Branch: refs/heads/branch-2.1
Commit: ee17513235e41847c8b52f807429959b5e14dbd7
Parents: 26669fa
Author: Dmytro Sen <dsen@apache.org>
Authored: Wed Nov 11 16:10:30 2015 +0200
Committer: Dmytro Sen <dsen@apache.org>
Committed: Wed Nov 11 16:10:30 2015 +0200

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog213.java       | 66 +++++++++++++++-
 .../HIVE/0.12.0.2.0/configuration/hive-env.xml  |  6 +-
 .../services/HIVE/configuration/hive-env.xml    |  8 +-
 .../services/HIVE/configuration/hive-env.xml    | 10 ++-
 .../server/upgrade/UpgradeCatalog213Test.java   | 79 ++++++++++++++++++++
 5 files changed, 161 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ee175132/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
index 3365397..beffa9e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.regex.Matcher;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
@@ -92,6 +93,7 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
   private static final String HBASE_ENV_CONFIG = "hbase-env";
   private static final String FLUME_ENV_CONFIG = "flume-env";
   private static final String HIVE_SITE_CONFIG = "hive-site";
+  private static final String HIVE_ENV_CONFIG = "hive-env";
   private static final String RANGER_ENV_CONFIG = "ranger-env";
   private static final String ZOOKEEPER_LOG4J_CONFIG = "zookeeper-log4j";
   private static final String HADOOP_ENV_CONFIG = "hadoop-env";
@@ -225,7 +227,7 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
    */
   protected void executeStageDDLUpdates() throws SQLException {
     dbAccessor.addColumn(STAGE_TABLE,
-        new DBAccessor.DBColumnInfo("supports_auto_skip_failure", Integer.class, 1, 0, false));
+            new DBAccessor.DBColumnInfo("supports_auto_skip_failure", Integer.class, 1, 0,
false));
   }
 
   /**
@@ -791,6 +793,27 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
           updateConfigurationPropertiesForCluster(cluster, HIVE_SITE_CONFIG, updates, true,
false);
         }
       }
+      StackId stackId = cluster.getCurrentStackVersion();
+      boolean isStackNotLess23 = (stackId != null && stackId.getStackName().equals("HDP")
&&
+              VersionUtils.compareVersions(stackId.getStackVersion(), "2.3") >= 0);
+
+      Config hiveEnvConfig = cluster.getDesiredConfigByType(HIVE_ENV_CONFIG);
+      if (hiveEnvConfig != null) {
+        Map<String, String> hiveEnvProps = new HashMap<String, String>();
+        String content = hiveEnvConfig.getProperties().get(CONTENT_PROPERTY);
+        // For HDP-2.3 we need to add hive heap size management to content,
+        // for others we need to update content
+        if(content != null) {
+          if(isStackNotLess23) {
+            content = updateHiveEnvContentHDP23(content);
+          } else {
+            content = updateHiveEnvContent(content);
+          }
+          hiveEnvProps.put(CONTENT_PROPERTY, content);
+          updateConfigurationPropertiesForCluster(cluster, HIVE_ENV_CONFIG, hiveEnvProps,
true, true);
+        }
+      }
+
     }
   }
 
@@ -1031,6 +1054,47 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
     return content;
   }
 
+  protected String updateHiveEnvContent(String hiveEnvContent) {
+    if(hiveEnvContent == null) {
+      return null;
+    }
+    // There are two cases here
+    // We do not have "export HADOOP_CLIENT_OPTS" and we need to add it
+    // We have "export HADOOP_CLIENT_OPTS" with wrong order
+    String exportHadoopClientOpts = "(?s).*export\\s*HADOOP_CLIENT_OPTS.*";
+    if (hiveEnvContent.matches(exportHadoopClientOpts)) {
+      String oldHeapSizeRegex = "export\\s*HADOOP_CLIENT_OPTS=\"-Xmx\\$\\{HADOOP_HEAPSIZE\\}m\\s*\\$HADOOP_CLIENT_OPTS\"";
+      String newHeapSizeRegex = "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m\"";
+      return hiveEnvContent.replaceAll(oldHeapSizeRegex, Matcher.quoteReplacement(newHeapSizeRegex));
+    } else {
+      String oldHeapSizeRegex = "export\\s*HADOOP_HEAPSIZE\\s*=\\s*\"\\{\\{hive_heapsize\\}\\}\"\\.*\\n\\s*fi\\s*\\n";
+      String newHeapSizeRegex = "export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2
and Client\n" +
+              "fi\n" +
+              "\n" +
+              "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m\"";
+      return hiveEnvContent.replaceAll(oldHeapSizeRegex, Matcher.quoteReplacement(newHeapSizeRegex));
+    }
+  }
+
+  protected String updateHiveEnvContentHDP23(String hiveEnvContent) {
+    if(hiveEnvContent == null) {
+      return null;
+    }
+    String oldHeapSizeRegex = "# The heap size of the jvm stared by hive shell script can
be controlled via:\\s*\\n";
+    String newHeapSizeRegex = "# The heap size of the jvm stared by hive shell script can
be controlled via:\n" +
+            "\n" +
+            "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+            "  export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore\n"
+
+            "else\n" +
+            "  export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\n"
+
+            "fi\n" +
+            "\n" +
+            "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m\"\n"
+
+            "\n";
+    return hiveEnvContent.replaceFirst(oldHeapSizeRegex, Matcher.quoteReplacement(newHeapSizeRegex));
+  }
+
+
   private String memoryToIntMb(String memorySize) {
     if (memorySize == null) {
       return "0";

http://git-wip-us.apache.org/repos/asf/ambari/blob/ee175132/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
index 447993d..86165be 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
@@ -182,12 +182,12 @@
 # The heap size of the jvm stared by hive shell script can be controlled via:
 
 if [ "$SERVICE" = "metastore" ]; then
-  export HADOOP_HEAPSIZE="{{hive_metastore_heapsize}}"
+  export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
 else
-  export HADOOP_HEAPSIZE="{{hive_heapsize}}"
+  export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
 fi
 
-export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m"
 
 # Larger heap size may be required when running queries over large number of files or partitions.
 # By default hive shell scripts use a heap size of 256 (MB).  Larger heap size would also
be

http://git-wip-us.apache.org/repos/asf/ambari/blob/ee175132/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
index c331802..f84bc01 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
@@ -50,13 +50,15 @@
  fi
 
 # The heap size of the jvm stared by hive shell script can be controlled via:
-export SERVICE=$SERVICE
+
 if [ "$SERVICE" = "metastore" ]; then
-  export HADOOP_HEAPSIZE="{{hive_metastore_heapsize}}"
+  export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
 else
-  export HADOOP_HEAPSIZE="{{hive_heapsize}}"
+  export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
 fi
 
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m"
+
 # Larger heap size may be required when running queries over large number of files or partitions.
 # By default hive shell scripts use a heap size of 256 (MB).  Larger heap size would also
be
 # appropriate for hive server (hwi etc).

http://git-wip-us.apache.org/repos/asf/ambari/blob/ee175132/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
index 30141c6..77de518 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
@@ -37,6 +37,14 @@
 
 # The heap size of the jvm stared by hive shell script can be controlled via:
 
+if [ "$SERVICE" = "metastore" ]; then
+  export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
+else
+  export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
+fi
+
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m"
+
 # Larger heap size may be required when running queries over large number of files or partitions.
 # By default hive shell scripts use a heap size of 256 (MB).  Larger heap size would also
be
 # appropriate for hive server (hwi etc).
@@ -68,4 +76,4 @@ export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}"
     </value>
   </property>
 
-</configuration>
\ No newline at end of file
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/ee175132/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
index 68c5448..a03dfc7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
@@ -1170,7 +1170,11 @@ public class UpgradeCatalog213Test {
     final Map<String, String> propertiesHiveSiteExpected = new HashMap<String, String>()
{{
       put("hive.server2.logging.operation.log.location", "/tmp/hive/operation_logs");
     }};
+    final Map<String, String> propertiesHiveEnv = new HashMap<String, String>()
{{
+      put("content", "test content");
+    }};
     final Config hiveSiteConf = easyMockSupport.createNiceMock(Config.class);
+    final Config hiveEnvConf = easyMockSupport.createNiceMock(Config.class);
     final Injector mockInjector = Guice.createInjector(new AbstractModule() {
       @Override
       protected void configure() {
@@ -1187,8 +1191,10 @@ public class UpgradeCatalog213Test {
       put("normal", mockClusterExpected);
     }}).atLeastOnce();
     expect(mockClusterExpected.getDesiredConfigByType("hive-site")).andReturn(hiveSiteConf).atLeastOnce();
+    expect(mockClusterExpected.getDesiredConfigByType("hive-env")).andReturn(hiveEnvConf).atLeastOnce();
 
     expect(hiveSiteConf.getProperties()).andReturn(propertiesHiveSite).once();
+    expect(hiveEnvConf.getProperties()).andReturn(propertiesHiveEnv).once();
 
     UpgradeCatalog213 upgradeCatalog213 = createMockBuilder(UpgradeCatalog213.class)
             .withConstructor(Injector.class)
@@ -1199,6 +1205,9 @@ public class UpgradeCatalog213Test {
     upgradeCatalog213.updateConfigurationPropertiesForCluster(mockClusterExpected,
             "hive-site", propertiesHiveSiteExpected, true, false);
     expectLastCall().once();
+    upgradeCatalog213.updateConfigurationPropertiesForCluster(mockClusterExpected,
+            "hive-env", propertiesHiveEnv, true, true);
+    expectLastCall().once();
 
     easyMockSupport.replayAll();
     replay(upgradeCatalog213);
@@ -1206,4 +1215,74 @@ public class UpgradeCatalog213Test {
     easyMockSupport.verifyAll();
 
   }
+
+  @Test
+  public void testUpdateHiveEnvContentHDP23() throws Exception {
+    UpgradeCatalog213 upgradeCatalog213 = new UpgradeCatalog213(injector);
+    String testContent = "# The heap size of the jvm stared by hive shell script can be controlled
via:\n" +
+            "\n" +
+            "# Larger heap size may be required when running queries over large number of
files or partitions.\n";
+    String expectedResult = "# The heap size of the jvm stared by hive shell script can be
controlled via:\n" +
+            "\n" +
+            "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+            "  export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore\n"
+
+            "else\n" +
+            "  export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\n"
+
+            "fi\n" +
+            "\n" +
+            "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m\"\n"
+
+            "\n" +
+            "# Larger heap size may be required when running queries over large number of
files or partitions.\n";
+    Assert.assertEquals(expectedResult, upgradeCatalog213.updateHiveEnvContentHDP23(testContent));
+  }
+
+
+  @Test
+  public void testUpdateHiveEnvContent() throws Exception {
+    UpgradeCatalog213 upgradeCatalog213 = new UpgradeCatalog213(injector);
+    // Test first case
+    String testContent = "# The heap size of the jvm stared by hive shell script can be controlled
via:\n" +
+            "\n" +
+            "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+            "  export HADOOP_HEAPSIZE=\"{{hive_metastore_heapsize}}\"\n" +
+            "else\n" +
+            "  export HADOOP_HEAPSIZE=\"{{hive_heapsize}}\"\n" +
+            "fi\n" +
+            "\n" +
+            "export HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n"
+
+            "\n" +
+            "# Larger heap size may be required when running queries over large number of
files or partitions.\n";
+    String expectedResult = "# The heap size of the jvm stared by hive shell script can be
controlled via:\n" +
+            "\n" +
+            "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+            "  export HADOOP_HEAPSIZE=\"{{hive_metastore_heapsize}}\"\n" +
+            "else\n" +
+            "  export HADOOP_HEAPSIZE=\"{{hive_heapsize}}\"\n" +
+            "fi\n" +
+            "\n" +
+            "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m\"\n"
+
+            "\n" +
+            "# Larger heap size may be required when running queries over large number of
files or partitions.\n";
+    Assert.assertEquals(expectedResult, upgradeCatalog213.updateHiveEnvContent(testContent));
+    // Test second case
+    testContent = "# The heap size of the jvm stared by hive shell script can be controlled
via:\n" +
+            "export SERVICE=$SERVICE\n" +
+            "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+            "  export HADOOP_HEAPSIZE=\"{{hive_metastore_heapsize}}\"\n" +
+            "else\n" +
+            "  export HADOOP_HEAPSIZE=\"{{hive_heapsize}}\"\n" +
+            "fi\n" +
+            "\n";
+    expectedResult = "# The heap size of the jvm stared by hive shell script can be controlled
via:\n" +
+            "export SERVICE=$SERVICE\n" +
+            "if [ \"$SERVICE\" = \"metastore\" ]; then\n" +
+            "  export HADOOP_HEAPSIZE=\"{{hive_metastore_heapsize}}\"\n" +
+            "else\n" +
+            "  export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\n"
+
+            "fi\n" +
+            "\n" +
+            "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m\"";
+    Assert.assertEquals(expectedResult, upgradeCatalog213.updateHiveEnvContent(testContent));
+  }
+
 }


Mime
View raw message