ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From abaranc...@apache.org
Subject ambari git commit: AMBARI-10714 - Separate memory configuration for Hive CLI vs HiveServer2 (metastore part)
Date Tue, 19 May 2015 11:24:59 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk fddbaf4dd -> 211c78bbb


AMBARI-10714 - Separate memory configuration for Hive CLI vs HiveServer2 (metastore part)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/211c78bb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/211c78bb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/211c78bb

Branch: refs/heads/trunk
Commit: 211c78bbbae794a0c5d7fdb04327bf2960bb96ea
Parents: fddbaf4
Author: Artem Baranchuk <abaranchuk@hortonworks.con>
Authored: Mon May 18 17:31:37 2015 +0300
Committer: Artem Baranchuk <abaranchuk@hortonworks.con>
Committed: Tue May 19 14:24:23 2015 +0300

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog210.java       | 33 ++++++++++++++++++++
 .../HIVE/0.12.0.2.0/configuration/hive-env.xml  | 13 +++++++-
 .../0.12.0.2.0/package/scripts/params_linux.py  |  2 ++
 .../services/HIVE/configuration/hive-env.xml    |  5 +++
 .../server/upgrade/UpgradeCatalog210Test.java   |  9 +++++-
 5 files changed, 60 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
index b10d3a5..69c82ef 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
@@ -873,6 +873,39 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
 
     // Initialize all default widgets and widget layouts
     initializeClusterAndServiceWidgets();
+
+    addMissingConfigs();
+  }
+
+  protected void addMissingConfigs() throws AmbariException {
+    updateHiveConfigs();
+  }
+
+  protected void updateHiveConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(
+            AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = clusters.getClusters();
+      Map<String, String> prop = new HashMap<String, String>();
+
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+          //hive metastore and client_heapsize are added for HDP2, we should check if it
exists and not add it for HDP1
+          if(cluster.getDesiredConfigByType("hive-env") != null) {
+            Map<String, String> hiveProps = new HashMap<String, String>();
+            if (!cluster.getDesiredConfigByType("hive-env").getProperties().containsKey("hive.client.heapsize"))
{
+              hiveProps.put("hive.client.heapsize", "512m");
+            }
+            if (!cluster.getDesiredConfigByType("hive-env").getProperties().containsKey("hive.metastore.heapsize"))
{
+              hiveProps.put("hive.metastore.heapsize", "1024m");
+            }
+            updateConfigurationPropertiesForCluster(cluster, "hive-env", hiveProps, false,
true);
+          }
+        }
+      }
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
index 923a9ba..db2c927 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
@@ -28,6 +28,12 @@
   </property>
 
   <property>
+    <name>hive.metastore.heapsize</name>
+    <value>1024</value>
+    <description>Hive Metastore Java heap size</description>
+  </property>
+
+  <property>
     <name>hive_database_type</name>
     <value>mysql</value>
     <description>Default HIVE DB type.</description>
@@ -106,7 +112,12 @@
 
 # The heap size of the jvm stared by hive shell script can be controlled via:
 
-export HADOOP_HEAPSIZE="{{hive_heapsize}}"
+if [ "$SERVICE" = "metastore" ]; then
+  export HADOOP_HEAPSIZE="{{hive_metastore_heapsize}}"
+else
+  export HADOOP_HEAPSIZE="{{hive_heapsize}}"
+fi
+
 export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"
 
 # Larger heap size may be required when running queries over large number of files or partitions.

http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 7254f3c..4396401 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -253,6 +253,8 @@ if 'role' in config and config['role'] in ["HIVE_SERVER", "HIVE_METASTORE"]:
 else:
   hive_heapsize = config['configurations']['hive-env']['hive.client.heapsize']
 
+hive_metastore_heapsize = config['configurations']['hive-env']['hive.metastore.heapsize']
+
 java64_home = config['hostLevelParams']['java_home']
 
 ##### MYSQL

http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
index f542bda..7d1aa2d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
@@ -26,6 +26,11 @@
     <deleted>true</deleted>
   </property>
 
+  <property>
+    <name>hive.metastore.heapsize</name>
+    <deleted>true</deleted>
+  </property>
+
   <!-- hive-env.sh -->
   <property>
     <name>content</name>

http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
index 13435e7..ac85f0d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
@@ -151,9 +151,13 @@ public class UpgradeCatalog210Test {
     Method initializeClusterAndServiceWidgets =
       UpgradeCatalog210.class.getDeclaredMethod("initializeClusterAndServiceWidgets");
 
+    Method addMissingConfigs =
+            UpgradeCatalog210.class.getDeclaredMethod("addMissingConfigs");
+
     UpgradeCatalog210 upgradeCatalog210 = createMockBuilder(UpgradeCatalog210.class)
       .addMockedMethod(addNewConfigurationsFromXml)
-      .addMockedMethod(initializeClusterAndServiceWidgets).createMock();
+      .addMockedMethod(initializeClusterAndServiceWidgets)
+      .addMockedMethod(addMissingConfigs).createMock();
 
     upgradeCatalog210.addNewConfigurationsFromXml();
     expectLastCall().once();
@@ -161,6 +165,9 @@ public class UpgradeCatalog210Test {
     upgradeCatalog210.initializeClusterAndServiceWidgets();
     expectLastCall().once();
 
+    upgradeCatalog210.addMissingConfigs();
+    expectLastCall().once();
+
     replay(upgradeCatalog210);
 
     upgradeCatalog210.executeDMLUpdates();


Mime
View raw message