ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vbrodets...@apache.org
Subject ambari git commit: AMBARI-13388. Update hbase-env config without removing custom changes.(vbrodetskyi)
Date Tue, 13 Oct 2015 11:50:27 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 cbd3f97af -> ba40d1024


AMBARI-13388. Update hbase-env config without removing custom changes.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ba40d102
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ba40d102
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ba40d102

Branch: refs/heads/branch-2.1
Commit: ba40d102454573147d0860ac4065534c1dfe274c
Parents: cbd3f97
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Tue Oct 13 14:49:13 2015 +0300
Committer: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Committed: Tue Oct 13 14:49:13 2015 +0300

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog213.java       | 33 +++++++++--
 .../services/HBASE/configuration/hbase-env.xml  |  7 ++-
 .../services/HBASE/configuration/hbase-env.xml  |  9 ++-
 .../server/upgrade/UpgradeCatalog213Test.java   | 59 ++++++++++++++++++--
 4 files changed, 94 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ba40d102/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
index 1455f05..b60404e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
@@ -52,6 +52,7 @@ import org.apache.ambari.server.state.alert.SourceType;
 import org.apache.ambari.server.state.stack.upgrade.Direction;
 import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
 import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
+import org.apache.ambari.server.utils.VersionUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -61,6 +62,7 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.text.MessageFormat;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
@@ -73,6 +75,8 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
   private static final String STORM_SITE = "storm-site";
   private static final String AMS_ENV = "ams-env";
   private static final String AMS_HBASE_ENV = "ams-hbase-env";
+  private static final String HBASE_ENV_CONFIG = "hbase-env";
+  private static final String CONTENT_PROPERTY = "content";
 
   public static final String UPGRADE_PACKAGE_COL = "upgrade_package";
   public static final String UPGRADE_TYPE_COL = "upgrade_type";
@@ -147,8 +151,10 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
     executeStackUpgradeDDLUpdates();
     bootstrapRepoVersionForHDP21();
 
-    addMissingConfigs();
+    addNewConfigurationsFromXml();
+    updateStormConfigs();
     updateAMSConfigs();
+    updateHbaseEnvConfig();
     updateAlertDefinitions();
   }
 
@@ -546,10 +552,6 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
     return rootJson.toString();
   }
 
-  protected void addMissingConfigs() throws AmbariException {
-    updateStormConfigs();
-  }
-
   protected void updateStormConfigs() throws AmbariException {
     AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
     Clusters clusters = ambariManagementController.getClusters();
@@ -575,6 +577,27 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
     }
   }
 
+  protected void updateHbaseEnvConfig() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+
+    for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values())
{
+      StackId stackId = cluster.getCurrentStackVersion();
+      if (stackId != null && stackId.getStackName().equals("HDP") &&
+               VersionUtils.compareVersions(stackId.getStackVersion(), "2.2") >= 0) {
+        Config hbaseEnvConfig = cluster.getDesiredConfigByType(HBASE_ENV_CONFIG);
+        if (hbaseEnvConfig != null) {
+          String content = hbaseEnvConfig.getProperties().get(CONTENT_PROPERTY);
+          if (content != null && content.indexOf("MaxDirectMemorySize={{hbase_max_direct_memory_size}}m")
< 0) {
+            String newPartOfContent = "\n\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS
{% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m
{% endif %}\"\n\n";
+            content += newPartOfContent;
+            Map<String, String> updates = Collections.singletonMap(CONTENT_PROPERTY,
content);
+            updateConfigurationPropertiesForCluster(cluster, HBASE_ENV_CONFIG, updates, true,
false);
+          }
+        }
+      }
+    }
+  }
+
   protected void updateAMSConfigs() throws AmbariException {
     AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
     Clusters clusters = ambariManagementController.getClusters();

http://git-wip-us.apache.org/repos/asf/ambari/blob/ba40d102/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml
index 5f6b1bd..a68295e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-env.xml
@@ -162,12 +162,15 @@ export HBASE_MANAGES_ZK=false
 {% if security_enabled %}
 export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log
-Djava.security.auth.login.config={{client_jaas_config_file}}"
 export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70
 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} {% if hbase_max_direct_memory_size
%} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70
 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
 {% else %}
 export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log"
 export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70
 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} {% if hbase_max_direct_memory_size
%} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70
 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
 {% endif %}
+
+# HBase off-heap MaxDirectMemorySize
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS {% if hbase_max_direct_memory_size
%} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}"
     </value>
   </property>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/ba40d102/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml
index 80348dd..1ecad34 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/configuration/hbase-env.xml
@@ -86,18 +86,21 @@ export HBASE_MANAGES_ZK=false
 
 {% if java_version &lt; 8 %}
 JDK_DEPENDED_OPTS="-XX:PermSize=128m -XX:MaxPermSize=128m"
-{% endif %}      
+{% endif %}
       
 {% if security_enabled %}
 export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log
-Djava.security.auth.login.config={{client_jaas_config_file}}"
 export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}
$JDK_DEPENDED_OPTS"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70
 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} {% if hbase_max_direct_memory_size
%} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}
$JDK_DEPENDED_OPTS"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70
 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}
$JDK_DEPENDED_OPTS"
 export PHOENIX_QUERYSERVER_OPTS="$PHOENIX_QUERYSERVER_OPTS -Djava.security.auth.login.config={{queryserver_jaas_config_file}}"
 {% else %}
 export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log"
 export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} $JDK_DEPENDED_OPTS"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70
 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} {% if hbase_max_direct_memory_size
%} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %} $JDK_DEPENDED_OPTS"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70
 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} $JDK_DEPENDED_OPTS"
 {% endif %}
+
+# HBase off-heap MaxDirectMemorySize
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS {% if hbase_max_direct_memory_size
%} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}"
     </value>
   </property>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/ba40d102/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
index 1be1446..d04e4c9 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
@@ -167,25 +167,34 @@ public class UpgradeCatalog213Test {
     f.set(upgradeCatalog, configuration);
     */
 
-    Method addMissingConfigs = UpgradeCatalog213.class.getDeclaredMethod("addMissingConfigs");
-    Method updateAMSConfigs = UpgradeCatalog213.class.getDeclaredMethod("updateAMSConfigs");
     Method updateAlertDefinitions = UpgradeCatalog213.class.getDeclaredMethod("updateAlertDefinitions");
     Method executeStackUpgradeDDLUpdates = UpgradeCatalog213.class.getDeclaredMethod("executeStackUpgradeDDLUpdates");
     Method bootstrapRepoVersionForHDP21 = UpgradeCatalog213.class.getDeclaredMethod("bootstrapRepoVersionForHDP21");
+    Method updateStormConfigs = UpgradeCatalog213.class.getDeclaredMethod("updateStormConfigs");
+    Method updateAMSConfigs = UpgradeCatalog213.class.getDeclaredMethod("updateAMSConfigs");
+    Method updateHbaseEnvConfig = UpgradeCatalog213.class.getDeclaredMethod("updateHbaseEnvConfig");
+    Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml");
+
 
     UpgradeCatalog213 upgradeCatalog213 = createMockBuilder(UpgradeCatalog213.class)
-        .addMockedMethod(addMissingConfigs)
+        .addMockedMethod(addNewConfigurationsFromXml)
         .addMockedMethod(updateAMSConfigs)
         .addMockedMethod(updateAlertDefinitions)
         .addMockedMethod(executeStackUpgradeDDLUpdates)
         .addMockedMethod(bootstrapRepoVersionForHDP21)
+        .addMockedMethod(updateStormConfigs)
+        .addMockedMethod(updateHbaseEnvConfig)
         .createMock();
 
+    upgradeCatalog213.addNewConfigurationsFromXml();
+    expectLastCall().once();
     upgradeCatalog213.executeStackUpgradeDDLUpdates();
     expectLastCall().once();
     upgradeCatalog213.bootstrapRepoVersionForHDP21();
     expectLastCall().once();
-    upgradeCatalog213.addMissingConfigs();
+    upgradeCatalog213.updateStormConfigs();
+    expectLastCall().once();
+    upgradeCatalog213.updateHbaseEnvConfig();
     expectLastCall().once();
     upgradeCatalog213.updateAMSConfigs();
     expectLastCall().once();
@@ -204,6 +213,48 @@ public class UpgradeCatalog213Test {
     //upgradeSectionDDL.verify(dbAccessor);
   }
 
+  @Test
+  public void testUpdateHbaseEnvConfig() throws AmbariException {
+    EasyMockSupport easyMockSupport = new EasyMockSupport();
+    final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);
+    final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class);
+    final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class);
+    final Map<String, String> propertiesHbaseEnv = new HashMap<String, String>()
{
+      {
+        put("content", "test");
+      }
+    };
+
+    final Config mockHbaseEnv = easyMockSupport.createNiceMock(Config.class);
+    expect(mockHbaseEnv.getProperties()).andReturn(propertiesHbaseEnv).once();
+
+    final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+      @Override
+      protected void configure() {
+        bind(AmbariManagementController.class).toInstance(mockAmbariManagementController);
+        bind(Clusters.class).toInstance(mockClusters);
+        bind(EntityManager.class).toInstance(entityManager);
+
+        bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+        bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+      }
+    });
+
+    expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once();
+    expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+      put("normal", mockClusterExpected);
+    }}).atLeastOnce();
+    expect(mockClusterExpected.getCurrentStackVersion()).andReturn(new StackId("HDP","2.2"));
+
+    expect(mockClusterExpected.getDesiredConfigByType("hbase-env")).andReturn(mockHbaseEnv).atLeastOnce();
+    expect(mockHbaseEnv.getProperties()).andReturn(propertiesHbaseEnv).atLeastOnce();
+
+    easyMockSupport.replayAll();
+    mockInjector.getInstance(UpgradeCatalog213.class).updateHbaseEnvConfig();
+    easyMockSupport.verifyAll();
+
+  }
+
   /**
    * Verify that when bootstrapping HDP 2.1, records get inserted into the
    * repo_version, cluster_version, and host_version tables.


Mime
View raw message