ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject [1/9] ambari git commit: AMBARI-18238. HBase Master doesn't start after upgrading from HDP 2.3.6 using Ambari (dgrinenko via dlisnichenko)
Date Tue, 30 Aug 2016 01:43:35 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 5d8ae6207 -> c93946607


AMBARI-18238. HBase Master doesn't start after upgrading from HDP 2.3.6 using Ambari (dgrinenko
via dlisnichenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bbd170b7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bbd170b7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bbd170b7

Branch: refs/heads/branch-2.4
Commit: bbd170b7236189cc99762cf43755787188bed1f3
Parents: 5d8ae62
Author: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Authored: Tue Aug 23 19:26:09 2016 +0300
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Mon Aug 29 18:26:43 2016 -0700

----------------------------------------------------------------------
 .../HBaseEnvMaxDirectMemorySizeAction.java      |  87 +++++++++
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml |   4 +
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml |   4 +
 .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml     |   9 +
 .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml     |   9 +
 .../HBaseEnvMaxDirectMemorySizeActionTest.java  | 194 +++++++++++++++++++
 6 files changed, 307 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/bbd170b7/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java
new file mode 100644
index 0000000..b238bca
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeAction.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Inject;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Computes HBase Env content property.
+ * This class is only used when moving from HDP-2.3 to HDP-2.4 and HDP-2.3 to HDP-2.5
+ */
+public class HBaseEnvMaxDirectMemorySizeAction extends AbstractServerAction {
+  private static final String SOURCE_CONFIG_TYPE = "hbase-env";
+  private static final String CONTENT_NAME = "content";
+  private static final String APPEND_CONTENT_LINE = "export HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS
{% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m
{% endif %}\"";
+  private static final String CHECK_REGEX = "^.*\\s*(HBASE_MASTER_OPTS)\\s*=.*(XX:MaxDirectMemorySize).*$";
+  private static final Pattern REGEX = Pattern.compile(CHECK_REGEX, Pattern.MULTILINE);
+
+  @Inject
+  private Clusters clusters;
+
+  @Override
+  public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+    throws AmbariException, InterruptedException {
+
+    String clusterName = getExecutionCommand().getClusterName();
+    Cluster cluster = clusters.getCluster(clusterName);
+    Config config = cluster.getDesiredConfigByType(SOURCE_CONFIG_TYPE);
+
+    if (config == null) {
+      return  createCommandReport(0, HostRoleStatus.FAILED,"{}",
+                                   String.format("Source type %s not found", SOURCE_CONFIG_TYPE),
"");
+    }
+
+    Map<String, String> properties = config.getProperties();
+    String content = properties.get(CONTENT_NAME);
+
+
+    if (content == null) {
+      return  createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+        String.format("The %s/%s property was not found. The -XX:MaxDirectMemorySize will
not be added", SOURCE_CONFIG_TYPE, CONTENT_NAME), "");
+    }
+
+    Matcher m = REGEX.matcher(content);
+
+    if (m.find()){
+      return  createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+        String.format("The %s/%s property already contains an entry for %s and will not be
modified", SOURCE_CONFIG_TYPE, CONTENT_NAME, APPEND_CONTENT_LINE), "");
+    }
+
+    String appendedContent = content + "\n" + APPEND_CONTENT_LINE;
+    properties.put(CONTENT_NAME, appendedContent);
+
+    config.setProperties(properties);
+    config.persist(false);
+
+    return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+      String.format("The %s/%s property was appended with %s", SOURCE_CONFIG_TYPE, CONTENT_NAME,
APPEND_CONTENT_LINE),"");
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/bbd170b7/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index 450e350..eb6f333 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -273,6 +273,10 @@
         <task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/>
       </execute-stage>
 
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
+        <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/>
+      </execute-stage>
+
       <!-- TEZ -->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes
for Tez">
         <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/bbd170b7/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index b9067c7..f5f06c0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -299,6 +299,10 @@
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/>
       </execute-stage>
 
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
+        <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/>
+      </execute-stage>
+
       <!-- TEZ -->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes
for Tez">
         <task xsi:type="configure" id="hdp_2_5_0_0_tez_client_adjust_tez_lib_uris_property"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/bbd170b7/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 0e566c8..5ac0540 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -133,6 +133,15 @@
       </service>
     </group>
 
+    <group name="HBASE" title="Update HBase Configuration">
+      <skippable>true</skippable>
+
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
+        <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/>
+      </execute-stage>
+
+    </group>
+
     <group name="KAFKA" title="Kafka">
       <skippable>true</skippable>
       <service name="KAFKA">

http://git-wip-us.apache.org/repos/asf/ambari/blob/bbd170b7/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 7237a6a..09bd2ac 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -150,6 +150,15 @@
       </service>
     </group>
 
+    <group name="HBASE" title="Update HBase Configuration">
+      <skippable>true</skippable>
+
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
+        <task xsi:type="server_action" summary="Update HBase Env Configuration" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"/>
+      </execute-stage>
+
+    </group>
+
     <!-- This group should exist for all RUs that cross a major stack version. -->
     <group xsi:type="cluster" name="UPDATE_CLIENT_CONFIGS" title="Update Client Configs">
       <direction>UPGRADE</direction>

http://git-wip-us.apache.org/repos/asf/ambari/blob/bbd170b7/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java
new file mode 100644
index 0000000..4c1d7a3
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/HBaseEnvMaxDirectMemorySizeActionTest.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Injector;
+import junit.framework.Assert;
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigImpl;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests HiveEnvClasspathAction logic
+ */
+public class HBaseEnvMaxDirectMemorySizeActionTest {
+  private Injector injector;
+  private Clusters clusters;
+  private Field m_clusterField;
+
+  @Before
+  public void setup() throws Exception {
+    injector = EasyMock.createMock(Injector.class);
+    clusters = EasyMock.createMock(Clusters.class);
+    Cluster cluster = EasyMock.createMock(Cluster.class);
+
+    Config hbaseEnv = new ConfigImpl("hbase-env") {
+      Map<String, String> mockProperties = new HashMap<String, String>() {{
+        put("content","# Set environment variables here.\n" +
+          "\n" +
+          "# The java implementation to use. Java 1.6 required.\n" +
+          "export JAVA_HOME={{java64_home}}\n" +
+          "\n" +
+          "# HBase Configuration directory\n" +
+          "export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n" +
+          "\n" +
+          "# Extra Java CLASSPATH elements. Optional.\n" +
+          "export HBASE_CLASSPATH=${HBASE_CLASSPATH}\n" +
+          "\n" +
+          "# The maximum amount of heap to use, in MB. Default is 1000.\n" +
+          "# export HBASE_HEAPSIZE=1000\n" +
+          "\n" +
+          "# Extra Java runtime options.\n" +
+          "# Below are what we set by default. May only work with SUN JVM.\n" +
+          "# For more on why as well as other possible settings,\n" +
+          "# see http://wiki.apache.org/hadoop/PerformanceTuning\n" +
+          "export SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps
-Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n" +
+          "# Uncomment below to enable java garbage collection logging.\n" +
+          "# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps
-Xloggc:$HBASE_HOME/logs/gc-hbase.log -Djava.io.tmpdir={{java_io_tmpdir}}\"\n" +
+          "\n" +
+          "# Uncomment and adjust to enable JMX exporting\n" +
+          "# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure
remote password access.\n" +
+          "# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n"
+
+          "#\n" +
+          "# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n"
+
+          "# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with
proper direct memory size\n" +
+          "# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n"
+
+          "# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n"
+
+          "\n" +
+          "# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers
by default.\n" +
+          "export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n" +
+          "\n" +
+          "# Extra ssh options. Empty by default.\n" +
+          "# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n" +
+          "\n" +
+          "# Where log files are stored. $HBASE_HOME/logs by default.\n" +
+          "export HBASE_LOG_DIR={{log_dir}}\n" +
+          "\n" +
+          "# A string representing this instance of hbase. $USER by default.\n" +
+          "# export HBASE_IDENT_STRING=$USER\n" +
+          "\n" +
+          "# The scheduling priority for daemon processes. See 'man nice'.\n" +
+          "# export HBASE_NICENESS=10\n" +
+          "\n" +
+          "# The directory where pid files are stored. /tmp by default.\n" +
+          "export HBASE_PID_DIR={{pid_dir}}\n" +
+          "\n" +
+          "# Seconds to sleep between slave commands. Unset by default. This\n" +
+          "# can be useful in large clusters, where, e.g., slave rsyncs can\n" +
+          "# otherwise arrive faster than the master can service them.\n" +
+          "# export HBASE_SLAVE_SLEEP=0.1\n" +
+          "\n" +
+          "# Tell HBase whether it should manage it's own instance of Zookeeper or not.\n"
+
+          "export HBASE_MANAGES_ZK=false\n" +
+          "\n" +
+          "{% if security_enabled %}\n" +
+          "export HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log
-Djava.security.auth.login.config={{client_jaas_config_file}} -Djava.io.tmpdir={{java_io_tmpdir}}\"\n"
+
+          "export HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\n"
+
+          "export HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}}
-XX:CMSInitiatingOccupancyFraction=70  -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}
-Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n" +
+          "{% else %}\n" +
+          "export HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log
-Djava.io.tmpdir={{java_io_tmpdir}}\"\n" +
+          "export HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\n" +
+          "export HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}}
-XX:CMSInitiatingOccupancyFraction=70  -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n"
+
+          "{% endif %}");
+      }};
+      @Override
+      public Map<String, String> getProperties() {
+        return mockProperties;
+      }
+
+      @Override
+      public void setProperties(Map<String, String> properties) {
+        mockProperties.putAll(properties);
+      }
+
+      @Override
+      public void persist(boolean newConfig) {
+        // no-op
+      }
+    };
+
+
+    expect(cluster.getDesiredConfigByType("hbase-env")).andReturn(hbaseEnv).atLeastOnce();
+
+    expect(clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
+    expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce();
+
+    replay(injector, clusters, cluster);
+
+    m_clusterField = HBaseEnvMaxDirectMemorySizeAction.class.getDeclaredField("clusters");
+    m_clusterField.setAccessible(true);
+  }
+
+
+  @Test
+  public void testAction() throws Exception {
+    Pattern regex = Pattern.compile("^.*\\s*(HBASE_MASTER_OPTS)\\s*=.*(XX:MaxDirectMemorySize).*$",
Pattern.MULTILINE);
+    Map<String, String> commandParams = new HashMap<String, String>();
+    commandParams.put("clusterName", "c1");
+
+    ExecutionCommand executionCommand = new ExecutionCommand();
+    executionCommand.setCommandParams(commandParams);
+    executionCommand.setClusterName("c1");
+
+    HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+    expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+    expect(hrc.getStageId()).andReturn(2L).anyTimes();
+    expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+    replay(hrc);
+
+    HBaseEnvMaxDirectMemorySizeAction action = new HBaseEnvMaxDirectMemorySizeAction();
+
+    m_clusterField.set(action, clusters);
+
+    action.setExecutionCommand(executionCommand);
+    action.setHostRoleCommand(hrc);
+
+    CommandReport report = action.execute(null);
+    assertNotNull(report);
+
+    Cluster c = clusters.getCluster("c1");
+    Config config = c.getDesiredConfigByType("hbase-env");
+    Map<String, String> map = config.getProperties();
+
+    Assert.assertTrue(map.containsKey("content"));
+    String content = map.get("content");
+    Matcher m = regex.matcher(content);
+
+    assertEquals(true, m.find());
+  }
+
+}


Mime
View raw message