ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject ambari git commit: AMBARI-9051. Rolling Upgrade: fix HBase tasks that changed definition (ncole)
Date Fri, 09 Jan 2015 01:17:20 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 5350550c5 -> 942706041


AMBARI-9051. Rolling Upgrade: fix HBase tasks that changed definition (ncole)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/94270604
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/94270604
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/94270604

Branch: refs/heads/trunk
Commit: 942706041919a82d4c2752969a23e4cb3c7f97d9
Parents: 5350550
Author: Nate Cole <ncole@hortonworks.com>
Authored: Thu Jan 8 15:58:10 2015 -0500
Committer: Nate Cole <ncole@hortonworks.com>
Committed: Thu Jan 8 20:16:46 2015 -0500

----------------------------------------------------------------------
 .../0.96.0.2.0/package/scripts/hbase_upgrade.py |  37 ++++
 .../stacks/HDP/2.2/upgrades/upgrade-2.2.xml     |   5 +-
 .../stacks/2.0.6/HBASE/test_hbase_master.py     |  15 +-
 .../stacks/2.0.6/configs/hbase-preupgrade.json  | 194 +++++++++++++++++++
 4 files changed, 249 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/94270604/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py
new file mode 100644
index 0000000..610f527
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management.libraries.script import Script
+from resource_management.core.resources.system import Execute
+
+class HbaseMasterUpgrade(Script):
+
+  def snapshot(self, env):
+    import params
+
+    snap_cmd = "echo 'snapshot_all' | {0} shell".format(params.hbase_cmd)
+
+    exec_cmd = "{0} {1}".format(params.kinit_cmd, snap_cmd)
+
+    Execute(exec_cmd, user=params.hbase_user)
+
+if __name__ == "__main__":
+  HbaseMasterUpgrade().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/94270604/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
index 4549da0..ed6b50c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
@@ -85,6 +85,7 @@
     </group>
 
     <group name="KAFKA" title="Kafka">
+      <skippable>true</skippable>
       <service name="KAFKA">
         <component>KAFKA_BROKER</component>
       </service>
@@ -122,6 +123,7 @@
     </group>
 
     <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize Upgrade">
+      <skippable>true</skippable>
       <execute-stage title="Confirm Finalize">
         <task xsi:type="manual">
           <message>Please confirm you are ready to finalize</message>
@@ -256,7 +258,8 @@
       <component name="HBASE_MASTER">
         <pre-upgrade>
           <task xsi:type="execute" hosts="master">
-            <command>su - {{hbase-env/hbase_user}} -c 'echo "snapshot_all" | hbase
shell'</command>
+            <script>scripts/hbase_upgrade.py</script>
+            <function>snapshot</function>
           </task>
         </pre-upgrade>
         <upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/94270604/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
index 0d04acf..c350a27 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
@@ -641,4 +641,17 @@ class TestHBaseMaster(RMFTestCase):
                    hdp_stack_version = self.STACK_VERSION,
                    target = RMFTestCase.TARGET_COMMON_SERVICES
     )
-    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
\ No newline at end of file
+    put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+  def test_upgrade_backup(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_upgrade.py",
+                   classname = "HbaseMasterUpgrade",
+                   command = "snapshot",
+                   config_file="hbase-preupgrade.json",
+                   hdp_stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES)
+
+    self.assertResourceCalled('Execute', " echo 'snapshot_all' | /usr/hdp/current/hbase-client/bin/hbase
shell",
+      user = 'hbase')
+  
+    self.assertNoMoreResources()

http://git-wip-us.apache.org/repos/asf/ambari/blob/94270604/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-preupgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-preupgrade.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-preupgrade.json
new file mode 100644
index 0000000..ee46527
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-preupgrade.json
@@ -0,0 +1,194 @@
+{
+    "configuration_attributes": {}, 
+    "roleCommand": "ACTIONEXECUTE", 
+    "kerberosCommandParams": [], 
+    "clusterName": "c1", 
+    "componentName": "", 
+    "hostname": "c6401.ambari.apache.org", 
+    "hostLevelParams": {
+        "jdk_location": "http://10.0.0.15:8080/resources/", 
+        "ambari_db_rca_password": "mapred", 
+        "java_home": "/usr/jdk64/jdk1.7.0_45", 
+        "ambari_db_rca_url": "jdbc:postgresql://10.0.0.15/ambarirca", 
+        "jce_name": "UnlimitedJCEPolicyJDK7.zip", 
+        "oracle_jdbc_url": "http://10.0.0.15:8080/resources//ojdbc6.jar", 
+        "stack_version": "2.2", 
+        "stack_name": "HDP", 
+        "db_name": "ambari", 
+        "ambari_db_rca_driver": "org.postgresql.Driver", 
+        "jdk_name": "jdk-7u45-linux-x64.tar.gz", 
+        "ambari_db_rca_username": "mapred", 
+        "db_driver_filename": "mysql-connector-java.jar", 
+        "agentCacheDir": "/var/lib/ambari-agent/cache", 
+        "mysql_jdbc_url": "http://10.0.0.15:8080/resources//mysql-connector-java.jar"
+    }, 
+    "commandType": "EXECUTION_COMMAND", 
+    "roleParams": {
+        "tasks1": "[{\"type\":\"EXECUTE\",\"hosts\":\"master\",\"command\":\"su - hbase -c
\\u0027echo \\\"snapshot_all\\\" | hbase shell\\u0027\"}]", 
+        "tasks": "[{\"type\":\"EXECUTE\",\"hosts\":\"master\",\"script\":\"scripts/hbase_upgrade.py\",\"function\":
\"snapshot\"}]", 
+        "service_package_folder": "common-services/HBASE/0.96.0.2.0/package", 
+        "version": "2.2.0.0-2041", 
+        "hooks_folder": "HDP/2.0.6/hooks"
+    }, 
+    "serviceName": "", 
+    "role": "ru_execute_tasks", 
+    "forceRefreshConfigTags": [], 
+    "taskId": 402, 
+    "public_hostname": "c6401.ambari.apache.org", 
+    "configurations": {
+        "hbase-site": {
+            "hbase.master.info.bindAddress": "0.0.0.0", 
+            "hbase.client.keyvalue.maxsize": "10485760", 
+            "hbase.hstore.compactionThreshold": "3", 
+            "hbase.hregion.majorcompaction.jitter": "0.50", 
+            "hbase.rootdir": "hdfs://c6401.ambari.apache.org:8020/apps/hbase/data", 
+            "hbase.regionserver.handler.count": "60", 
+            "hbase.hregion.majorcompaction": "604800000", 
+            "hbase.hregion.memstore.block.multiplier": "4", 
+            "hbase.hregion.memstore.flush.size": "134217728", 
+            "hbase.superuser": "hbase", 
+            "hbase.regionserver.global.memstore.lowerLimit": "0.38", 
+            "hbase.zookeeper.property.clientPort": "2181", 
+            "hbase.regionserver.global.memstore.upperLimit": "0.4", 
+            "zookeeper.session.timeout": "30000", 
+            "hbase.client.scanner.caching": "100", 
+            "hbase.tmp.dir": "/hadoop/hbase", 
+            "hfile.block.cache.size": "0.40", 
+            "hbase.hregion.max.filesize": "10737418240", 
+            "hbase.security.authentication": "simple", 
+            "hbase.defaults.for.version.skip": "true", 
+            "hbase.master.info.port": "60010", 
+            "hbase.zookeeper.quorum": "c6403.ambari.apache.org,c6401.ambari.apache.org,c6402.ambari.apache.org",

+            "hbase.regionserver.info.port": "60030", 
+            "zookeeper.znode.parent": "/hbase-unsecure", 
+            "hbase.zookeeper.useMulti": "true", 
+            "hbase.hstore.blockingStoreFiles": "10", 
+            "hbase.master.port": "60000", 
+            "hbase.security.authorization": "false", 
+            "hbase.local.dir": "${hbase.tmp.dir}/local", 
+            "hbase.cluster.distributed": "true", 
+            "hbase.hregion.memstore.mslab.enabled": "true", 
+            "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket", 
+            "hbase.coprocessor.region.classes": "", 
+            "hbase.coprocessor.master.classes": ""
+        }, 
+        "hbase-env": {
+            "hbase_pid_dir": "/var/run/hbase", 
+            "hbase_regionserver_xmn_max": "512", 
+            "hbase_regionserver_xmn_ratio": "0.2", 
+            "hbase_user": "hbase", 
+            "hbase_master_heapsize": "1024m", 
+            "content": "\n# Set environment variables here.\n\n# The java implementation
to use. Java 1.6 required.\nexport JAVA_HOME={{java64_home}}\n\n# HBase Configuration directory\nexport
HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n\n# Extra Java CLASSPATH elements. Optional.\nexport
HBASE_CLASSPATH=${HBASE_CLASSPATH}\n\n\n# The maximum amount of heap to use, in MB. Default
is 1000.\n# export HBASE_HEAPSIZE=1000\n\n# Extra Java runtime options.\n# Below are what
we set by default. May only work with SUN JVM.\n# For more on why as well as other possible
settings,\n# see http://wiki.apache.org/hadoop/PerformanceTuning\nexport SERVER_GC_OPTS=\"-verbose:gc
-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n#
Uncomment below to enable java garbage collection logging.\n# export HBASE_OPTS=\"$HBASE_OPTS
-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log\"\n\n#
Uncomment and adjust to enable 
 JMX exporting\n# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management
to configure remote password access.\n# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n#\n#
export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n#
If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct
memory size\n# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n#
export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n\n#
File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\nexport
HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n\n# Extra ssh options. Empty by default.\n#
export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n\n# Where log files
are stored. $HBASE_HOME/logs by default.\nexport HBASE_LOG_DIR={{l
 og_dir}}\n\n# A string representing this instance of hbase. $USER by default.\n# export HBASE_IDENT_STRING=$USER\n\n#
The scheduling priority for daemon processes. See 'man nice'.\n# export HBASE_NICENESS=10\n\n#
The directory where pid files are stored. /tmp by default.\nexport HBASE_PID_DIR={{pid_dir}}\n\n#
Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large
clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service
them.\n# export HBASE_SLAVE_SLEEP=0.1\n\n# Tell HBase whether it should manage it's own instance
of Zookeeper or not.\nexport HBASE_MANAGES_ZK=false\n\n{% if security_enabled %}\nexport HBASE_OPTS=\"$HBASE_OPTS
-XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}}\"\nexport
HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\nexport
HBASE_REGIONSERV
 ER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70
 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n{%
else %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log\"\nexport
HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS
-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70  -Xms{{regionserver_heapsize}}
-Xmx{{regionserver_heapsize}}\"\n{% endif %}", 
+            "hbase_regionserver_heapsize": "1024m", 
+            "hbase_log_dir": "/var/log/hbase"
+        }, 
+        "cluster-env": {
+            "security_enabled": "false", 
+            "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", 
+            "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version
}}/mapreduce/", 
+            "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", 
+            "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/",

+            "smokeuser": "ambari-qa", 
+            "ignore_groupsusers_create": "false", 
+            "hadoop-streaming_tar_source": "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar",

+            "kerberos_domain": "EXAMPLE.COM", 
+            "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/",

+            "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/",

+            "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",

+            "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz", 
+            "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/",

+            "user_group": "hadoop", 
+            "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", 
+            "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz", 
+            "hadoop.user.password": "", 
+            "hadoop.user.name": "hadoop"
+        }
+    }, 
+    "commandParams": {
+        "tasks": "[{\"type\":\"EXECUTE\",\"hosts\":\"master\",\"command\":\"su - hbase -c
\\u0027echo \\\"snapshot_all\\\" | hbase shell\\u0027\"}]", 
+        "service_package_folder": "common-services/HBASE/0.96.0.2.0/package", 
+        "hooks_folder": "HDP/2.0.6/hooks", 
+        "script": "ru_execute_tasks.py", 
+        "version": "2.2.0.0-2041", 
+        "command_timeout": "60", 
+        "script_type": "PYTHON"
+    }, 
+    "commandId": "33-1", 
+    "clusterHostInfo": {
+        "ganglia_monitor_hosts": [
+            "c6403.ambari.apache.org", 
+            "c6401.ambari.apache.org", 
+            "c6402.ambari.apache.org"
+        ], 
+        "nm_hosts": [
+            "c6403.ambari.apache.org"
+        ], 
+        "app_timeline_server_hosts": [
+            "c6402.ambari.apache.org"
+        ], 
+        "hive_metastore_host": [
+            "c6402.ambari.apache.org"
+        ], 
+        "zkfc_hosts": [
+            "c6401.ambari.apache.org", 
+            "c6402.ambari.apache.org"
+        ], 
+        "all_ping_ports": [
+            "8670", 
+            "8670", 
+            "8670"
+        ], 
+        "journalnode_hosts": [
+            "c6403.ambari.apache.org", 
+            "c6401.ambari.apache.org", 
+            "c6402.ambari.apache.org"
+        ], 
+        "rm_host": [
+            "c6402.ambari.apache.org"
+        ], 
+        "all_hosts": [
+            "c6403.ambari.apache.org", 
+            "c6401.ambari.apache.org", 
+            "c6402.ambari.apache.org"
+        ], 
+        "hbase_rs_hosts": [
+            "c6401.ambari.apache.org"
+        ], 
+        "slave_hosts": [
+            "c6403.ambari.apache.org"
+        ], 
+        "namenode_host": [
+            "c6401.ambari.apache.org", 
+            "c6402.ambari.apache.org"
+        ], 
+        "ganglia_server_host": [
+            "c6401.ambari.apache.org"
+        ], 
+        "hbase_master_hosts": [
+            "c6401.ambari.apache.org"
+        ], 
+        "hive_mysql_host": [
+            "c6402.ambari.apache.org"
+        ], 
+        "oozie_server": [
+            "c6403.ambari.apache.org"
+        ], 
+        "webhcat_server_host": [
+            "c6402.ambari.apache.org"
+        ], 
+        "ambari_server_host": [
+            "10.0.0.15"
+        ], 
+        "zookeeper_hosts": [
+            "c6403.ambari.apache.org", 
+            "c6401.ambari.apache.org", 
+            "c6402.ambari.apache.org"
+        ], 
+        "hs_host": [
+            "c6402.ambari.apache.org"
+        ], 
+        "hive_server_host": [
+            "c6402.ambari.apache.org"
+        ]
+    }
+}


Mime
View raw message