Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7987B1188C for ; Tue, 15 Apr 2014 12:34:38 +0000 (UTC) Received: (qmail 39656 invoked by uid 500); 15 Apr 2014 12:34:37 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 39597 invoked by uid 500); 15 Apr 2014 12:34:34 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 39590 invoked by uid 99); 15 Apr 2014 12:34:33 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Apr 2014 12:34:33 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 331EB8BC5AF; Tue, 15 Apr 2014 12:34:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dsen@apache.org To: commits@ambari.apache.org Message-Id: <236dfd4354dc4cefabe3cc88170f0a6c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: AMBARI-5472 Use SchemaTool in Hive for init metastore DB schema (dsen) Date: Tue, 15 Apr 2014 12:34:33 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/trunk a3be8c960 -> 0d31527f6 AMBARI-5472 Use SchemaTool in Hive for init metastore DB schema (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0d31527f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0d31527f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0d31527f Branch: refs/heads/trunk Commit: 0d31527f6e97db445438c3c1cf8308294df59323 Parents: a3be8c9 Author: Dmitry Sen Authored: Tue Apr 15 15:28:23 2014 +0300 Committer: Dmitry Sen Committed: Tue Apr 15 15:33:19 2014 +0300 ---------------------------------------------------------------------- .../2.0.6/services/HIVE/package/scripts/hive.py | 14 ++ .../stacks/2.1/HIVE/test_hive_metastore.py | 234 +++++++++++++++++++ 2 files changed, 248 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0d31527f/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py index a2cbee8..e472947 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py @@ -78,6 +78,20 @@ def hive(name=None): mode=0755, content=StaticFile('startMetastore.sh') ) + if params.init_metastore_schema: + create_schema_cmd = format("{hive_bin}/schematool -initSchema " + "-dbType {hive_metastore_db_type} " + "-userName {hive_metastore_user_name} " + "-passWord {hive_metastore_user_passwd}") + + check_schema_created_cmd = format("{hive_bin}/schematool -info " + "-dbType {hive_metastore_db_type} " + "-userName {hive_metastore_user_name} " + "-passWord {hive_metastore_user_passwd}") + + Execute(create_schema_cmd, + not_if = check_schema_created_cmd + ) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, http://git-wip-us.apache.org/repos/asf/ambari/blob/0d31527f/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py new file mode 100644 index 0000000..351571e --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +from mock.mock import MagicMock, call, patch +from stacks.utils.RMFTestCase import * + +class TestHiveMetastore(RMFTestCase): + + def test_configure_default(self): + self.executeScript("2.0.6/services/HIVE/package/scripts/hive_metastore.py", + classname = "HiveMetastore", + command = "configure", + config_file="../../2.1/configs/default.json" + ) + self.assert_configure_default() + + def test_start_default(self): + self.executeScript("2.0.6/services/HIVE/package/scripts/hive_metastore.py", + classname = "HiveMetastore", + command = "start", + config_file="../../2.1/configs/default.json" + ) + + self.assert_configure_default() + self.assertResourceCalled('Execute', 'env HADOOP_HOME=/usr JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', + not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps `cat /var/run/hive/hive.pid` >/dev/null 2>&1', + user = 'hive' + ) + + self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive asd com.mysql.jdbc.Driver', + path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'] + ) + + self.assertNoMoreResources() + + def test_stop_default(self): + self.executeScript("2.0.6/services/HIVE/package/scripts/hive_metastore.py", + classname = "HiveMetastore", + command = "stop", + config_file="../../2.1/configs/default.json" + ) + + self.assertResourceCalled('Execute', 'kill `cat /var/run/hive/hive.pid` >/dev/null 2>&1 && rm -f /var/run/hive/hive.pid') + self.assertNoMoreResources() + + def test_configure_secured(self): + self.executeScript("2.0.6/services/HIVE/package/scripts/hive_metastore.py", + classname = "HiveMetastore", + command = "configure", + config_file="../../2.1/configs/secured.json" + ) + self.assert_configure_default() + self.assertNoMoreResources() + + def test_start_secured(self): + self.executeScript("2.0.6/services/HIVE/package/scripts/hive_metastore.py", + classname = "HiveMetastore", + command = "start", + config_file="../../2.1/configs/secured.json" + ) + + self.assert_configure_secured() + self.assertResourceCalled('Execute', 'env HADOOP_HOME=/usr JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.log /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive', + not_if = 'ls /var/run/hive/hive.pid >/dev/null 2>&1 && ps `cat /var/run/hive/hive.pid` >/dev/null 2>&1', + user = 'hive' + ) + + self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive asd com.mysql.jdbc.Driver', + path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'] + ) + + self.assertNoMoreResources() + + def test_stop_secured(self): + self.executeScript("2.0.6/services/HIVE/package/scripts/hive_metastore.py", + classname = "HiveMetastore", + command = "stop", + config_file="../../2.1/configs/secured.json" + ) + + self.assertResourceCalled('Execute', 'kill `cat /var/run/hive/hive.pid` >/dev/null 2>&1 && rm -f /var/run/hive/hive.pid') + self.assertNoMoreResources() + + def assert_configure_default(self): + self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/HDP-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Directory', '/etc/hive/conf.server', + owner = 'hive', + group = 'hadoop', + recursive = True, + ) + self.assertResourceCalled('XmlConfig', 'mapred-site.xml', + owner = 'hive', + group = 'hadoop', + mode = 0600, + conf_dir = '/etc/hive/conf.server', + configurations = self.getConfig()['configurations']['mapred-site'], + ) + self.assertResourceCalled('XmlConfig', 'hive-site.xml', + owner = 'hive', + group = 'hadoop', + mode = 0600, + conf_dir = '/etc/hive/conf.server', + configurations = self.getConfig()['configurations']['hive-site'], + ) + self.assertResourceCalled('Execute', "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar'", + not_if = '[ -f DBConnectionVerification.jar]', + ) + self.assertResourceCalled('File', '/tmp/start_metastore_script', + content = StaticFile('startMetastore.sh'), + mode = 0755, + ) + self.assertResourceCalled('Execute', "/usr/lib/hive/bin/schematool -initSchema -dbType mysql -userName hive -passWord asd", + not_if = '/usr/lib/hive/bin/schematool -info -dbType mysql -userName hive -passWord asd', + ) + self.assertResourceCalled('Directory', '/var/run/hive', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + ) + self.assertResourceCalled('Directory', '/var/log/hive', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + ) + self.assertResourceCalled('Directory', '/var/lib/hive', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', + content = Template('hive-env.sh.j2', conf_dir="/etc/hive/conf.server"), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + + def assert_configure_secured(self): + self.assertResourceCalled('Execute', 'hive mkdir -p /tmp/HDP-artifacts/ ; cp /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib//mysql-connector-java.jar', + creates = '/usr/lib/hive/lib//mysql-connector-java.jar', + path = ['/bin', '/usr/bin/'], + not_if = 'test -f /usr/lib/hive/lib//mysql-connector-java.jar', + ) + self.assertResourceCalled('Directory', '/etc/hive/conf.server', + owner = 'hive', + group = 'hadoop', + recursive = True, + ) + self.assertResourceCalled('XmlConfig', 'mapred-site.xml', + owner = 'hive', + group = 'hadoop', + mode = 0600, + conf_dir = '/etc/hive/conf.server', + configurations = self.getConfig()['configurations']['mapred-site'], + ) + self.assertResourceCalled('XmlConfig', 'hive-site.xml', + owner = 'hive', + group = 'hadoop', + mode = 0600, + conf_dir = '/etc/hive/conf.server', + configurations = self.getConfig()['configurations']['hive-site'], + ) + self.assertResourceCalled('Execute', "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar'", + not_if = '[ -f DBConnectionVerification.jar]', + ) + self.assertResourceCalled('File', '/tmp/start_metastore_script', + content = StaticFile('startMetastore.sh'), + mode = 0755, + ) + self.assertResourceCalled('Execute', "/usr/lib/hive/bin/schematool -initSchema -dbType mysql -userName hive -passWord asd", + not_if = '/usr/lib/hive/bin/schematool -info -dbType mysql -userName hive -passWord asd', + ) + self.assertResourceCalled('Directory', '/var/run/hive', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + ) + self.assertResourceCalled('Directory', '/var/log/hive', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + ) + self.assertResourceCalled('Directory', '/var/lib/hive', + owner = 'hive', + group = 'hadoop', + mode = 0755, + recursive = True, + ) + self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh', + content = Template('hive-env.sh.j2', conf_dir="/etc/hive/conf.server"), + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template', + owner = 'hive', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template', + owner = 'hive', + group = 'hadoop', + ) + +