Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id EA0F0200C50 for ; Fri, 24 Mar 2017 13:51:53 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id E885C160B93; Fri, 24 Mar 2017 12:51:53 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E8718160B9F for ; Fri, 24 Mar 2017 13:51:50 +0100 (CET) Received: (qmail 21232 invoked by uid 500); 24 Mar 2017 12:51:49 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 21109 invoked by uid 99); 24 Mar 2017 12:51:49 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 24 Mar 2017 12:51:49 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 707BEE00EE; Fri, 24 Mar 2017 12:51:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vbrodetskyi@apache.org To: commits@ambari.apache.org Date: Fri, 24 Mar 2017 12:51:52 -0000 Message-Id: <890d9aaf80e44adeac9bdc556d60210c@git.apache.org> In-Reply-To: <695661a8ac964ac1b28efe0fcb150c38@git.apache.org> References: <695661a8ac964ac1b28efe0fcb150c38@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [04/12] ambari git commit: AMBARI-19832. HDP 3.0 support for Hive with configs, kerberos, widgets, metrics, quicklinks, and themes.(vbrodetskyi) archived-at: Fri, 24 Mar 2017 12:51:54 -0000 http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveTezSetup.cmd ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveTezSetup.cmd b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveTezSetup.cmd new file mode 100644 index 0000000..10d6a1c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveTezSetup.cmd @@ -0,0 +1,58 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +if not defined HADOOP_HOME ( + set EXITCODE=5 + goto :errorexit +) +if not defined HIVE_HOME ( + set EXITCODE=6 + goto :errorexit +) +if not defined TEZ_HOME ( + set EXITCODE=7 + goto :errorexit +) + +set EXITCODE=0 + +if not exist %HIVE_HOME%\conf\hive-tez-configured ( + %HADOOP_HOME%\bin\hadoop.cmd fs -mkdir /apps/tez + set EXITCODE=%ERRORLEVEL% + if %EXITCODE% neq 0 goto :errorexit + + %HADOOP_HOME%\bin\hadoop.cmd fs -chmod -R 755 /apps/tez + set EXITCODE=%ERRORLEVEL% + if %EXITCODE% neq 0 goto :errorexit + + %HADOOP_HOME%\bin\hadoop.cmd fs -chown -R hadoop:users /apps/tez + set EXITCODE=%ERRORLEVEL% + if %EXITCODE% neq 0 goto :errorexit + + %HADOOP_HOME%\bin\hadoop.cmd fs -put %TEZ_HOME%\* /apps/tez + set EXITCODE=%ERRORLEVEL% + if %EXITCODE% neq 0 goto :errorexit + + %HADOOP_HOME%\bin\hadoop.cmd fs -rm -r -skipTrash /apps/tez/conf + set EXITCODE=%ERRORLEVEL% + if %EXITCODE% neq 0 goto :errorexit + + echo done > %HIVE_HOME%\conf\hive-tez-configured +) +goto :eof + +:errorexit +exit /B %EXITCODE% http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveserver2.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveserver2.sql b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveserver2.sql new file mode 100644 index 0000000..99a3865 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveserver2.sql @@ -0,0 +1,23 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +CREATE EXTERNAL TABLE IF NOT EXISTS hiveserver2smoke20408 ( foo INT, bar STRING ); +DESCRIBE hiveserver2smoke20408; http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveserver2Smoke.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveserver2Smoke.sh b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveserver2Smoke.sh new file mode 100644 index 0000000..77d7b3e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/hiveserver2Smoke.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +smokeout=`/usr/lib/hive/bin/beeline -u $1 -n fakeuser -p fakepwd -d org.apache.hive.jdbc.HiveDriver -e '!run $2' 2>&1| awk '{print}'|grep Error` + +if [ "x$smokeout" == "x" ]; then + echo "Smoke test of hiveserver2 passed" + exit 0 +else + echo "Smoke test of hiveserver2 wasnt passed" + echo $smokeout + exit 1 +fi http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/pigSmoke.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/pigSmoke.sh b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/pigSmoke.sh new file mode 100644 index 0000000..2e90ac0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/pigSmoke.sh @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +A = load 'passwd' using PigStorage(':'); +B = foreach A generate \$0 as id; +store B into 'pigsmoke.out'; http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/removeMysqlUser.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/removeMysqlUser.sh b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/removeMysqlUser.sh new file mode 100644 index 0000000..7b6d331 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/removeMysqlUser.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +mysqldservice=$1 +mysqldbuser=$2 +userhost=$3 +myhostname=$(hostname -f) +sudo_prefix = "/var/lib/ambari-agent/ambari-sudo.sh -H -E" + +$sudo_prefix service $mysqldservice start +echo "Removing user $mysqldbuser@$userhost" +/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"DROP USER '$mysqldbuser'@'$userhost';\"" +/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"flush privileges;\"" +$sudo_prefix service $mysqldservice stop http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/startMetastore.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/startMetastore.sh b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/startMetastore.sh new file mode 100644 index 0000000..86541f0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/startMetastore.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +HIVE_BIN=${HIVE_BIN:-"hive"} + +HIVE_CONF_DIR=$4 $HIVE_BIN --service metastore -hiveconf hive.log.file=hivemetastore.log -hiveconf hive.log.dir=$5 > $1 2> $2 & +echo $!|cat>$3 http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/templetonSmoke.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/templetonSmoke.sh b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/templetonSmoke.sh new file mode 100644 index 0000000..f5a0da1 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/files/templetonSmoke.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +export ttonhost=$1 +export smoke_test_user=$2 +export templeton_port=$3 +export ttonTestScript=$4 +export has_pig=$5 +export smoke_user_keytab=$6 +export security_enabled=$7 +export kinit_path_local=$8 +export smokeuser_principal=$9 +export tmp_dir=${10} +export ttonurl="http://${ttonhost}:${templeton_port}/templeton/v1" + +if [[ $security_enabled == "true" ]]; then + kinitcmd="${kinit_path_local} -kt ${smoke_user_keytab} ${smokeuser_principal}; " +else + kinitcmd="" +fi + +export no_proxy=$ttonhost +cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' $ttonurl/status 2>&1" +retVal=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd"` +httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` + +# try again for 2.3 username requirement +if [[ "$httpExitCode" == "500" ]] ; then + cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' $ttonurl/status?user.name=$smoke_test_user 2>&1" + retVal=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd"` + httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` +fi + +if [[ "$httpExitCode" -ne "200" ]] ; then + echo "Templeton Smoke Test (status cmd): Failed. : $retVal" + export TEMPLETON_EXIT_CODE=1 + exit 1 +fi + +#try hcat ddl command +/var/lib/ambari-agent/ambari-sudo.sh rm -f ${tmp_dir}/show_db.post.txt +echo "user.name=${smoke_test_user}&exec=show databases;" > ${tmp_dir}/show_db.post.txt +/var/lib/ambari-agent/ambari-sudo.sh chown ${smoke_test_user} ${tmp_dir}/show_db.post.txt +cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' -d @${tmp_dir}/show_db.post.txt $ttonurl/ddl 2>&1" +retVal=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd"` +httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` + +if [[ "$httpExitCode" -ne "200" ]] ; then + echo "Templeton Smoke Test (ddl cmd): Failed. : $retVal" + export TEMPLETON_EXIT_CODE=1 + exit 1 +fi + +# NOT SURE?? SUHAS +if [[ $security_enabled == "true" ]]; then + echo "Templeton Pig Smoke Tests not run in secure mode" + exit 0 +fi + +if [[ $has_pig != "True" ]]; then + echo "Templeton Pig Smoke Tests are not run, because Pig is not installed" + exit 0 +fi + +#try pig query + +#create, copy post args file +/var/lib/ambari-agent/ambari-sudo.sh rm -f ${tmp_dir}/pig_post.txt +echo -n "user.name=${smoke_test_user}&file=/tmp/$ttonTestScript" > ${tmp_dir}/pig_post.txt +/var/lib/ambari-agent/ambari-sudo.sh chown ${smoke_test_user} ${tmp_dir}/pig_post.txt + +#submit pig query +cmd="curl --negotiate -u : -s -w 'http_code <%{http_code}>' -d @${tmp_dir}/pig_post.txt $ttonurl/pig 2>&1" +retVal=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd"` +httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` +if [[ "$httpExitCode" -ne "200" ]] ; then + echo "Templeton Smoke Test (pig cmd): Failed. : $retVal" + export TEMPLETON_EXIT_CODE=1 + exit 1 +fi + +exit 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/__init__.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/__init__.py new file mode 100644 index 0000000..5561e10 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat.py new file mode 100644 index 0000000..faa8c7e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +# Python Imports +import os +import sys + +# Local Imports +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.format import format +from resource_management.core.resources.system import Directory, File +from resource_management.core.source import InlineTemplate +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook +from ambari_commons import OSConst +from ambari_commons.constants import SERVICE + + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def hcat(): + import params + + XmlConfig("hive-site.xml", + conf_dir = params.hive_conf_dir, + configurations = params.config['configurations']['hive-site'], + owner=params.hive_user, + configuration_attributes=params.config['configuration_attributes']['hive-site'] + ) + + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def hcat(): + import params + + Directory(params.hive_conf_dir, + create_parents = True, + owner=params.hive_user, + group=params.user_group, + ) + + + Directory(params.hcat_conf_dir, + create_parents = True, + owner=params.webhcat_user, + group=params.user_group, + ) + + Directory(params.hcat_pid_dir, + owner=params.webhcat_user, + create_parents = True + ) + + XmlConfig("hive-site.xml", + conf_dir=params.hive_client_conf_dir, + configurations=params.config['configurations']['hive-site'], + configuration_attributes=params.config['configuration_attributes']['hive-site'], + owner=params.hive_user, + group=params.user_group, + mode=0644) + + File(format("{hcat_conf_dir}/hcat-env.sh"), + owner=params.webhcat_user, + group=params.user_group, + content=InlineTemplate(params.hcat_env_sh_template) + ) + + # Generate atlas-application.properties.xml file + if params.enable_atlas_hook: + atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) + setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat_client.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat_client.py new file mode 100644 index 0000000..47bbc41 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat_client.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from hcat import hcat +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyImpl +from resource_management.core.logger import Logger +from resource_management.core.exceptions import ClientComponentHasNoStatus +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.constants import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.script.script import Script + + +class HCatClient(Script): + def install(self, env): + import params + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + hcat() + + def status(self, env): + raise ClientComponentHasNoStatus() + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class HCatClientWindows(HCatClient): + pass + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class HCatClientDefault(HCatClient): + def get_component_name(self): + # HCat client doesn't have a first-class entry in . Since clients always + # update after daemons, this ensures that the hcat directories are correct on hosts + # which do not include the WebHCat daemon + return "hive-webhcat" + + + def pre_upgrade_restart(self, env, upgrade_type=None): + """ + Execute before reconfiguring this client to the new stack version. + + :param env: + :param upgrade_type: + :return: + """ + Logger.info("Executing Hive HCat Client Stack Upgrade pre-restart") + + import params + env.set_params(params) + + # this function should not execute if the stack version does not support rolling upgrade + if not (params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version)): + return + + # HCat client doesn't have a first-class entry in . Since clients always + # update after daemons, this ensures that the hcat directories are correct on hosts + # which do not include the WebHCat daemon + stack_select.select("hive-webhcat", params.version) + + +if __name__ == "__main__": + HCatClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat_service_check.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat_service_check.py new file mode 100644 index 0000000..25d0c81 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hcat_service_check.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +from resource_management.libraries.functions import get_unique_id_and_date +from resource_management.libraries.functions.format import format +from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop +from resource_management.core.source import StaticFile +from resource_management.core.resources.system import Execute, File +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def hcat_service_check(): + import params + smoke_cmd = os.path.join(params.stack_root, "Run-SmokeTests.cmd") + service = "HCatalog" + Execute(format("cmd /C {smoke_cmd} {service}"), user=params.webhcat_user, logoutput=True) + + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def hcat_service_check(): + import params + unique = get_unique_id_and_date() + output_file = format("{hive_apps_whs_dir}/hcatsmoke{unique}") + test_cmd = format("fs -test -e {output_file}") + + if params.security_enabled: + kinit_cmd = format( + "{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ") + else: + kinit_cmd = "" + + File(format("{tmp_dir}/hcatSmoke.sh"), + content=StaticFile("hcatSmoke.sh"), + mode=0755 + ) + + prepare_cmd = format("{kinit_cmd}env JAVA_HOME={java64_home} {tmp_dir}/hcatSmoke.sh hcatsmoke{unique} prepare {purge_tables}") + + exec_path = params.execute_path + if params.version and params.stack_root: + upgrade_hive_bin = format("{stack_root}/{version}/hive/bin") + exec_path = os.environ['PATH'] + os.pathsep + params.hadoop_bin_dir + os.pathsep + upgrade_hive_bin + + Execute(prepare_cmd, + tries=3, + user=params.smokeuser, + try_sleep=5, + path=['/usr/sbin', '/usr/local/bin', '/bin', '/usr/bin', exec_path], + logoutput=True) + + if params.security_enabled: + Execute (format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), + user = params.hdfs_user, + ) + + ExecuteHadoop(test_cmd, + user=params.hdfs_user, + logoutput=True, + conf_dir=params.hadoop_conf_dir, + bin_dir=params.execute_path + ) + + cleanup_cmd = format("{kinit_cmd} {tmp_dir}/hcatSmoke.sh hcatsmoke{unique} cleanup {purge_tables}") + + Execute(cleanup_cmd, + tries=3, + user=params.smokeuser, + try_sleep=5, + path=['/usr/sbin', '/usr/local/bin', '/bin', '/usr/bin', exec_path], + logoutput=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py new file mode 100644 index 0000000..b7b04a2 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +import glob +from urlparse import urlparse + +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.libraries.functions.get_config import get_config +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.core.resources.service import ServiceConfig +from resource_management.core.resources.system import File, Execute, Directory +from resource_management.core.source import StaticFile, Template, DownloadSource, InlineTemplate +from resource_management.core.shell import as_user +from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.format import format +from resource_management.core.exceptions import Fail +from resource_management.core.shell import as_sudo +from resource_management.core.shell import quote_bash_args +from resource_management.core.logger import Logger +from resource_management.core import utils +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook +from resource_management.libraries.functions.security_commons import update_credential_provider_path +from ambari_commons.constants import SERVICE + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def hive(name=None): + import params + + hive_client_conf_path = format("{stack_root}/current/{component_directory}/conf") + # Permissions 644 for conf dir (client) files, and 600 for conf.server + mode_identified = 0644 if params.hive_config_dir == hive_client_conf_path else 0600 + + Directory(params.hive_etc_dir_prefix, + mode=0755 + ) + + # We should change configurations for client as well as for server. + # The reason is that stale-configs are service-level, not component. + Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) + for conf_dir in params.hive_conf_dirs_list: + fill_conf_dir(conf_dir) + + params.hive_site_config = update_credential_provider_path(params.hive_site_config, + 'hive-site', + os.path.join(params.hive_conf_dir, 'hive-site.jceks'), + params.hive_user, + params.user_group + ) + XmlConfig("hive-site.xml", + conf_dir=params.hive_config_dir, + configurations=params.hive_site_config, + configuration_attributes=params.config['configuration_attributes']['hive-site'], + owner=params.hive_user, + group=params.user_group, + mode=mode_identified) + + # Generate atlas-application.properties.xml file + if params.enable_atlas_hook: + atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) + setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) + + File(format("{hive_config_dir}/hive-env.sh"), + owner=params.hive_user, + group=params.user_group, + content=InlineTemplate(params.hive_env_sh_template), + mode=mode_identified + ) + + # On some OS this folder could be not exists, so we will create it before pushing there files + Directory(params.limits_conf_dir, + create_parents = True, + owner='root', + group='root' + ) + + File(os.path.join(params.limits_conf_dir, 'hive.conf'), + owner='root', + group='root', + mode=0644, + content=Template("hive.conf.j2") + ) + if params.security_enabled: + File(os.path.join(params.hive_config_dir, 'zkmigrator_jaas.conf'), + owner=params.hive_user, + group=params.user_group, + content=Template("zkmigrator_jaas.conf.j2") + ) + + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), + content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), + mode = 0644, + ) + + if name != "client": + setup_non_client() + if name == 'hiveserver2': + setup_hiveserver2() + if name == 'metastore': + setup_metastore() + +def setup_hiveserver2(): + import params + + File(params.start_hiveserver2_path, + mode=0755, + content=Template(format('{start_hiveserver2_script}')) + ) + + File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hiveserver2.properties"), + owner=params.hive_user, + group=params.user_group, + content=Template("hadoop-metrics2-hiveserver2.properties.j2"), + mode=0600 + ) + XmlConfig("hiveserver2-site.xml", + conf_dir=params.hive_server_conf_dir, + configurations=params.config['configurations']['hiveserver2-site'], + configuration_attributes=params.config['configuration_attributes']['hiveserver2-site'], + owner=params.hive_user, + group=params.user_group, + mode=0600) + + # copy tarball to HDFS feature not supported + if not (params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major)): + params.HdfsResource(params.webhcat_apps_dir, + type="directory", + action="create_on_execute", + owner=params.webhcat_user, + mode=0755 + ) + + # Create webhcat dirs. + if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: + params.HdfsResource(params.hcat_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.webhcat_user, + mode=params.hcat_hdfs_user_mode + ) + + params.HdfsResource(params.webhcat_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.webhcat_user, + mode=params.webhcat_hdfs_user_mode + ) + + # ****** Begin Copy Tarballs ****** + # ********************************* + # if copy tarball to HDFS feature supported copy mapreduce.tar.gz and tez.tar.gz to HDFS + if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major): + copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) + copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) + + # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. + # This can use a different source and dest location to account + copy_to_hdfs("pig", + params.user_group, + params.hdfs_user, + file_mode=params.tarballs_mode, + custom_source_file=params.pig_tar_source, + custom_dest_file=params.pig_tar_dest_file, + skip=params.sysprep_skip_copy_tarballs_hdfs) + copy_to_hdfs("hive", + params.user_group, + params.hdfs_user, + file_mode=params.tarballs_mode, + custom_source_file=params.hive_tar_source, + custom_dest_file=params.hive_tar_dest_file, + skip=params.sysprep_skip_copy_tarballs_hdfs) + + wildcard_tarballs = ["sqoop", "hadoop_streaming"] + for tarball_name in wildcard_tarballs: + source_file_pattern = eval("params." + tarball_name + "_tar_source") + dest_dir = eval("params." + tarball_name + "_tar_dest_dir") + + if source_file_pattern is None or dest_dir is None: + continue + + source_files = glob.glob(source_file_pattern) if "*" in source_file_pattern else [source_file_pattern] + for source_file in source_files: + src_filename = os.path.basename(source_file) + dest_file = os.path.join(dest_dir, src_filename) + + copy_to_hdfs(tarball_name, + params.user_group, + params.hdfs_user, + file_mode=params.tarballs_mode, + custom_source_file=source_file, + custom_dest_file=dest_file, + skip=params.sysprep_skip_copy_tarballs_hdfs) + # ******* End Copy Tarballs ******* + # ********************************* + + # if warehouse directory is in DFS + if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(params.default_fs).scheme: + # Create Hive Metastore Warehouse Dir + params.HdfsResource(params.hive_apps_whs_dir, + type="directory", + action="create_on_execute", + owner=params.hive_user, + mode=0777 + ) + else: + Logger.info(format("Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS.")) + + # Create Hive User Dir + params.HdfsResource(params.hive_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.hive_user, + mode=params.hive_hdfs_user_mode + ) + + if not is_empty(params.hive_exec_scratchdir) and not urlparse(params.hive_exec_scratchdir).path.startswith("/tmp"): + params.HdfsResource(params.hive_exec_scratchdir, + type="directory", + action="create_on_execute", + owner=params.hive_user, + group=params.hdfs_user, + mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir + + params.HdfsResource(None, action="execute") + +def setup_non_client(): + import params + + Directory(params.hive_pid_dir, + create_parents = True, + cd_access='a', + owner=params.hive_user, + group=params.user_group, + mode=0755) + Directory(params.hive_log_dir, + create_parents = True, + cd_access='a', + owner=params.hive_user, + group=params.user_group, + mode=0755) + Directory(params.hive_var_lib, + create_parents = True, + cd_access='a', + owner=params.hive_user, + group=params.user_group, + mode=0755) + + if params.hive_jdbc_target is not None and not os.path.exists(params.hive_jdbc_target): + jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar) + if params.hive2_jdbc_target is not None and not os.path.exists(params.hive2_jdbc_target): + jdbc_connector(params.hive2_jdbc_target, params.hive2_previous_jdbc_jar) + +def setup_metastore(): + import params + + if params.hive_metastore_site_supported: + hivemetastore_site_config = get_config("hivemetastore-site") + if hivemetastore_site_config: + XmlConfig("hivemetastore-site.xml", + conf_dir=params.hive_server_conf_dir, + configurations=params.config['configurations']['hivemetastore-site'], + configuration_attributes=params.config['configuration_attributes']['hivemetastore-site'], + owner=params.hive_user, + group=params.user_group, + mode=0600) + + File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), + owner=params.hive_user, + group=params.user_group, + content=Template("hadoop-metrics2-hivemetastore.properties.j2"), + mode=0600 + ) + + File(params.start_metastore_path, + mode=0755, + content=StaticFile('startMetastore.sh') + ) + +def create_metastore_schema(): + import params + + create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " + "{hive_schematool_bin}/schematool -initSchema " + "-dbType {hive_metastore_db_type} " + "-userName {hive_metastore_user_name} " + "-passWord {hive_metastore_user_passwd!p} -verbose") + + check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " + "{hive_schematool_bin}/schematool -info " + "-dbType {hive_metastore_db_type} " + "-userName {hive_metastore_user_name} " + "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) + + # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. + # Fixing it with the hack below: + quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd)) + if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ + or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': + quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1] + Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace( + format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) + + Execute(create_schema_cmd, + not_if = check_schema_created_cmd, + user = params.hive_user + ) + +""" +Writes configuration files required by Hive. +""" +def fill_conf_dir(component_conf_dir): + import params + hive_client_conf_path = os.path.realpath(format("{stack_root}/current/{component_directory}/conf")) + component_conf_dir = os.path.realpath(component_conf_dir) + mode_identified_for_file = 0644 if component_conf_dir == hive_client_conf_path else 0600 + mode_identified_for_dir = 0755 if component_conf_dir == hive_client_conf_path else 0700 + Directory(component_conf_dir, + owner=params.hive_user, + group=params.user_group, + create_parents = True, + mode=mode_identified_for_dir + ) + + XmlConfig("mapred-site.xml", + conf_dir=component_conf_dir, + configurations=params.config['configurations']['mapred-site'], + configuration_attributes=params.config['configuration_attributes']['mapred-site'], + owner=params.hive_user, + group=params.user_group, + mode=mode_identified_for_file) + + + File(format("{component_conf_dir}/hive-default.xml.template"), + owner=params.hive_user, + group=params.user_group, + mode=mode_identified_for_file + ) + + File(format("{component_conf_dir}/hive-env.sh.template"), + owner=params.hive_user, + group=params.user_group, + mode=mode_identified_for_file + ) + + # Create hive-log4j.properties and hive-exec-log4j.properties + # in /etc/hive/conf and not in /etc/hive2/conf + if params.log4j_version == '1': + log4j_exec_filename = 'hive-exec-log4j.properties' + if (params.log4j_exec_props != None): + File(format("{component_conf_dir}/{log4j_exec_filename}"), + mode=mode_identified_for_file, + group=params.user_group, + owner=params.hive_user, + content=InlineTemplate(params.log4j_exec_props) + ) + elif (os.path.exists("{component_conf_dir}/{log4j_exec_filename}.template")): + File(format("{component_conf_dir}/{log4j_exec_filename}"), + mode=mode_identified_for_file, + group=params.user_group, + owner=params.hive_user, + content=StaticFile(format("{component_conf_dir}/{log4j_exec_filename}.template")) + ) + + log4j_filename = 'hive-log4j.properties' + if (params.log4j_props != None): + File(format("{component_conf_dir}/{log4j_filename}"), + mode=mode_identified_for_file, + group=params.user_group, + owner=params.hive_user, + content=InlineTemplate(params.log4j_props) + ) + elif (os.path.exists("{component_conf_dir}/{log4j_filename}.template")): + File(format("{component_conf_dir}/{log4j_filename}"), + mode=mode_identified_for_file, + group=params.user_group, + owner=params.hive_user, + content=StaticFile(format("{component_conf_dir}/{log4j_filename}.template")) + ) + pass # if params.log4j_version == '1' + + +def jdbc_connector(target, hive_previous_jdbc_jar): + """ + Shared by Hive Batch, Hive Metastore, and Hive Interactive + :param target: Target of jdbc jar name, which could be for any of the components above. + """ + import params + + if not params.jdbc_jar_name: + return + + if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db: + environment = { + "no_proxy": format("{ambari_server_hostname}") + } + + if hive_previous_jdbc_jar and os.path.isfile(hive_previous_jdbc_jar): + File(hive_previous_jdbc_jar, action='delete') + + # TODO: should be removed after ranger_hive_plugin will not provide jdbc + if params.prepackaged_jdbc_name != params.jdbc_jar_name: + Execute(('rm', '-f', params.prepackaged_ojdbc_symlink), + path=["/bin", "/usr/bin/"], + sudo = True) + + File(params.downloaded_custom_connector, + content = DownloadSource(params.driver_curl_source)) + + # maybe it will be more correcvly to use db type + if params.sqla_db_used: + untar_sqla_type2_driver = ('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir) + + Execute(untar_sqla_type2_driver, sudo = True) + + Execute(format("yes | {sudo} cp {jars_path_in_archive} {hive_lib}")) + + Directory(params.jdbc_libs_dir, + create_parents = True) + + Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}")) + + Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*")) + + else: + Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target), + #creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc + path=["/bin", "/usr/bin/"], + sudo = True) + + else: + #for default hive db (Mysql) + Execute(('cp', '--remove-destination', format('/usr/share/java/{jdbc_jar_name}'), target), + #creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc + path=["/bin", "/usr/bin/"], + sudo=True + ) + pass + + File(target, + mode = 0644, + ) + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def hive(name=None): + import params + + XmlConfig("hive-site.xml", + conf_dir = params.hive_conf_dir, + configurations = params.config['configurations']['hive-site'], + owner=params.hive_user, + configuration_attributes=params.config['configuration_attributes']['hive-site'] + ) + + if name in ["hiveserver2","metastore"]: + # Manually overriding service logon user & password set by the installation package + service_name = params.service_map[name] + ServiceConfig(service_name, + action="change_user", + username = params.hive_user, + password = Script.get_password(params.hive_user)) + Execute(format("cmd /c hadoop fs -mkdir -p {hive_warehouse_dir}"), logoutput=True, user=params.hadoop_user) + + if name == 'metastore': + if params.init_metastore_schema: + check_schema_created_cmd = format('cmd /c "{hive_bin}\\hive.cmd --service schematool -info ' + '-dbType {hive_metastore_db_type} ' + '-userName {hive_metastore_user_name} ' + '-passWord {hive_metastore_user_passwd!p}' + '&set EXITCODE=%ERRORLEVEL%&exit /B %EXITCODE%"', #cmd "feature", propagate the process exit code manually + hive_bin=params.hive_bin, + hive_metastore_db_type=params.hive_metastore_db_type, + hive_metastore_user_name=params.hive_metastore_user_name, + hive_metastore_user_passwd=params.hive_metastore_user_passwd) + try: + Execute(check_schema_created_cmd) + except Fail: + create_schema_cmd = format('cmd /c {hive_bin}\\hive.cmd --service schematool -initSchema ' + '-dbType {hive_metastore_db_type} ' + '-userName {hive_metastore_user_name} ' + '-passWord {hive_metastore_user_passwd!p}', + hive_bin=params.hive_bin, + hive_metastore_db_type=params.hive_metastore_db_type, + hive_metastore_user_name=params.hive_metastore_user_name, + hive_metastore_user_passwd=params.hive_metastore_user_passwd) + Execute(create_schema_cmd, + user = params.hive_user, + logoutput=True + ) + + if name == "hiveserver2": + if params.hive_execution_engine == "tez": + # Init the tez app dir in hadoop + script_file = __file__.replace('/', os.sep) + cmd_file = os.path.normpath(os.path.join(os.path.dirname(script_file), "..", "files", "hiveTezSetup.cmd")) + + Execute("cmd /c " + cmd_file, logoutput=True, user=params.hadoop_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_client.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_client.py new file mode 100644 index 0000000..83d82df --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_client.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import sys +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import conf_select, stack_select +from resource_management.libraries.functions.constants import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.core.logger import Logger +from hive import hive +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst +from resource_management.core.exceptions import ClientComponentHasNoStatus + +class HiveClient(Script): + def install(self, env): + import params + self.install_packages(env) + self.configure(env) + + def status(self, env): + raise ClientComponentHasNoStatus() + + def configure(self, env): + import params + env.set_params(params) + hive(name='client') + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class HiveClientWindows(HiveClient): + pass + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class HiveClientDefault(HiveClient): + def get_component_name(self): + return "hadoop-client" + + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing Hive client Stack Upgrade pre-restart") + + import params + env.set_params(params) + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "hive", params.version) + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-client", params.version) + + +if __name__ == "__main__": + HiveClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_interactive.py new file mode 100644 index 0000000..2ed3e3a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_interactive.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +# Python Imports +import os +import glob +from urlparse import urlparse + +# Resource Management and Common Imports +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import compare_versions +from resource_management.core.resources.service import ServiceConfig +from resource_management.core.resources.system import File, Directory +from resource_management.core.source import Template, DownloadSource, InlineTemplate +from resource_management.core.shell import as_user +from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.format import format +from resource_management.core.shell import as_sudo +from resource_management.core.shell import quote_bash_args +from resource_management.core.logger import Logger +from resource_management.core import utils +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook +from resource_management.libraries.functions.security_commons import update_credential_provider_path +from ambari_commons.constants import SERVICE + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst +from hive import fill_conf_dir, jdbc_connector + + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def hive_interactive(name=None): + pass + +""" +Sets up the configs, jdbc connection and tarball copy to HDFS for Hive Server Interactive. +""" +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def hive_interactive(name=None): + import params + MB_TO_BYTES = 1048576 + + # Create Hive User Dir + params.HdfsResource(params.hive_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.hive_user, + mode=params.hive_hdfs_user_mode + ) + + # list of properties that should be excluded from the config + # this approach is a compromise against adding a dedicated config + # type for hive_server_interactive or needed config groups on a + # per component basis + exclude_list = ['hive.enforce.bucketing', + 'hive.enforce.sorting'] + + # List of configs to be excluded from hive2 client, but present in Hive2 server. + exclude_list_for_hive2_client = ['javax.jdo.option.ConnectionPassword', + 'hadoop.security.credential.provider.path'] + + # Copy Tarballs in HDFS. + if params.stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted_major): + resource_created = copy_to_hdfs("tez_hive2", + params.user_group, + params.hdfs_user, + file_mode=params.tarballs_mode, + skip=params.sysprep_skip_copy_tarballs_hdfs) + + if resource_created: + params.HdfsResource(None, action="execute") + + Directory(params.hive_interactive_etc_dir_prefix, + mode=0755 + ) + + Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) + for conf_dir in params.hive_conf_dirs_list: + fill_conf_dir(conf_dir) + + ''' + As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml, + we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml. + ''' + merged_hive_interactive_site = {} + merged_hive_interactive_site.update(params.config['configurations']['hive-site']) + merged_hive_interactive_site.update(params.config['configurations']['hive-interactive-site']) + for item in exclude_list: + if item in merged_hive_interactive_site.keys(): + del merged_hive_interactive_site[item] + + ''' + Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to + convert it to bytes before we write it down to config file. + ''' + if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys(): + hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get("hive.llap.io.memory.size") + hive_llap_io_mem_size_in_bytes = long(hive_llap_io_mem_size_in_mb) * MB_TO_BYTES + merged_hive_interactive_site['hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes + Logger.info("Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing " + "it to config file.".format(hive_llap_io_mem_size_in_mb, hive_llap_io_mem_size_in_bytes)) + + ''' + Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook', + which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed. + ''' + # Generate atlas-application.properties.xml file + if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive: + Logger.info("Setup for Atlas Hive2 Hook started.") + + atlas_hook_filepath = os.path.join(params.hive_server_interactive_conf_dir, params.atlas_hook_filename) + setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) + + Logger.info("Setup for Atlas Hive2 Hook done.") + else: + # Required for HDP 2.5 stacks + Logger.info("Skipping setup for Atlas Hook, as it is disabled/ not supported.") + remove_atlas_hook_if_exists(merged_hive_interactive_site) + + ''' + As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml, + we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml. + ''' + merged_tez_interactive_site = {} + if 'tez-site' in params.config['configurations']: + merged_tez_interactive_site.update(params.config['configurations']['tez-site']) + Logger.info("Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'.") + else: + Logger.error("Tez's 'tez-site' couldn't be retrieved from passed-in configurations.") + + merged_tez_interactive_site.update(params.config['configurations']['tez-interactive-site']) + XmlConfig("tez-site.xml", + conf_dir = params.tez_interactive_config_dir, + configurations = merged_tez_interactive_site, + configuration_attributes=params.config['configuration_attributes']['tez-interactive-site'], + owner = params.tez_interactive_user, + group = params.user_group, + mode = 0664) + + ''' + Merge properties from hiveserver2-interactive-site into hiveserver2-site + ''' + merged_hiveserver2_interactive_site = {} + if 'hiveserver2-site' in params.config['configurations']: + merged_hiveserver2_interactive_site.update(params.config['configurations']['hiveserver2-site']) + Logger.info("Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'.") + else: + Logger.error("'hiveserver2-site' couldn't be retrieved from passed-in configurations.") + merged_hiveserver2_interactive_site.update(params.config['configurations']['hiveserver2-interactive-site']) + + + # Create config files under /etc/hive2/conf and /etc/hive2/conf/conf.server: + # hive-site.xml + # hive-env.sh + # llap-daemon-log4j2.properties + # llap-cli-log4j2.properties + # hive-log4j2.properties + # hive-exec-log4j2.properties + # beeline-log4j2.properties + + hive2_conf_dirs_list = params.hive_conf_dirs_list + hive2_client_conf_path = format("{stack_root}/current/{component_directory}/conf") + + # Making copy of 'merged_hive_interactive_site' in 'merged_hive_interactive_site_copy', and deleting 'javax.jdo.option.ConnectionPassword' + # config from there, as Hive2 client shouldn't have that config. + merged_hive_interactive_site_copy = merged_hive_interactive_site.copy() + for item in exclude_list_for_hive2_client: + if item in merged_hive_interactive_site.keys(): + del merged_hive_interactive_site_copy[item] + + for conf_dir in hive2_conf_dirs_list: + mode_identified = 0644 if conf_dir == hive2_client_conf_path else 0600 + if conf_dir == hive2_client_conf_path: + XmlConfig("hive-site.xml", + conf_dir=conf_dir, + configurations=merged_hive_interactive_site_copy, + configuration_attributes=params.config['configuration_attributes']['hive-interactive-site'], + owner=params.hive_user, + group=params.user_group, + mode=0644) + else: + merged_hive_interactive_site = update_credential_provider_path(merged_hive_interactive_site, + 'hive-site', + os.path.join(conf_dir, 'hive-site.jceks'), + params.hive_user, + params.user_group + ) + XmlConfig("hive-site.xml", + conf_dir=conf_dir, + configurations=merged_hive_interactive_site, + configuration_attributes=params.config['configuration_attributes']['hive-interactive-site'], + owner=params.hive_user, + group=params.user_group, + mode=0600) + XmlConfig("hiveserver2-site.xml", + conf_dir=conf_dir, + configurations=merged_hiveserver2_interactive_site, + configuration_attributes=params.config['configuration_attributes']['hiveserver2-interactive-site'], + owner=params.hive_user, + group=params.user_group, + mode=mode_identified) + + hive_server_interactive_conf_dir = conf_dir + + File(format("{hive_server_interactive_conf_dir}/hive-env.sh"), + owner=params.hive_user, + group=params.user_group, + mode=mode_identified, + content=InlineTemplate(params.hive_interactive_env_sh_template)) + + llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"), + mode=mode_identified, + group=params.user_group, + owner=params.hive_user, + content=InlineTemplate(params.llap_daemon_log4j)) + + llap_cli_log4j2_filename = 'llap-cli-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"), + mode=mode_identified, + group=params.user_group, + owner=params.hive_user, + content=InlineTemplate(params.llap_cli_log4j2)) + + hive_log4j2_filename = 'hive-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"), + mode=mode_identified, + group=params.user_group, + owner=params.hive_user, + content=InlineTemplate(params.hive_log4j2)) + + hive_exec_log4j2_filename = 'hive-exec-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"), + mode=mode_identified, + group=params.user_group, + owner=params.hive_user, + content=InlineTemplate(params.hive_exec_log4j2)) + + beeline_log4j2_filename = 'beeline-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"), + mode=mode_identified, + group=params.user_group, + owner=params.hive_user, + content=InlineTemplate(params.beeline_log4j2)) + + File(os.path.join(hive_server_interactive_conf_dir, "hadoop-metrics2-hiveserver2.properties"), + owner=params.hive_user, + group=params.user_group, + mode=mode_identified, + content=Template("hadoop-metrics2-hiveserver2.properties.j2") + ) + + File(format("{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"), + owner=params.hive_user, + group=params.user_group, + mode=mode_identified, + content=Template("hadoop-metrics2-llapdaemon.j2")) + + File(format("{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"), + owner=params.hive_user, + group=params.user_group, + mode=mode_identified, + content=Template("hadoop-metrics2-llaptaskscheduler.j2")) + + + # On some OS this folder could be not exists, so we will create it before pushing there files + Directory(params.limits_conf_dir, + create_parents = True, + owner='root', + group='root') + + File(os.path.join(params.limits_conf_dir, 'hive.conf'), + owner='root', + group='root', + mode=0644, + content=Template("hive.conf.j2")) + + if not os.path.exists(params.target_hive_interactive): + jdbc_connector(params.target_hive_interactive, params.hive_intaractive_previous_jdbc_jar) + + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), + content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), + mode = 0644) + File(params.start_hiveserver2_interactive_path, + mode=0755, + content=Template(format('{start_hiveserver2_interactive_script}'))) + + Directory(params.hive_pid_dir, + create_parents=True, + cd_access='a', + owner=params.hive_user, + group=params.user_group, + mode=0755) + Directory(params.hive_log_dir, + create_parents=True, + cd_access='a', + owner=params.hive_user, + group=params.user_group, + mode=0755) + Directory(params.hive_interactive_var_lib, + create_parents=True, + cd_access='a', + owner=params.hive_user, + group=params.user_group, + mode=0755) + +""" +Remove 'org.apache.atlas.hive.hook.HiveHook' value from Hive2/hive-site.xml config 'hive.exec.post.hooks', if exists. +""" +def remove_atlas_hook_if_exists(merged_hive_interactive_site): + if 'hive.exec.post.hooks' in merged_hive_interactive_site.keys(): + existing_hive_exec_post_hooks = merged_hive_interactive_site.get('hive.exec.post.hooks') + if existing_hive_exec_post_hooks: + hook_splits = existing_hive_exec_post_hooks.split(",") + updated_hook_splits = [hook for hook in hook_splits if not hook.strip() == 'org.apache.atlas.hive.hook.HiveHook'] + updated_hooks_str = ",".join((str(hook)).strip() for hook in updated_hook_splits) + if updated_hooks_str != existing_hive_exec_post_hooks: + merged_hive_interactive_site['hive.exec.post.hooks'] = updated_hooks_str + Logger.info("Updated Hive2/hive-site.xml 'hive.exec.post.hooks' value from : '{0}' to : '{1}'" + .format(existing_hive_exec_post_hooks, updated_hooks_str)) + else: + Logger.info("No change done to Hive2/hive-site.xml 'hive.exec.post.hooks' value.") + else: + Logger.debug("'hive.exec.post.hooks' doesn't exist in Hive2/hive-site.xml") http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_metastore.py new file mode 100644 index 0000000..8b69e45 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive_metastore.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute, Directory +from resource_management.libraries.script import Script +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.constants import Direction +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.security_commons import build_expectations +from resource_management.libraries.functions.security_commons import cached_kinit_executor +from resource_management.libraries.functions.security_commons import get_params_from_filesystem +from resource_management.libraries.functions.security_commons import validate_security_config_properties +from resource_management.libraries.functions.security_commons import FILE_TYPE_XML +from resource_management.core.resources.system import File + +from hive import create_metastore_schema, hive, jdbc_connector +from hive_service import hive_service +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst + +# the legacy conf.server location in previous stack versions +LEGACY_HIVE_SERVER_CONF = "/etc/hive/conf.server" + +class HiveMetastore(Script): + def install(self, env): + import params + self.install_packages(env) + + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + + # writing configurations on start required for securtity + self.configure(env) + if params.init_metastore_schema: + create_metastore_schema() # execute without config lock + + hive_service('metastore', action='start', upgrade_type=upgrade_type) + + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + hive_service('metastore', action='stop', upgrade_type=upgrade_type) + + + def configure(self, env): + import params + env.set_params(params) + hive(name = 'metastore') + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class HiveMetastoreWindows(HiveMetastore): + def status(self, env): + import status_params + from resource_management.libraries.functions import check_windows_service_status + check_windows_service_status(status_params.hive_metastore_win_service_name) + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class HiveMetastoreDefault(HiveMetastore): + def get_component_name(self): + return "hive-metastore" + + + def status(self, env): + import status_params + from resource_management.libraries.functions import check_process_status + env.set_params(status_params) + + # Recursively check all existing gmetad pid files + check_process_status(status_params.hive_metastore_pid) + + + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing Metastore Stack Upgrade pre-restart") + import params + + env.set_params(params) + + is_upgrade = params.upgrade_direction == Direction.UPGRADE + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "hive", params.version) + stack_select.select("hive-metastore", params.version) + + if is_upgrade and params.stack_version_formatted_major and \ + check_stack_feature(StackFeature.HIVE_METASTORE_UPGRADE_SCHEMA, params.stack_version_formatted_major): + self.upgrade_schema(env) + + + def security_status(self, env): + import status_params + env.set_params(status_params) + if status_params.security_enabled: + props_value_check = {"hive.server2.authentication": "KERBEROS", + "hive.metastore.sasl.enabled": "true", + "hive.security.authorization.enabled": "true"} + props_empty_check = ["hive.metastore.kerberos.keytab.file", + "hive.metastore.kerberos.principal"] + + props_read_check = ["hive.metastore.kerberos.keytab.file"] + hive_site_props = build_expectations('hive-site', props_value_check, props_empty_check, + props_read_check) + + hive_expectations ={} + hive_expectations.update(hive_site_props) + + security_params = get_params_from_filesystem(status_params.hive_conf_dir, + {'hive-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, hive_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if 'hive-site' not in security_params \ + or 'hive.metastore.kerberos.keytab.file' not in security_params['hive-site'] \ + or 'hive.metastore.kerberos.principal' not in security_params['hive-site']: + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hive_user, + security_params['hive-site']['hive.metastore.kerberos.keytab.file'], + security_params['hive-site']['hive.metastore.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + + + def upgrade_schema(self, env): + """ + Executes the schema upgrade binary. This is its own function because it could + be called as a standalone task from the upgrade pack, but is safe to run it for each + metastore instance. The schema upgrade on an already upgraded metastore is a NOOP. + + The metastore schema upgrade requires a database driver library for most + databases. During an upgrade, it's possible that the library is not present, + so this will also attempt to copy/download the appropriate driver. + + This function will also ensure that configurations are written out to disk before running + since the new configs will most likely not yet exist on an upgrade. + + Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades. + """ + Logger.info("Upgrading Hive Metastore Schema") + import status_params + import params + env.set_params(params) + + # ensure that configurations are written out before trying to upgrade the schema + # since the schematool needs configs and doesn't know how to use the hive conf override + self.configure(env) + + if params.security_enabled: + cached_kinit_executor(status_params.kinit_path_local, + status_params.hive_user, + params.hive_metastore_keytab_path, + params.hive_metastore_principal, + status_params.hostname, + status_params.tmp_dir) + + # ensure that the JDBC drive is present for the schema tool; if it's not + # present, then download it first + if params.hive_jdbc_driver in params.hive_jdbc_drivers_list: + target_directory = format("{stack_root}/{version}/hive/lib") + + # download it if it does not exist + if not os.path.exists(params.source_jdbc_file): + jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar) + + target_directory_and_filename = os.path.join(target_directory, os.path.basename(params.source_jdbc_file)) + + if params.sqla_db_used: + target_native_libs_directory = format("{target_directory}/native/lib64") + + Execute(format("yes | {sudo} cp {jars_in_hive_lib} {target_directory}")) + + Directory(target_native_libs_directory, create_parents = True) + + Execute(format("yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}")) + + Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*")) + else: + # copy the JDBC driver from the older metastore location to the new location only + # if it does not already exist + if not os.path.exists(target_directory_and_filename): + Execute(('cp', params.source_jdbc_file, target_directory), + path=["/bin", "/usr/bin/"], sudo = True) + + File(target_directory_and_filename, mode = 0644) + + # build the schema tool command + binary = format("{hive_schematool_ver_bin}/schematool") + + # the conf.server directory changed locations between stack versions + # since the configurations have not been written out yet during an upgrade + # we need to choose the original legacy location + schematool_hive_server_conf_dir = params.hive_server_conf_dir + if params.current_version is not None: + current_version = format_stack_version(params.current_version) + if not(check_stack_feature(StackFeature.CONFIG_VERSIONING, current_version)): + schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF + + env_dict = { + 'HIVE_CONF_DIR': schematool_hive_server_conf_dir + } + + command = format("{binary} -dbType {hive_metastore_db_type} -upgradeSchema") + Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True) + + def get_log_folder(self): + import params + return params.hive_log_dir + + def get_user(self): + import params + return params.hive_user + + def get_pid_files(self): + import status_params + return [status_params.hive_metastore_pid] + + +if __name__ == "__main__": + HiveMetastore().execute()