bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject [41/52] bigtop git commit: BIGTOP-1406. package Ambari in Bigtop
Date Wed, 22 Mar 2017 06:10:32 GMT
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server_upgrade.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server_upgrade.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server_upgrade.py
deleted file mode 100755
index 478c240..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server_upgrade.py
+++ /dev/null
@@ -1,141 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import re
-from resource_management.core.logger import Logger
-from resource_management.core.exceptions import Fail
-from resource_management.core.resources.system import Execute
-from resource_management.core import shell
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.version import format_stack_version
-
-
-def post_upgrade_deregister():
-  """
-  Runs the "hive --service hiveserver2 --deregister <version>" command to
-  de-provision the server in preparation for an upgrade. This will contact
-  ZooKeeper to remove the server so that clients that attempt to connect
-  will be directed to other servers automatically. Once all
-  clients have drained, the server will shutdown automatically; this process
-  could take a very long time.
-  This function will obtain the Kerberos ticket if security is enabled.
-  :return:
-  """
-  import params
-
-  Logger.info('HiveServer2 executing "deregister" command to complete upgrade...')
-
-  if params.security_enabled:
-    kinit_command=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ")
-    Execute(kinit_command,user=params.smokeuser)
-
-  # calculate the current hive server version
-  current_hiveserver_version = _get_current_hiveserver_version()
-  if current_hiveserver_version is None:
-    raise Fail('Unable to determine the current HiveServer2 version to deregister.')
-
-  # fallback when upgrading because <stack-root>/current/hive-server2/conf/conf.server may not exist
-  hive_server_conf_dir = params.hive_server_conf_dir
-  if not os.path.exists(hive_server_conf_dir):
-    hive_server_conf_dir = "/etc/hive/conf.server"
-
-  # deregister
-  hive_execute_path = params.execute_path
-  # If upgrading, the upgrade-target hive binary should be used to call the --deregister command.
-  # If downgrading, the downgrade-source hive binary should be used to call the --deregister command.
-  # By now <stack-selector-tool> has been called to set 'current' to target-stack
-  if "downgrade" == params.upgrade_direction:
-    # hive_bin
-    downgrade_version = params.current_version
-    if params.downgrade_from_version:
-      downgrade_version = params.downgrade_from_version
-    hive_execute_path = _get_hive_execute_path(downgrade_version)
-
-  command = format('hive --config {hive_server_conf_dir} --service hiveserver2 --deregister ' + current_hiveserver_version)
-  Execute(command, user=params.hive_user, path=hive_execute_path, tries=1 )
-
-
-def _get_hive_execute_path(stack_version_formatted):
-  """
-  Returns the exact execute path to use for the given stack-version.
-  This method does not return the "current" path
-  :param stack_version_formatted: Exact stack-version to use in the new path
-  :return: Hive execute path for the exact stack-version
-  """
-  import params
-
-  hive_execute_path = params.execute_path
-  formatted_stack_version = format_stack_version(stack_version_formatted)
-  if formatted_stack_version and check_stack_feature(StackFeature.ROLLING_UPGRADE, formatted_stack_version):
-    # hive_bin
-    new_hive_bin = format('{stack_root}/{stack_version_formatted}/hive/bin')
-    if (os.pathsep + params.hive_bin) in hive_execute_path:
-      hive_execute_path = hive_execute_path.replace(os.pathsep + params.hive_bin, os.pathsep + new_hive_bin)
-    # hadoop_bin_dir
-    new_hadoop_bin = stack_select.get_hadoop_dir_for_stack_version("bin", stack_version_formatted)
-    old_hadoop_bin = params.hadoop_bin_dir
-    if new_hadoop_bin and len(new_hadoop_bin) > 0 and (os.pathsep + old_hadoop_bin) in hive_execute_path:
-      hive_execute_path = hive_execute_path.replace(os.pathsep + old_hadoop_bin, os.pathsep + new_hadoop_bin)
-  return hive_execute_path
-
-
-def _get_current_hiveserver_version():
-  """
-  Runs "hive --version" and parses the result in order
-  to obtain the current version of hive.
-
-  :return:  the hiveserver2 version, returned by "hive --version"
-  """
-  import params
-
-  try:
-    # When downgrading the source version should be the version we are downgrading from
-    if "downgrade" == params.upgrade_direction:
-      if not params.downgrade_from_version:
-        raise Fail('The version from which we are downgrading from should be provided in \'downgrade_from_version\'')
-      source_version = params.downgrade_from_version
-    else:
-      source_version = params.current_version
-    hive_execute_path = _get_hive_execute_path(source_version)
-    version_hive_bin = params.hive_bin
-    formatted_source_version = format_stack_version(source_version)
-    if formatted_source_version and check_stack_feature(StackFeature.ROLLING_UPGRADE, formatted_source_version):
-      version_hive_bin = format('{stack_root}/{source_version}/hive/bin')
-    command = format('{version_hive_bin}/hive --version')
-    return_code, output = shell.call(command, user=params.hive_user, path=hive_execute_path)
-  except Exception, e:
-    Logger.error(str(e))
-    raise Fail('Unable to execute hive --version command to retrieve the hiveserver2 version.')
-
-  if return_code != 0:
-    raise Fail('Unable to determine the current HiveServer2 version because of a non-zero return code of {0}'.format(str(return_code)))
-
-  match = re.search('^(Hive) ([0-9]+.[0-9]+.\S+)', output, re.MULTILINE)
-
-  if match:
-    current_hive_server_version = match.group(2)
-    return current_hive_server_version
-  else:
-    raise Fail('The extracted hiveserver2 version "{0}" does not matching any known pattern'.format(output))
-
-

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_service.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_service.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_service.py
deleted file mode 100755
index 22b4061..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_service.py
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import time
-
-from ambari_commons.constants import UPGRADE_TYPE_ROLLING
-from resource_management.core.logger import Logger
-from resource_management.core import shell
-from resource_management.libraries.functions.format import format
-from resource_management.core.resources.system import File, Execute
-from resource_management.core.resources.service import Service
-from resource_management.core.exceptions import Fail
-from resource_management.core.shell import as_user
-from resource_management.libraries.functions.hive_check import check_thrift_port_sasl
-from resource_management.libraries.functions import get_user_call_output
-from resource_management.libraries.functions.show_logs import show_logs
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from ambari_commons import OSConst
-
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def hive_service(name, action='start', upgrade_type=None):
-  import params
-  if name == 'metastore':
-    if action == 'start' or action == 'stop':
-      Service(params.hive_metastore_win_service_name, action=action)
-
-  if name == 'hiveserver2':
-    if action == 'start' or action == 'stop':
-      Service(params.hive_server_win_service_name, action=action)
-
-
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def hive_service(name, action='start', upgrade_type=None):
-
-  import params
-
-  if name == 'metastore':
-    pid_file = format("{hive_pid_dir}/{hive_metastore_pid}")
-    cmd = format("{start_metastore_path} {hive_log_dir}/hive.out {hive_log_dir}/hive.err {pid_file} {hive_server_conf_dir} {hive_log_dir}")
-  elif name == 'hiveserver2':
-    pid_file = format("{hive_pid_dir}/{hive_pid}")
-    cmd = format("{start_hiveserver2_path} {hive_log_dir}/hive-server2.out {hive_log_dir}/hive-server2.err {pid_file} {hive_server_conf_dir} {hive_log_dir}")
-
-
-    if params.security_enabled and params.current_version and check_stack_feature(StackFeature.HIVE_SERVER2_KERBERIZED_ENV, params.current_version):
-      hive_kinit_cmd = format("{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; ")
-      Execute(hive_kinit_cmd, user=params.hive_user)
-
-  pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"), user=params.hive_user, is_checked_call=False)[1]
-  process_id_exists_command = format("ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")
-
-  if action == 'start':
-    if name == 'hiveserver2':
-      check_fs_root(params.hive_server_conf_dir, params.execute_path)
-
-    daemon_cmd = cmd
-    hadoop_home = params.hadoop_home
-    hive_bin = "hive"
-
-    # upgrading hiveserver2 (rolling_restart) means that there is an existing,
-    # de-registering hiveserver2; the pid will still exist, but the new
-    # hiveserver is spinning up on a new port, so the pid will be re-written
-    if upgrade_type == UPGRADE_TYPE_ROLLING:
-      process_id_exists_command = None
-
-      if params.version and params.stack_root:
-        hadoop_home = format("{stack_root}/{version}/hadoop")
-        hive_bin = os.path.join(params.hive_bin, hive_bin)
-      
-    Execute(daemon_cmd, 
-      user = params.hive_user,
-      environment = { 'HADOOP_HOME': hadoop_home, 'JAVA_HOME': params.java64_home, 'HIVE_BIN': hive_bin },
-      path = params.execute_path,
-      not_if = process_id_exists_command)
-
-    if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or \
-       params.hive_jdbc_driver == "org.postgresql.Driver" or \
-       params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
-
-      validation_called = False
-
-      if params.hive_jdbc_target is not None:
-        validation_called = True
-        validate_connection(params.hive_jdbc_target, params.hive_lib)
-      if params.hive2_jdbc_target is not None:
-        validation_called = True
-        validate_connection(params.hive2_jdbc_target, params.hive_server2_hive2_lib)
-
-      if not validation_called:
-        emessage = "ERROR! DB connection check should be executed at least one time!"
-        Logger.error(emessage)
-
-  elif action == 'stop':
-
-    daemon_kill_cmd = format("{sudo} kill {pid}")
-    daemon_hard_kill_cmd = format("{sudo} kill -9 {pid}")
-
-    Execute(daemon_kill_cmd,
-      not_if = format("! ({process_id_exists_command})")
-    )
-
-    wait_time = 5
-    Execute(daemon_hard_kill_cmd,
-      not_if = format("! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )"),
-      ignore_failures = True
-    )
-
-    try:
-      # check if stopped the process, else fail the task
-      Execute(format("! ({process_id_exists_command})"),
-        tries=20,
-        try_sleep=3,
-      )
-    except:
-      show_logs(params.hive_log_dir, params.hive_user)
-      raise
-
-    File(pid_file,
-         action = "delete"
-    )
-
-def validate_connection(target_path_to_jdbc, hive_lib_path):
-  import params
-
-  path_to_jdbc = target_path_to_jdbc
-  if not params.jdbc_jar_name:
-    path_to_jdbc = format("{hive_lib_path}/") + \
-                   params.default_connectors_map[params.hive_jdbc_driver] if params.hive_jdbc_driver in params.default_connectors_map else None
-    if not os.path.isfile(path_to_jdbc):
-      path_to_jdbc = format("{hive_lib_path}/") + "*"
-      error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
-                      " in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
-      Logger.error(error_message)
-
-  db_connection_check_command = format(
-    "{java64_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
-
-  try:
-    Execute(db_connection_check_command,
-            path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10)
-  except:
-    show_logs(params.hive_log_dir, params.hive_user)
-    raise
-
-
-def check_fs_root(conf_dir, execution_path):
-  import params
-
-  if not params.fs_root.startswith("hdfs://"):
-    Logger.info("Skipping fs root check as fs_root does not start with hdfs://")
-    return
-
-  metatool_cmd = format("hive --config {conf_dir} --service metatool")
-  cmd = as_user(format("{metatool_cmd} -listFSRoot", env={'PATH': execution_path}), params.hive_user) \
-        + format(" 2>/dev/null | grep hdfs:// | cut -f1,2,3 -d '/' | grep -v '{fs_root}' | head -1")
-  code, out = shell.call(cmd)
-
-  if code == 0 and out.strip() != "" and params.fs_root.strip() != out.strip():
-    out = out.strip()
-    cmd = format("{metatool_cmd} -updateLocation {fs_root} {out}")
-    Execute(cmd,
-            user=params.hive_user,
-            environment={'PATH': execution_path}
-    )
-

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_service_interactive.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_service_interactive.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_service_interactive.py
deleted file mode 100755
index eaf95ad..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_service_interactive.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-# Python Imports
-
-# Ambari Commons & Resource Management imports
-import os
-from resource_management.core.logger import Logger
-from resource_management.libraries.functions.format import format
-from resource_management.core.resources.system import File, Execute
-from resource_management.libraries.functions import get_user_call_output
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from ambari_commons import OSConst
-
-# Local Imports
-from hive_service import check_fs_root
-
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def hive_service_interactive(name, action='start', upgrade_type=None):
-  pass
-
-
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def hive_service_interactive(name, action='start', upgrade_type=None):
-  import params
-
-  pid_file = format("{hive_pid_dir}/{hive_interactive_pid}")
-  cmd = format("{start_hiveserver2_interactive_path} {hive_pid_dir}/hive-server2-interactive.out {hive_log_dir}/hive-server2-interactive.err {pid_file} {hive_server_interactive_conf_dir} {hive_log_dir}")
-
-  pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"), user=params.hive_user, is_checked_call=False)[1]
-  process_id_exists_command = format("ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")
-
-  if action == 'start':
-    check_fs_root(params.hive_server_interactive_conf_dir, params.execute_path_hive_interactive)
-    daemon_cmd = cmd
-    hadoop_home = params.hadoop_home
-    hive_interactive_bin = "hive2"
-
-    Execute(daemon_cmd,
-            user = params.hive_user,
-            environment = { 'HADOOP_HOME': hadoop_home, 'JAVA_HOME': params.java64_home, 'HIVE_BIN': hive_interactive_bin },
-            path = params.execute_path,
-            not_if = process_id_exists_command)
-
-    if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or \
-        params.hive_jdbc_driver == "org.postgresql.Driver" or \
-        params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
-
-      path_to_jdbc = params.target_hive_interactive
-      if not params.jdbc_jar_name:
-        path_to_jdbc = format("{hive_interactive_lib}/") + \
-                       params.default_connectors_map[params.hive_jdbc_driver] if params.hive_jdbc_driver in params.default_connectors_map else None
-        if not os.path.isfile(path_to_jdbc):
-          path_to_jdbc = format("{hive_interactive_lib}/") + "*"
-          error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
-                " in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
-          Logger.error(error_message)
-
-      db_connection_check_command = format(
-        "{java64_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
-      Execute(db_connection_check_command,
-              path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10)
-  elif action == 'stop':
-
-    daemon_kill_cmd = format("{sudo} kill {pid}")
-    daemon_hard_kill_cmd = format("{sudo} kill -9 {pid}")
-
-    Execute(daemon_kill_cmd,
-            not_if = format("! ({process_id_exists_command})")
-            )
-
-    # check if stopped the process, otherwise send hard kill command.
-    try:
-      Execute(format("! ({process_id_exists_command})"),
-              tries=10,
-              try_sleep=3,
-              )
-    except:
-      Execute(daemon_hard_kill_cmd,
-              not_if = format("! ({process_id_exists_command}) ")
-              )
-
-    # check if stopped the process, else fail the task
-    Execute(format("! ({process_id_exists_command})"),
-            tries=20,
-            try_sleep=3,
-            )
-
-    File(pid_file,
-         action = "delete"
-         )

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_server.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_server.py
deleted file mode 100755
index 851dc02..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_server.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import sys
-import os
-import mysql_users
-from resource_management import *
-
-from mysql_service import mysql_service
-from mysql_utils import mysql_configure
-
-
-class MysqlServer(Script):
-  def install(self, env):
-    import params
-    self.install_packages(env)
-    self.configure(env)
-
-  def clean(self, env):
-    import params
-    env.set_params(params)
-    mysql_users.mysql_deluser()
-
-  def configure(self, env):
-    import params
-    env.set_params(params)
-    mysql_configure()
-
-  def start(self, env, rolling_restart=False):
-    import params
-    env.set_params(params)
-    mysql_service(daemon_name=params.daemon_name, action='start')
-
-  def stop(self, env, rolling_restart=False):
-    import params
-    env.set_params(params)
-    mysql_service(daemon_name=params.daemon_name, action='stop')
-
-  def status(self, env):
-    import status_params
-    env.set_params(status_params)
-
-    mysql_service(daemon_name=status_params.daemon_name, action='status')
-
-
-if __name__ == "__main__":
-  MysqlServer().execute()

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_service.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_service.py
deleted file mode 100755
index 8b98ed1..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_service.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-
-
-def mysql_service(daemon_name=None, action='start'): 
-  status_cmd = format("pgrep -l '^{process_name}$'")
-  cmd = ('service', daemon_name, action)
-
-  if action == 'status':
-    try:
-      Execute(status_cmd)
-    except Fail:
-      raise ComponentIsNotRunning()
-  elif action == 'stop':
-    import params
-    Execute(cmd,
-            logoutput = True,
-            only_if = status_cmd,
-            sudo = True,
-    )
-  elif action == 'start':
-    import params   
-    Execute(cmd,
-      logoutput = True,
-      not_if = status_cmd,
-      sudo = True,
-    )
-
-
-

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_users.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_users.py
deleted file mode 100755
index c023548..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_users.py
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-
-# Used to add hive access to the needed components
-def mysql_adduser():
-  import params
-  
-  File(params.mysql_adduser_path,
-       mode=0755,
-       content=StaticFile('addMysqlUser.sh')
-  )
-  hive_server_host = format("{hive_server_host}")
-  hive_metastore_host = format("{hive_metastore_host}")
-
-  add_metastore_cmd = "bash -x {mysql_adduser_path} {daemon_name} {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_metastore_host}"
-  add_hiveserver_cmd = "bash -x {mysql_adduser_path} {daemon_name} {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_server_host}"
-  if (hive_server_host == hive_metastore_host):
-    cmd = format(add_hiveserver_cmd)
-  else:
-    cmd = format(add_hiveserver_cmd + ";" + add_metastore_cmd)
-  Execute(cmd,
-          tries=3,
-          try_sleep=5,
-          logoutput=False,
-          path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
-  )
-
-# Removes hive access from components
-def mysql_deluser():
-  import params
-  
-  File(params.mysql_deluser_path,
-       mode=0755,
-       content=StaticFile('removeMysqlUser.sh')
-  )
-  hive_server_host = format("{hive_server_host}")
-  hive_metastore_host = format("{hive_metastore_host}")
-
-  del_hiveserver_cmd = "bash -x {mysql_deluser_path} {daemon_name} {hive_metastore_user_name} {hive_server_host}"
-  del_metastore_cmd = "bash -x {mysql_deluser_path} {daemon_name} {hive_metastore_user_name} {hive_metastore_host}"
-  if (hive_server_host == hive_metastore_host):
-    cmd = format(del_hiveserver_cmd)
-  else:
-    cmd = format(
-      del_hiveserver_cmd + ";" + del_metastore_cmd)
-  Execute(cmd,
-          tries=3,
-          try_sleep=5,
-          path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-  )
-

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_utils.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_utils.py
deleted file mode 100755
index 5006b56..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/mysql_utils.py
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-import mysql_users
-
-def mysql_configure():
-  import params
-
-  # required for running hive
-  replace_bind_address = ('sed','-i','s|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|',params.mysql_configname)
-  Execute(replace_bind_address,
-          sudo = True,
-  )
-  
-  # this also will start mysql-server
-  mysql_users.mysql_adduser()
-  
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params.py
deleted file mode 100755
index f10a3f3..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params.py
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from ambari_commons import OSCheck
-from resource_management.libraries.functions.default import default
-
-if OSCheck.is_windows_family():
-  from params_windows import *
-else:
-  from params_linux import *
-
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
-retryAble = default("/commandParams/command_retry_enabled", False)

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params_linux.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params_linux.py
deleted file mode 100755
index 9d79e12..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params_linux.py
+++ /dev/null
@@ -1,735 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import status_params
-import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
-import os
-
-from urlparse import urlparse
-
-from ambari_commons.constants import AMBARI_SUDO_BINARY
-from ambari_commons.os_check import OSCheck
-
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions.is_empty import is_empty
-from resource_management.libraries.functions.copy_tarball import STACK_ROOT_PATTERN, STACK_NAME_PATTERN, STACK_VERSION_PATTERN
-from resource_management.libraries.functions import get_kinit_path
-from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.stack_features import get_stack_feature_version
-from resource_management.libraries.functions.get_port_from_url import get_port_from_url
-from resource_management.libraries.functions.expect import expect
-from resource_management.libraries import functions
-from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
-
-# Default log4j version; put config files under /etc/hive/conf
-log4j_version = '1'
-
-# server configurations
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-sudo = AMBARI_SUDO_BINARY
-
-stack_root = status_params.stack_root
-stack_name = status_params.stack_name
-stack_name_uppercase = stack_name.upper()
-agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
-agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
-
-# Needed since this is an Atlas Hook service.
-cluster_name = config['clusterName']
-
-# node hostname
-hostname = config["hostname"]
-
-# This is expected to be of the form #.#.#.#
-stack_version_unformatted = status_params.stack_version_unformatted
-stack_version_formatted_major = status_params.stack_version_formatted_major
-
-# this is not available on INSTALL action because <stack-selector-tool> is not available
-stack_version_formatted = functions.get_stack_version('hive-server2')
-
-# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade.
-# It cannot be used during the initial Cluser Install because the version is not yet known.
-version = default("/commandParams/version", None)
-
-# current host stack version
-current_version = default("/hostLevelParams/current_version", None)
-
-# When downgrading the 'version' and 'current_version' are both pointing to the downgrade-target version
-# downgrade_from_version provides the source-version the downgrade is happening from
-downgrade_from_version = default("/commandParams/downgrade_from_version", None)
-
-# get the correct version to use for checking stack features
-version_for_stack_feature_checks = get_stack_feature_version(config)
-
-# Upgrade direction
-upgrade_direction = default("/commandParams/upgrade_direction", None)
-stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
-stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
-
-# component ROLE directory (like hive-metastore or hive-server2-hive2)
-component_directory = status_params.component_directory
-component_directory_interactive = status_params.component_directory_interactive
-
-hadoop_home = '/usr/lib/hadoop'
-hive_bin = '/usr/lib/hive/bin'
-hive_schematool_ver_bin = '/usr/lib/hive/bin'
-hive_schematool_bin = '/usr/lib/hive/bin'
-hive_lib = '/usr/lib/hive/lib'
-hive_version_lib = '/usr/lib/hive/lib'
-#hadoop_home = format('{stack_root}/current/hadoop-client')
-#hive_bin = format('{stack_root}/current/{component_directory}/bin')
-#hive_schematool_ver_bin = format('{stack_root}/{version}/hive/bin')
-#hive_schematool_bin = format('{stack_root}/current/{component_directory}/bin')
-#hive_lib = format('{stack_root}/current/{component_directory}/lib')
-#hive_version_lib = format('{stack_root}/{version}/hive/lib')
-hive_var_lib = '/var/lib/hive'
-hive_user_home_dir = "/home/hive"
-
-# starting on stacks where HSI is supported, we need to begin using the 'hive2' schematool
-hive_server2_hive2_dir = None
-hive_server2_hive2_lib = None
-
-version = default("/commandParams/version", None)
-
-if check_stack_feature(StackFeature.HIVE_SERVER_INTERACTIVE, version_for_stack_feature_checks):
-  # the name of the hiveserver2-hive2 component
-  hive_server2_hive2_component = status_params.SERVER_ROLE_DIRECTORY_MAP["HIVE_SERVER_INTERACTIVE"]
-
-  # when using the version, we can just specify the component as "hive2"
-  hive_schematool_ver_bin = format('{stack_root}/{version}/hive2/bin')
-
-  # use the schematool which ships with hive2
-  hive_schematool_bin = format('{stack_root}/current/{hive_server2_hive2_component}/bin')
-
-  # <stack-root>/<version>/hive2 (as opposed to <stack-root>/<version>/hive)
-  hive_server2_hive2_dir = format('{stack_root}/current/{hive_server2_hive2_component}')
-
-  # <stack-root>/<version>/hive2 (as opposed to <stack-root>/<version>/hive)
-  hive_server2_hive2_version_dir = format('{stack_root}/{version}/hive2')
-
-  # <stack-root>/current/hive-server2-hive2/lib -> <stack-root>/<version>/hive2/lib
-  hive_server2_hive2_lib = format('{hive_server2_hive2_dir}/lib')
-
-  # <stack-root>/<version>/hive2/lib
-  hive_server2_hive2_version_lib = format('{hive_server2_hive2_version_dir}/lib')
-
-
-hive_interactive_bin = format('{stack_root}/current/{component_directory_interactive}/bin')
-hive_interactive_lib = format('{stack_root}/current/{component_directory_interactive}/lib')
-
-# Hive Interactive related paths
-hive_interactive_var_lib = '/var/lib/hive2'
-
-# These tar folders were used in previous stack versions, e.g., HDP 2.1
-hadoop_streaming_jars = '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar'
-pig_tar_file = format('/usr/share/{stack_name_uppercase}-webhcat/pig.tar.gz')
-hive_tar_file = format('/usr/share/{stack_name_uppercase}-webhcat/hive.tar.gz')
-sqoop_tar_file = format('/usr/share/{stack_name_uppercase}-webhcat/sqoop*.tar.gz')
-
-hive_metastore_site_supported = False
-hive_etc_dir_prefix = "/etc/hive"
-hive_interactive_etc_dir_prefix = "/etc/hive2"
-limits_conf_dir = "/etc/security/limits.d"
-
-hive_user_nofile_limit = default("/configurations/hive-env/hive_user_nofile_limit", "32000")
-hive_user_nproc_limit = default("/configurations/hive-env/hive_user_nproc_limit", "16000")
-
-# use the directories from status_params as they are already calculated for
-# the correct stack version
-hadoop_conf_dir = status_params.hadoop_conf_dir
-hadoop_bin_dir = status_params.hadoop_bin_dir
-webhcat_conf_dir = status_params.webhcat_conf_dir
-hive_conf_dir = status_params.hive_conf_dir
-hive_home_dir = status_params.hive_home_dir
-hive_config_dir = status_params.hive_config_dir
-hive_client_conf_dir = status_params.hive_client_conf_dir
-hive_server_conf_dir = status_params.hive_server_conf_dir
-
-hcat_conf_dir = '/etc/hive-hcatalog/conf'
-config_dir = '/etc/hive-webhcat/conf'
-hcat_lib = '/usr/lib/hive-hcatalog/share/hcatalog'
-webhcat_bin_dir = '/usr/lib/hive-hcatalog/sbin'
-
-# --- Tarballs ---
-# DON'T CHANGE THESE VARIABLE NAMES
-# Values don't change from those in copy_tarball.py
-webhcat_apps_dir = "/apps/webhcat"
-hive_tar_source = "{0}/{1}/hive/hive.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN)
-pig_tar_source = "{0}/{1}/pig/pig.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN)
-hive_tar_dest_file = "/{0}/apps/{1}/hive/hive.tar.gz".format(STACK_NAME_PATTERN,STACK_VERSION_PATTERN)
-pig_tar_dest_file = "/{0}/apps/{1}/pig/pig.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)
-
-hadoop_streaming_tar_source = "{0}/{1}/hadoop-mapreduce/hadoop-streaming.jar".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN)
-sqoop_tar_source = "{0}/{1}/sqoop/sqoop.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN)
-hadoop_streaming_tar_dest_dir = "/{0}/apps/{1}/mapreduce/".format(STACK_NAME_PATTERN,STACK_VERSION_PATTERN)
-sqoop_tar_dest_dir = "/{0}/apps/{1}/sqoop/".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)
-
-tarballs_mode = 0444
-
-purge_tables = "false"
-# Starting from stack version for feature hive_purge_table drop should be executed with purge
-purge_tables = 'true'
-
-# this is NOT a typo.  Configs for hcatalog/webhcat point to a
-# specific directory which is NOT called 'conf'
-# FIXME: ODPi
-# hcat_conf_dir = format('{stack_root}/current/hive-webhcat/etc/hcatalog')
-# config_dir = format('{stack_root}/current/hive-webhcat/etc/webhcat')
-hcat_conf_dir = format('/etc/hive-hcatalog/conf')
-config_dir = format('/etc/hive-webhcat/conf')
-
-hive_metastore_site_supported = True
-
-execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir
-
-hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName']
-hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL']
-
-hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword']
-hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd
-hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type']
-
-#HACK Temporarily use dbType=azuredb while invoking schematool
-if hive_metastore_db_type == "mssql":
-  hive_metastore_db_type = "azuredb"
-
-#users
-hive_user = config['configurations']['hive-env']['hive_user']
-
-#JDBC driver jar name
-hive_jdbc_driver = config['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName']
-jdk_location = config['hostLevelParams']['jdk_location']
-java_share_dir = '/usr/share/java'
-hive_database_name = config['configurations']['hive-env']['hive_database_name']
-hive_database = config['configurations']['hive-env']['hive_database']
-hive_use_existing_db = hive_database.startswith('Existing')
-
-default_connectors_map = { "com.microsoft.sqlserver.jdbc.SQLServerDriver":"sqljdbc4.jar",
-                           "com.mysql.jdbc.Driver":"mysql-connector-java.jar",
-                           "org.postgresql.Driver":"postgresql-jdbc.jar",
-                           "oracle.jdbc.driver.OracleDriver":"ojdbc.jar",
-                           "sap.jdbc4.sqlanywhere.IDriver":"sajdbc4.jar"}
-
-# NOT SURE THAT IT'S A GOOD IDEA TO USE PATH TO CLASS IN DRIVER, MAYBE IT WILL BE BETTER TO USE DB TYPE.
-# BECAUSE PATH TO CLASSES COULD BE CHANGED
-sqla_db_used = False
-hive_previous_jdbc_jar_name = None
-if hive_jdbc_driver == "com.microsoft.sqlserver.jdbc.SQLServerDriver":
-  jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
-  hive_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
-elif hive_jdbc_driver == "com.mysql.jdbc.Driver":
-  jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
-  hive_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
-elif hive_jdbc_driver == "org.postgresql.Driver":
-  jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
-  hive_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
-elif hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
-  jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
-  hive_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
-elif hive_jdbc_driver == "sap.jdbc4.sqlanywhere.IDriver":
-  jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
-  hive_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
-  sqla_db_used = True
-
-default_mysql_jar_name = "mysql-connector-java.jar"
-default_mysql_target = format("{hive_lib}/{default_mysql_jar_name}")
-hive_previous_jdbc_jar = format("{hive_lib}/{hive_previous_jdbc_jar_name}")
-if not hive_use_existing_db:
-  jdbc_jar_name = default_mysql_jar_name
-
-
-downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}")
-
-hive_jdbc_target = format("{hive_lib}/{jdbc_jar_name}")
-hive2_jdbc_target = None
-if hive_server2_hive2_dir:
-  hive2_jdbc_target = format("{hive_server2_hive2_lib}/{jdbc_jar_name}")
-
-# during upgrade / downgrade, use the specific version to copy the JDBC JAR to
-if upgrade_direction:
-  hive_jdbc_target = format("{hive_version_lib}/{jdbc_jar_name}")
-  hive2_jdbc_target = format("{hive_server2_hive2_version_lib}/{jdbc_jar_name}") if hive2_jdbc_target is not None else None
-
-
-hive2_previous_jdbc_jar = format("{hive_server2_hive2_lib}/{hive_previous_jdbc_jar_name}") if hive_server2_hive2_lib is not None else None
-driver_curl_source = format("{jdk_location}/{jdbc_jar_name}")
-
-# normally, the JDBC driver would be referenced by <stack-root>/current/.../foo.jar
-# but in RU if <stack-selector-tool> is called and the restart fails, then this means that current pointer
-# is now pointing to the upgraded version location; that's bad for the cp command
-source_jdbc_file = format("{stack_root}/{current_version}/hive/lib/{jdbc_jar_name}")
-
-check_db_connection_jar_name = "DBConnectionVerification.jar"
-check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}")
-hive_jdbc_drivers_list = ["com.microsoft.sqlserver.jdbc.SQLServerDriver","com.mysql.jdbc.Driver",
-                          "org.postgresql.Driver","oracle.jdbc.driver.OracleDriver","sap.jdbc4.sqlanywhere.IDriver"]
-
-prepackaged_jdbc_name = "ojdbc6.jar"
-prepackaged_ojdbc_symlink = format("{hive_lib}/{prepackaged_jdbc_name}")
-templeton_port = config['configurations']['webhcat-site']['templeton.port']
-
-#constants for type2 jdbc
-jdbc_libs_dir = format("{hive_lib}/native/lib64")
-lib_dir_available = os.path.exists(jdbc_libs_dir)
-
-if sqla_db_used:
-  jars_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/java/*")
-  libs_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/native/lib64/*")
-  downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}")
-  libs_in_hive_lib = format("{jdbc_libs_dir}/*")
-
-
-# Start, Common Hosts and Ports
-ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
-
-hive_metastore_hosts = default('/clusterHostInfo/hive_metastore_host', [])
-hive_metastore_host = hive_metastore_hosts[0] if len(hive_metastore_hosts) > 0 else None
-hive_metastore_port = get_port_from_url(config['configurations']['hive-site']['hive.metastore.uris'])
-
-hive_server_hosts = default("/clusterHostInfo/hive_server_host", [])
-hive_server_host = hive_server_hosts[0] if len(hive_server_hosts) > 0 else None
-
-hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', [])
-hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None
-# End, Common Hosts and Ports
-
-hive_transport_mode = config['configurations']['hive-site']['hive.server2.transport.mode']
-
-if hive_transport_mode.lower() == "http":
-  hive_server_port = config['configurations']['hive-site']['hive.server2.thrift.http.port']
-else:
-  hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000")
-
-hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}")
-hive_http_endpoint = default('/configurations/hive-site/hive.server2.thrift.http.path', "cliservice")
-hive_server_principal = config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal']
-hive_server2_authentication = config['configurations']['hive-site']['hive.server2.authentication']
-
-# ssl options
-hive_ssl = default('/configurations/hive-site/hive.server2.use.SSL', False)
-hive_ssl_keystore_path = default('/configurations/hive-site/hive.server2.keystore.path', None)
-hive_ssl_keystore_password = default('/configurations/hive-site/hive.server2.keystore.password', None)
-
-smokeuser = config['configurations']['cluster-env']['smokeuser']
-smoke_test_sql = format("{tmp_dir}/hiveserver2.sql")
-smoke_test_path = format("{tmp_dir}/hiveserver2Smoke.sh")
-smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
-
-fs_root = config['configurations']['core-site']['fs.defaultFS']
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-
-kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-hive_metastore_keytab_path = config['configurations']['hive-site']['hive.metastore.kerberos.keytab.file']
-hive_metastore_principal = config['configurations']['hive-site']['hive.metastore.kerberos.principal']
-
-hive_server2_keytab = config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab']
-
-#hive_env
-hive_log_dir = config['configurations']['hive-env']['hive_log_dir']
-hive_pid_dir = status_params.hive_pid_dir
-hive_pid = status_params.hive_pid
-hive_interactive_pid = status_params.hive_interactive_pid
-
-#Default conf dir for client
-hive_conf_dirs_list = [hive_client_conf_dir]
-
-# These are the folders to which the configs will be written to.
-ranger_hive_component = status_params.SERVER_ROLE_DIRECTORY_MAP['HIVE_SERVER']
-if status_params.role == "HIVE_METASTORE" and hive_metastore_hosts is not None and hostname in hive_metastore_hosts:
-  hive_conf_dirs_list.append(hive_server_conf_dir)
-elif status_params.role == "HIVE_SERVER" and hive_server_hosts is not None and hostname in hive_server_host:
-  hive_conf_dirs_list.append(hive_server_conf_dir)
-elif status_params.role == "HIVE_SERVER_INTERACTIVE" and hive_server_interactive_hosts is not None and hostname in hive_server_interactive_hosts:
-  hive_conf_dirs_list.append(status_params.hive_server_interactive_conf_dir)
-  ranger_hive_component = status_params.SERVER_ROLE_DIRECTORY_MAP['HIVE_SERVER_INTERACTIVE']
-# log4j version is 2 for hive2; put config files under /etc/hive2/conf
-if status_params.role == "HIVE_SERVER_INTERACTIVE":
-  log4j_version = '2'
-
-#Starting hiveserver2
-start_hiveserver2_script = 'startHiveserver2.sh.j2'
-
-##Starting metastore
-start_metastore_script = 'startMetastore.sh'
-hive_metastore_pid = status_params.hive_metastore_pid
-
-# Hive Server Interactive
-slider_am_container_mb = default("/configurations/hive-interactive-env/slider_am_container_mb", 341)
-
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-yarn_user = config['configurations']['yarn-env']['yarn_user']
-user_group = config['configurations']['cluster-env']['user_group']
-artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
-# Need this for yarn.nodemanager.recovery.dir in yarn-site
-yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix']
-
-target_hive_interactive = format("{hive_interactive_lib}/{jdbc_jar_name}")
-hive_intaractive_previous_jdbc_jar = format("{hive_interactive_lib}/{hive_previous_jdbc_jar_name}")
-jars_in_hive_lib = format("{hive_lib}/*.jar")
-
-start_hiveserver2_path = format("{tmp_dir}/start_hiveserver2_script")
-start_metastore_path = format("{tmp_dir}/start_metastore_script")
-
-hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
-
-if 'role' in config and config['role'] in ["HIVE_SERVER", "HIVE_METASTORE"]:
-  hive_heapsize = config['configurations']['hive-env']['hive.heapsize']
-else:
-  hive_heapsize = config['configurations']['hive-env']['hive.client.heapsize']
-
-hive_metastore_heapsize = config['configurations']['hive-env']['hive.metastore.heapsize']
-
-java64_home = config['hostLevelParams']['java_home']
-java_version = expect("/hostLevelParams/java_version", int)
-
-##### MYSQL
-db_name = config['configurations']['hive-env']['hive_database_name']
-mysql_group = 'mysql'
-mysql_host = config['clusterHostInfo']['hive_mysql_host']
-
-mysql_adduser_path = format("{tmp_dir}/addMysqlUser.sh")
-mysql_deluser_path = format("{tmp_dir}/removeMysqlUser.sh")
-
-#### Metastore
-# initialize the schema only if not in an upgrade/downgrade
-init_metastore_schema = upgrade_direction is None
-
-########## HCAT
-hcat_dbroot = hcat_lib
-
-hcat_user = config['configurations']['hive-env']['hcat_user']
-webhcat_user = config['configurations']['hive-env']['webhcat_user']
-
-hcat_pid_dir = status_params.hcat_pid_dir
-hcat_log_dir = config['configurations']['hive-env']['hcat_log_dir']
-hcat_env_sh_template = config['configurations']['hcat-env']['content']
-
-#hive-log4j.properties.template
-if (('hive-log4j' in config['configurations']) and ('content' in config['configurations']['hive-log4j'])):
-  log4j_props = config['configurations']['hive-log4j']['content']
-else:
-  log4j_props = None
-
-#webhcat-log4j.properties.template
-if (('webhcat-log4j' in config['configurations']) and ('content' in config['configurations']['webhcat-log4j'])):
-  log4j_webhcat_props = config['configurations']['webhcat-log4j']['content']
-else:
-  log4j_webhcat_props = None
-
-#hive-exec-log4j.properties.template
-if (('hive-exec-log4j' in config['configurations']) and ('content' in config['configurations']['hive-exec-log4j'])):
-  log4j_exec_props = config['configurations']['hive-exec-log4j']['content']
-else:
-  log4j_exec_props = None
-
-daemon_name = status_params.daemon_name
-process_name = status_params.process_name
-hive_env_sh_template = config['configurations']['hive-env']['content']
-
-hive_hdfs_user_dir = format("/user/{hive_user}")
-hive_hdfs_user_mode = 0755
-hive_apps_whs_dir = config['configurations']['hive-site']["hive.metastore.warehouse.dir"]
-whs_dir_protocol = urlparse(hive_apps_whs_dir).scheme
-hive_exec_scratchdir = config['configurations']['hive-site']["hive.exec.scratchdir"]
-#for create_hdfs_directory
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', 'missing_principal').replace("_HOST", hostname)
-
-# Tez-related properties
-tez_user = config['configurations']['tez-env']['tez_user']
-
-# Tez jars
-tez_local_api_jars = '/usr/lib/tez/tez*.jar'
-tez_local_lib_jars = '/usr/lib/tez/lib/*.jar'
-
-# Tez libraries
-tez_lib_uris = default("/configurations/tez-site/tez.lib.uris", None)
-
-if OSCheck.is_ubuntu_family():
-  mysql_configname = '/etc/mysql/my.cnf'
-else:
-  mysql_configname = '/etc/my.cnf'
-
-mysql_user = 'mysql'
-
-# Hive security
-hive_authorization_enabled = config['configurations']['hive-site']['hive.security.authorization.enabled']
-
-mysql_jdbc_driver_jar = "/usr/share/java/mysql-connector-java.jar"
-
-hive_site_config = dict(config['configurations']['hive-site'])
-
-########################################################
-############# AMS related params #####################
-########################################################
-ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
-has_metric_collector = not len(ams_collector_hosts) == 0
-if has_metric_collector:
-  if 'cluster-env' in config['configurations'] and \
-      'metrics_collector_vip_host' in config['configurations']['cluster-env']:
-    metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
-  else:
-    metric_collector_host = ams_collector_hosts[0]
-  if 'cluster-env' in config['configurations'] and \
-      'metrics_collector_vip_port' in config['configurations']['cluster-env']:
-    metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
-  else:
-    metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188")
-    if metric_collector_web_address.find(':') != -1:
-      metric_collector_port = metric_collector_web_address.split(':')[1]
-    else:
-      metric_collector_port = '6188'
-  if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
-    metric_collector_protocol = 'https'
-  else:
-    metric_collector_protocol = 'http'
-  metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
-  metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
-  metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
-
-metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60)
-metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10)
-
-########################################################
-############# Atlas related params #####################
-########################################################
-#region Atlas Hooks
-hive_atlas_application_properties = default('/configurations/hive-atlas-application.properties', {})
-
-if has_atlas_in_cluster():
-  atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
-#endregion
-
-########################################################
-########### WebHCat related params #####################
-########################################################
-
-webhcat_env_sh_template = config['configurations']['webhcat-env']['content']
-templeton_log_dir = config['configurations']['hive-env']['hcat_log_dir']
-templeton_pid_dir = status_params.hcat_pid_dir
-
-webhcat_pid_file = status_params.webhcat_pid_file
-
-templeton_jar = config['configurations']['webhcat-site']['templeton.jar']
-
-
-webhcat_server_host = config['clusterHostInfo']['webhcat_server_host']
-
-hcat_hdfs_user_dir = format("/user/{hcat_user}")
-hcat_hdfs_user_mode = 0755
-webhcat_hdfs_user_dir = format("/user/{webhcat_user}")
-webhcat_hdfs_user_mode = 0755
-#for create_hdfs_directory
-security_param = "true" if security_enabled else "false"
-
-
-
-hdfs_site = config['configurations']['hdfs-site']
-default_fs = config['configurations']['core-site']['fs.defaultFS']
-
-dfs_type = default("/commandParams/dfs_type", "")
-
-import functools
-#create partial functions with common arguments for every HdfsResource call
-#to create hdfs directory we need to call params.HdfsResource in code
-HdfsResource = functools.partial(
- HdfsResource,
-  user = hdfs_user,
-  hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
-  security_enabled = security_enabled,
-  keytab = hdfs_user_keytab,
-  kinit_path_local = kinit_path_local,
-  hadoop_bin_dir = hadoop_bin_dir,
-  hadoop_conf_dir = hadoop_conf_dir,
-  principal_name = hdfs_principal_name,
-  hdfs_site = hdfs_site,
-  default_fs = default_fs,
-  immutable_paths = get_not_managed_resources(),
-  dfs_type = dfs_type
- )
-
-# Hive Interactive related
-hive_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', [])
-has_hive_interactive = len(hive_interactive_hosts) > 0
-if has_hive_interactive:
-  llap_daemon_log4j = config['configurations']['llap-daemon-log4j']['content']
-  llap_cli_log4j2 = config['configurations']['llap-cli-log4j2']['content']
-  hive_log4j2 = config['configurations']['hive-log4j2']['content']
-  hive_exec_log4j2 = config['configurations']['hive-exec-log4j2']['content']
-  beeline_log4j2 = config['configurations']['beeline-log4j2']['content']
-
-  hive_server_interactive_conf_dir = status_params.hive_server_interactive_conf_dir
-  execute_path_hive_interactive = os.path.join(os.environ['PATH'], hive_interactive_bin, hadoop_bin_dir)
-  start_hiveserver2_interactive_script = 'startHiveserver2Interactive.sh.j2'
-  start_hiveserver2_interactive_path = format("{tmp_dir}/start_hiveserver2_interactive_script")
-  hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content']
-  hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive', False)
-  llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}')
-
-  # Service check related
-  if hive_transport_mode.lower() == "http":
-    hive_server_interactive_port = config['configurations']['hive-interactive-site']['hive.server2.thrift.http.port']
-  else:
-    hive_server_interactive_port = default('/configurations/hive-interactive-site/hive.server2.thrift.port',"10500")
-  # Tez for Hive interactive related
-  tez_interactive_config_dir = "/etc/tez_hive2/conf"
-  tez_interactive_user = config['configurations']['tez-env']['tez_user']
-  num_retries_for_checking_llap_status = default('/configurations/hive-interactive-env/num_retries_for_checking_llap_status', 10)
-  # Used in LLAP slider package creation
-  num_llap_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes']
-  llap_daemon_container_size = config['configurations']['hive-interactive-site']['hive.llap.daemon.yarn.container.mb']
-  llap_log_level = config['configurations']['hive-interactive-env']['llap_log_level']
-  hive_llap_io_mem_size = config['configurations']['hive-interactive-site']['hive.llap.io.memory.size']
-  llap_heap_size = config['configurations']['hive-interactive-env']['llap_heap_size']
-  llap_app_name = config['configurations']['hive-interactive-env']['llap_app_name']
-  hive_llap_principal = None
-  if security_enabled:
-    hive_llap_keytab_file = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.keytab.file']
-    hive_llap_principal = (config['configurations']['hive-interactive-site']['hive.llap.zk.sm.principal']).replace('_HOST',hostname.lower())
-  pass
-
-# ranger host
-ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
-has_ranger_admin = not len(ranger_admin_hosts) == 0
-xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
-
-#ranger hive properties
-policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
-if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
-  policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
-xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
-xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
-xa_db_host = config['configurations']['admin-properties']['db_host']
-repo_name = str(config['clusterName']) + '_hive'
-
-jdbc_driver_class_name = config['configurations']['ranger-hive-plugin-properties']['jdbc.driverClassName']
-common_name_for_certificate = config['configurations']['ranger-hive-plugin-properties']['common.name.for.certificate']
-
-repo_config_username = config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
-
-ranger_env = config['configurations']['ranger-env']
-ranger_plugin_properties = config['configurations']['ranger-hive-plugin-properties']
-policy_user = config['configurations']['ranger-hive-plugin-properties']['policy_user']
-
-if security_enabled:
-  hive_principal = hive_server_principal.replace('_HOST',hostname.lower())
-  hive_keytab = config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab']
-
-#For curl command in ranger plugin to get db connector
-if has_ranger_admin:
-  enable_ranger_hive = (config['configurations']['hive-env']['hive_security_authorization'].lower() == 'ranger')
-  repo_config_password = unicode(config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
-  xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
-  ranger_previous_jdbc_jar_name = None
-
-  if stack_supports_ranger_audit_db:
-    if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql':
-      ranger_jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
-      ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
-      audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
-      jdbc_driver = "com.mysql.jdbc.Driver"
-    elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle':
-      ranger_jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
-      ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
-      colon_count = xa_db_host.count(':')
-      if colon_count == 2 or colon_count == 0:
-        audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
-      else:
-        audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
-      jdbc_driver = "oracle.jdbc.OracleDriver"
-    elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres':
-      ranger_jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
-      ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
-      audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
-      jdbc_driver = "org.postgresql.Driver"
-    elif xa_audit_db_flavor and xa_audit_db_flavor == 'mssql':
-      ranger_jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
-      ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
-      audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
-      jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
-    elif xa_audit_db_flavor and xa_audit_db_flavor == 'sqla':
-      ranger_jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
-      ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
-      audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
-      jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
-
-  ranger_downloaded_custom_connector = format("{tmp_dir}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
-  ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
-  ranger_driver_curl_target = format("{hive_lib}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
-  ranger_previous_jdbc_jar = format("{hive_lib}/{ranger_previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
-  sql_connector_jar = ''
-
-  hive_ranger_plugin_config = {
-    'username': repo_config_username,
-    'password': repo_config_password,
-    'jdbc.driverClassName': jdbc_driver_class_name,
-    'jdbc.url': format("{hive_url}/default;principal={hive_principal}") if security_enabled else hive_url,
-    'commonNameForCertificate': common_name_for_certificate
-  }
-
-  hive_ranger_plugin_repo = {
-    'isActive': 'true',
-    'config': json.dumps(hive_ranger_plugin_config),
-    'description': 'hive repo',
-    'name': repo_name,
-    'repositoryType': 'hive',
-    'assetType': '3'
-  }
-
-  if stack_supports_ranger_kerberos and security_enabled:
-    hive_ranger_plugin_config['policy.download.auth.users'] = hive_user
-    hive_ranger_plugin_config['tag.download.auth.users'] = hive_user
-    hive_ranger_plugin_config['policy.grantrevoke.auth.users'] = hive_user
-
-  if stack_supports_ranger_kerberos:
-    hive_ranger_plugin_config['ambari.service.check.user'] = policy_user
-
-    hive_ranger_plugin_repo = {
-      'isEnabled': 'true',
-      'configs': hive_ranger_plugin_config,
-      'description': 'hive repo',
-      'name': repo_name,
-      'type': 'hive'
-    }
-
-
-  xa_audit_db_is_enabled = False
-  xa_audit_db_password = ''
-  if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
-    xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
-  ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
-  if xml_configurations_supported and stack_supports_ranger_audit_db:
-    xa_audit_db_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.db']
-  xa_audit_hdfs_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
-  ssl_keystore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
-  ssl_truststore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
-  credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
-
-  #For SQLA explicitly disable audit to DB for Ranger
-  if xa_audit_db_flavor == 'sqla':
-    xa_audit_db_is_enabled = False
-

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params_windows.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params_windows.py
deleted file mode 100755
index 880fdb5..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/params_windows.py
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from status_params import *
-
-# server configurations
-config = Script.get_config()
-
-# This is expected to be of the form #.#.#.#
-stack_version_unformatted = config['hostLevelParams']['stack_version']
-stack_version_formatted = format_stack_version(stack_version_unformatted)
-
-stack_root = None
-hive_conf_dir = None
-hive_home = None
-hive_lib_dir = None
-hive_log_dir = None
-hive_opts = None
-hcat_home = None
-hcat_config_dir = None
-hive_bin = None
-
-try:
-  stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
-  hive_conf_dir = os.environ["HIVE_CONF_DIR"]
-  hive_home = os.environ["HIVE_HOME"]
-  hive_lib_dir = os.environ["HIVE_LIB_DIR"]
-  hive_log_dir = os.environ["HIVE_LOG_DIR"]
-  hive_opts = os.environ["HIVE_OPTS"]
-  hcat_home = os.environ["HCAT_HOME"]
-  hcat_config_dir = os.environ["WEBHCAT_CONF_DIR"]
-  hive_bin = os.path.join(hive_home, "bin")
-except:
-  pass
-
-hive_env_sh_template = config['configurations']['hive-env']['content']
-hive_warehouse_dir = config['configurations']['hive-site']['hive.metastore.warehouse.dir']
-hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
-hive_user = hadoop_user
-hcat_user = hadoop_user
-
-hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type']
-hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName']
-hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword']
-
-hive_execution_engine = config["configurations"]["hive-site"]["hive.execution.engine"]
-
-######## Metastore Schema
-init_metastore_schema = not config['configurations']['hive-site']['datanucleus.autoCreateSchema']
-
-service_map = {
-  "metastore" : hive_metastore_win_service_name,
-  "client" : hive_client_win_service_name,
-  "hiveserver2" : hive_server_win_service_name,
-  "templeton" : webhcat_server_win_service_name
-}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/service_check.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/service_check.py
deleted file mode 100755
index 1836d0f..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/service_check.py
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-import socket
-import sys
-import time
-import subprocess
-
-from hcat_service_check import hcat_service_check
-from webhcat_service_check import webhcat_service_check
-from ambari_commons import OSConst
-from ambari_commons.os_family_impl import OsFamilyImpl
-from resource_management.core import shell
-from resource_management.core.logger import Logger
-from resource_management.libraries.functions import get_unique_id_and_date
-
-class HiveServiceCheck(Script):
-  pass
-
-
-@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
-class HiveServiceCheckWindows(HiveServiceCheck):
-  def service_check(self, env):
-    import params
-    env.set_params(params)
-    smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd")
-    service = "HIVE"
-    Execute(format("cmd /C {smoke_cmd} {service}"), user=params.hive_user, logoutput=True)
-
-    hcat_service_check()
-    webhcat_service_check()
-
-
-@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
-class HiveServiceCheckDefault(HiveServiceCheck):
-
-  def __init__(self):
-    super(HiveServiceCheckDefault, self).__init__()
-    Logger.initialize_logger()
-
-  def service_check(self, env):
-    import params
-    env.set_params(params)
-
-    if params.security_enabled:
-      kinit_cmd = format(
-        "{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ")
-    else:
-      kinit_cmd = ""
-
-    # Check HiveServer
-    Logger.info("Running Hive Server checks")
-    Logger.info("--------------------------\n")
-    self.check_hive_server(env, 'Hive Server', kinit_cmd, params.hive_server_hosts,
-                           int(format("{hive_server_port}")))
-
-
-    if params.has_hive_interactive  and params.hive_interactive_enabled:
-      Logger.info("Running Hive Server2 checks")
-      Logger.info("--------------------------\n")
-
-      self.check_hive_server(env, 'Hive Server2', kinit_cmd, params.hive_interactive_hosts,
-                             int(format("{hive_server_interactive_port}")))
-
-      Logger.info("Running LLAP checks")
-      Logger.info("-------------------\n")
-      self.check_llap(env, kinit_cmd, params.hive_interactive_hosts, int(format("{hive_server_interactive_port}")),
-                      params.hive_llap_principal, params.hive_server2_authentication, params.hive_transport_mode,
-                      params.hive_http_endpoint)
-
-
-    Logger.info("Running HCAT checks")
-    Logger.info("-------------------\n")
-    hcat_service_check()
-
-    Logger.info("Running WEBHCAT checks")
-    Logger.info("---------------------\n")
-    webhcat_service_check()
-
-  def check_hive_server(self, env, server_component_name, kinit_cmd, address_list, server_port):
-    import params
-    env.set_params(params)
-    Logger.info("Server Address List : {0}, Port : {1}".format(address_list, server_port))
-
-    if not address_list:
-      raise Fail("Can not find any "+server_component_name+" ,host. Please check configuration.")
-
-    SOCKET_WAIT_SECONDS = 290
-
-    start_time = time.time()
-    end_time = start_time + SOCKET_WAIT_SECONDS
-
-    Logger.info("Waiting for the {0} to start...".format(server_component_name))
-
-    workable_server_available = False
-    i = 0
-    while time.time() < end_time and not workable_server_available:
-      address = address_list[i]
-      try:
-        check_thrift_port_sasl(address, server_port, params.hive_server2_authentication,
-                               params.hive_server_principal, kinit_cmd, params.smokeuser,
-                               transport_mode=params.hive_transport_mode, http_endpoint=params.hive_http_endpoint,
-                               ssl=params.hive_ssl, ssl_keystore=params.hive_ssl_keystore_path,
-                               ssl_password=params.hive_ssl_keystore_password)
-        Logger.info("Successfully connected to {0} on port {1}".format(address, server_port))
-        workable_server_available = True
-      except:
-        Logger.info("Connection to {0} on port {1} failed".format(address, server_port))
-        time.sleep(5)
-
-      i += 1
-      if i == len(address_list):
-        i = 0
-
-    elapsed_time = time.time() - start_time
-
-    if not workable_server_available:
-      raise Fail("Connection to '{0}' on host: {1} and port {2} failed after {3} seconds"
-                 .format(server_component_name, params.hostname, server_port, elapsed_time))
-
-    Logger.info("Successfully stayed connected to '{0}' on host: {1} and port {2} after {3} seconds"
-                .format(server_component_name, params.hostname, server_port, elapsed_time))
-
-  """
-  Performs Service check for LLAP app
-  """
-  def check_llap(self, env, kinit_cmd, address, port, key, hive_auth="NOSASL", transport_mode="binary", http_endpoint="cliservice"):
-    import params
-    env.set_params(params)
-
-    unique_id = get_unique_id_and_date()
-
-    beeline_url = ['jdbc:hive2://{address}:{port}/', "transportMode={transport_mode}"]
-
-    # Currently, HSI is supported on a single node only. The address list should be of size 1,
-    # thus picking the 1st node value.
-    address = address[0]
-
-    # append url according to used transport
-    if transport_mode == "http":
-      beeline_url.append('httpPath={http_endpoint}')
-
-    # append url according to used auth
-    if hive_auth == "NOSASL":
-      beeline_url.append('auth=noSasl')
-
-    # append url according to principal
-    if kinit_cmd:
-      beeline_url.append('principal={key}')
-
-    exec_path = params.execute_path
-    if params.version and params.stack_root:
-      upgrade_hive_bin = format("{stack_root}/{version}/hive2/bin")
-      exec_path =  os.environ['PATH'] + os.pathsep + params.hadoop_bin_dir + os.pathsep + upgrade_hive_bin
-
-    # beeline path
-    llap_cmd = "! beeline -u '%s'" % format(";".join(beeline_url))
-    # Append LLAP SQL script path
-    llap_cmd += format(" --hiveconf \"hiveLlapServiceCheck={unique_id}\" -f {stack_root}/current/hive-server2-hive2/scripts/llap/sql/serviceCheckScript.sql")
-    # Append grep patterns for detecting failure
-    llap_cmd += " -e '' 2>&1| awk '{print}'|grep -i -e 'Invalid status\|Invalid URL\|command not found\|Connection refused'"
-
-    Execute(llap_cmd,
-            user=params.hive_user,
-            path=['/usr/sbin', '/usr/local/bin', '/bin', '/usr/bin', exec_path],
-            tries=1,
-            wait_for_finish=True,
-            stderr=subprocess.PIPE,
-            logoutput=True)
-
-if __name__ == "__main__":
-  HiveServiceCheck().execute()
\ No newline at end of file


Mime
View raw message