bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject [42/52] bigtop git commit: BIGTOP-1406. package Ambari in Bigtop
Date Wed, 22 Mar 2017 06:10:33 GMT
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive.py
deleted file mode 100755
index 4f53ea9..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive.py
+++ /dev/null
@@ -1,481 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import glob
-from urlparse import urlparse
-
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.core.resources.service import ServiceConfig
-from resource_management.core.resources.system import File, Execute, Directory
-from resource_management.core.source import StaticFile, Template, DownloadSource, InlineTemplate
-from resource_management.core.shell import as_user
-from resource_management.libraries.functions.is_empty import is_empty
-from resource_management.libraries.resources.xml_config import XmlConfig
-from resource_management.libraries.functions.format import format
-from resource_management.core.exceptions import Fail
-from resource_management.core.shell import as_sudo
-from resource_management.core.shell import quote_bash_args
-from resource_management.core.logger import Logger
-from resource_management.core import utils
-from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook
-from ambari_commons.constants import SERVICE
-
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from ambari_commons import OSConst
-
-
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def hive(name=None):
-  import params
-
-  XmlConfig("hive-site.xml",
-            conf_dir = params.hive_conf_dir,
-            configurations = params.config['configurations']['hive-site'],
-            owner=params.hive_user,
-            configuration_attributes=params.config['configuration_attributes']['hive-site']
-  )
-
-  if name in ["hiveserver2","metastore"]:
-    # Manually overriding service logon user & password set by the installation package
-    service_name = params.service_map[name]
-    ServiceConfig(service_name,
-                  action="change_user",
-                  username = params.hive_user,
-                  password = Script.get_password(params.hive_user))
-    Execute(format("cmd /c hadoop fs -mkdir -p {hive_warehouse_dir}"), logoutput=True, user=params.hadoop_user)
-
-  if name == 'metastore':
-    if params.init_metastore_schema:
-      check_schema_created_cmd = format('cmd /c "{hive_bin}\\hive.cmd --service schematool -info '
-                                        '-dbType {hive_metastore_db_type} '
-                                        '-userName {hive_metastore_user_name} '
-                                        '-passWord {hive_metastore_user_passwd!p}'
-                                        '&set EXITCODE=%ERRORLEVEL%&exit /B %EXITCODE%"', #cmd "feature", propagate the process exit code manually
-                                        hive_bin=params.hive_bin,
-                                        hive_metastore_db_type=params.hive_metastore_db_type,
-                                        hive_metastore_user_name=params.hive_metastore_user_name,
-                                        hive_metastore_user_passwd=params.hive_metastore_user_passwd)
-      try:
-        Execute(check_schema_created_cmd)
-      except Fail:
-        create_schema_cmd = format('cmd /c {hive_bin}\\hive.cmd --service schematool -initSchema '
-                                   '-dbType {hive_metastore_db_type} '
-                                   '-userName {hive_metastore_user_name} '
-                                   '-passWord {hive_metastore_user_passwd!p}',
-                                   hive_bin=params.hive_bin,
-                                   hive_metastore_db_type=params.hive_metastore_db_type,
-                                   hive_metastore_user_name=params.hive_metastore_user_name,
-                                   hive_metastore_user_passwd=params.hive_metastore_user_passwd)
-        Execute(create_schema_cmd,
-                user = params.hive_user,
-                logoutput=True
-        )
-
-  if name == "hiveserver2":
-    if params.hive_execution_engine == "tez":
-      # Init the tez app dir in hadoop
-      script_file = __file__.replace('/', os.sep)
-      cmd_file = os.path.normpath(os.path.join(os.path.dirname(script_file), "..", "files", "hiveTezSetup.cmd"))
-
-      Execute("cmd /c " + cmd_file, logoutput=True, user=params.hadoop_user)
-
-
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def hive(name=None):
-  import params
-
-  if name == 'hiveserver2':
-    # copy tarball to HDFS feature not supported
-    if not (params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major)):  
-      params.HdfsResource(params.webhcat_apps_dir,
-                            type="directory",
-                            action="create_on_execute",
-                            owner=params.webhcat_user,
-                            mode=0755
-                          )
-    
-    # Create webhcat dirs.
-    if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
-      params.HdfsResource(params.hcat_hdfs_user_dir,
-                           type="directory",
-                           action="create_on_execute",
-                           owner=params.hcat_user,
-                           mode=params.hcat_hdfs_user_mode
-      )
-
-    params.HdfsResource(params.webhcat_hdfs_user_dir,
-                         type="directory",
-                         action="create_on_execute",
-                         owner=params.webhcat_user,
-                         mode=params.webhcat_hdfs_user_mode
-    )
-
-    # ****** Begin Copy Tarballs ******
-    # *********************************
-    #  if copy tarball to HDFS feature  supported copy mapreduce.tar.gz and tez.tar.gz to HDFS
-    if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major):
-      copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
-      copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
-
-    # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode.
-    # This can use a different source and dest location to account
-    copy_to_hdfs("pig",
-                 params.user_group,
-                 params.hdfs_user,
-                 file_mode=params.tarballs_mode,
-                 custom_source_file=params.pig_tar_source,
-                 custom_dest_file=params.pig_tar_dest_file,
-                 host_sys_prepped=params.host_sys_prepped)
-    copy_to_hdfs("hive",
-                 params.user_group,
-                 params.hdfs_user,
-                 file_mode=params.tarballs_mode,
-                 custom_source_file=params.hive_tar_source,
-                 custom_dest_file=params.hive_tar_dest_file,
-                 host_sys_prepped=params.host_sys_prepped)
-
-    wildcard_tarballs = ["sqoop", "hadoop_streaming"]
-    for tarball_name in wildcard_tarballs:
-      source_file_pattern = eval("params." + tarball_name + "_tar_source")
-      dest_dir = eval("params." + tarball_name + "_tar_dest_dir")
-
-      if source_file_pattern is None or dest_dir is None:
-        continue
-
-      source_files = glob.glob(source_file_pattern) if "*" in source_file_pattern else [source_file_pattern]
-      for source_file in source_files:
-        src_filename = os.path.basename(source_file)
-        dest_file = os.path.join(dest_dir, src_filename)
-
-        copy_to_hdfs(tarball_name,
-                     params.user_group,
-                     params.hdfs_user,
-                     file_mode=params.tarballs_mode,
-                     custom_source_file=source_file,
-                     custom_dest_file=dest_file,
-                     host_sys_prepped=params.host_sys_prepped)
-    # ******* End Copy Tarballs *******
-    # *********************************
-    
-    # if warehouse directory is in DFS
-    if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(params.default_fs).scheme:
-      # Create Hive Metastore Warehouse Dir
-      params.HdfsResource(params.hive_apps_whs_dir,
-                           type="directory",
-                            action="create_on_execute",
-                            owner=params.hive_user,
-                            mode=0777
-      )
-    else:
-      Logger.info(format("Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS."))
-
-    # Create Hive User Dir
-    params.HdfsResource(params.hive_hdfs_user_dir,
-                         type="directory",
-                          action="create_on_execute",
-                          owner=params.hive_user,
-                          mode=params.hive_hdfs_user_mode
-    )
-    
-    if not is_empty(params.hive_exec_scratchdir) and not urlparse(params.hive_exec_scratchdir).path.startswith("/tmp"):
-      params.HdfsResource(params.hive_exec_scratchdir,
-                           type="directory",
-                           action="create_on_execute",
-                           owner=params.hive_user,
-                           group=params.hdfs_user,
-                           mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
-      
-    params.HdfsResource(None, action="execute")
-
-  Directory(params.hive_etc_dir_prefix,
-            mode=0755
-  )
-
-  # We should change configurations for client as well as for server.
-  # The reason is that stale-configs are service-level, not component.
-  Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list))
-  for conf_dir in params.hive_conf_dirs_list:
-    fill_conf_dir(conf_dir)
-
-  XmlConfig("hive-site.xml",
-            conf_dir=params.hive_config_dir,
-            configurations=params.hive_site_config,
-            configuration_attributes=params.config['configuration_attributes']['hive-site'],
-            owner=params.hive_user,
-            group=params.user_group,
-            mode=0644)
-
-  # Generate atlas-application.properties.xml file
-  if has_atlas_in_cluster():
-    atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename)
-    setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
-  
-  if name == 'hiveserver2':
-    XmlConfig("hiveserver2-site.xml",
-              conf_dir=params.hive_server_conf_dir,
-              configurations=params.config['configurations']['hiveserver2-site'],
-              configuration_attributes=params.config['configuration_attributes']['hiveserver2-site'],
-              owner=params.hive_user,
-              group=params.user_group,
-              mode=0644)
-
-  if params.hive_metastore_site_supported and name == 'metastore':
-    XmlConfig("hivemetastore-site.xml",
-              conf_dir=params.hive_server_conf_dir,
-              configurations=params.config['configurations']['hivemetastore-site'],
-              configuration_attributes=params.config['configuration_attributes']['hivemetastore-site'],
-              owner=params.hive_user,
-              group=params.user_group,
-              mode=0644)
-  
-  File(format("{hive_config_dir}/hive-env.sh"),
-       owner=params.hive_user,
-       group=params.user_group,
-       content=InlineTemplate(params.hive_env_sh_template)
-  )
-
-  # On some OS this folder could be not exists, so we will create it before pushing there files
-  Directory(params.limits_conf_dir,
-            create_parents = True,
-            owner='root',
-            group='root'
-            )
-
-  File(os.path.join(params.limits_conf_dir, 'hive.conf'),
-       owner='root',
-       group='root',
-       mode=0644,
-       content=Template("hive.conf.j2")
-       )
-
-  if name == 'metastore' or name == 'hiveserver2':
-    if params.hive_jdbc_target is not None and not os.path.exists(params.hive_jdbc_target):
-      jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar)
-    if params.hive2_jdbc_target is not None and not os.path.exists(params.hive2_jdbc_target):
-      jdbc_connector(params.hive2_jdbc_target, params.hive2_previous_jdbc_jar)
-
-  File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
-       content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
-       mode = 0644,
-  )
-
-  if name == 'metastore':
-    File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"),
-         owner=params.hive_user,
-         group=params.user_group,
-         content=Template("hadoop-metrics2-hivemetastore.properties.j2")
-    )
-
-    File(params.start_metastore_path,
-         mode=0755,
-         content=StaticFile('startMetastore.sh')
-    )
-    if params.init_metastore_schema:
-      create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
-                                 "{hive_schematool_bin}/schematool -initSchema "
-                                 "-dbType {hive_metastore_db_type} "
-                                 "-userName {hive_metastore_user_name} "
-                                 "-passWord {hive_metastore_user_passwd!p} -verbose")
-
-      check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
-                                        "{hive_schematool_bin}/schematool -info "
-                                        "-dbType {hive_metastore_db_type} "
-                                        "-userName {hive_metastore_user_name} "
-                                        "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user)
-
-      # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords.
-      # Fixing it with the hack below:
-      quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd))
-      if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \
-          or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"':
-        quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1]
-      Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace(
-          format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING))
-
-      Execute(create_schema_cmd,
-              not_if = check_schema_created_cmd,
-              user = params.hive_user
-      )
-  elif name == 'hiveserver2':
-    File(params.start_hiveserver2_path,
-         mode=0755,
-         content=Template(format('{start_hiveserver2_script}'))
-    )
-
-    File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hiveserver2.properties"),
-         owner=params.hive_user,
-         group=params.user_group,
-         content=Template("hadoop-metrics2-hiveserver2.properties.j2")
-    )
-
-  if name != "client":
-    Directory(params.hive_pid_dir,
-              create_parents = True,
-              cd_access='a',
-              owner=params.hive_user,
-              group=params.user_group,
-              mode=0755)
-    Directory(params.hive_log_dir,
-              create_parents = True,
-              cd_access='a',
-              owner=params.hive_user,
-              group=params.user_group,
-              mode=0755)
-    Directory(params.hive_var_lib,
-              create_parents = True,
-              cd_access='a',
-              owner=params.hive_user,
-              group=params.user_group,
-              mode=0755)
-
-"""
-Writes configuration files required by Hive.
-"""
-def fill_conf_dir(component_conf_dir):
-  import params
-
-  Directory(component_conf_dir,
-            owner=params.hive_user,
-            group=params.user_group,
-            create_parents = True
-  )
-
-  XmlConfig("mapred-site.xml",
-            conf_dir=component_conf_dir,
-            configurations=params.config['configurations']['mapred-site'],
-            configuration_attributes=params.config['configuration_attributes']['mapred-site'],
-            owner=params.hive_user,
-            group=params.user_group,
-            mode=0644)
-
-
-  File(format("{component_conf_dir}/hive-default.xml.template"),
-       owner=params.hive_user,
-       group=params.user_group
-  )
-
-  File(format("{component_conf_dir}/hive-env.sh.template"),
-       owner=params.hive_user,
-       group=params.user_group
-  )
-
-  # Create hive-log4j.properties and hive-exec-log4j.properties
-  # in /etc/hive/conf and not in /etc/hive2/conf
-  if params.log4j_version == '1':
-    log4j_exec_filename = 'hive-exec-log4j.properties'
-    if (params.log4j_exec_props != None):
-      File(format("{component_conf_dir}/{log4j_exec_filename}"),
-           mode=0644,
-           group=params.user_group,
-           owner=params.hive_user,
-           content=params.log4j_exec_props
-      )
-    elif (os.path.exists("{component_conf_dir}/{log4j_exec_filename}.template")):
-      File(format("{component_conf_dir}/{log4j_exec_filename}"),
-           mode=0644,
-           group=params.user_group,
-           owner=params.hive_user,
-           content=StaticFile(format("{component_conf_dir}/{log4j_exec_filename}.template"))
-      )
-
-    log4j_filename = 'hive-log4j.properties'
-    if (params.log4j_props != None):
-      File(format("{component_conf_dir}/{log4j_filename}"),
-           mode=0644,
-           group=params.user_group,
-           owner=params.hive_user,
-           content=params.log4j_props
-      )
-    elif (os.path.exists("{component_conf_dir}/{log4j_filename}.template")):
-      File(format("{component_conf_dir}/{log4j_filename}"),
-           mode=0644,
-           group=params.user_group,
-           owner=params.hive_user,
-           content=StaticFile(format("{component_conf_dir}/{log4j_filename}.template"))
-      )
-    pass # if params.log4j_version == '1'
-
-
-def jdbc_connector(target, hive_previous_jdbc_jar):
-  """
-  Shared by Hive Batch, Hive Metastore, and Hive Interactive
-  :param target: Target of jdbc jar name, which could be for any of the components above.
-  """
-  import params
-
-  if not params.jdbc_jar_name:
-    return
-
-  if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
-    environment = {
-      "no_proxy": format("{ambari_server_hostname}")
-    }
-
-    if hive_previous_jdbc_jar and os.path.isfile(hive_previous_jdbc_jar):
-      File(hive_previous_jdbc_jar, action='delete')
-
-    # TODO: should be removed after ranger_hive_plugin will not provide jdbc
-    if params.prepackaged_jdbc_name != params.jdbc_jar_name:
-      Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
-              path=["/bin", "/usr/bin/"],
-              sudo = True)
-    
-    File(params.downloaded_custom_connector,
-         content = DownloadSource(params.driver_curl_source))
-
-    # maybe it will be more correcvly to use db type
-    if params.sqla_db_used:
-      untar_sqla_type2_driver = ('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir)
-
-      Execute(untar_sqla_type2_driver, sudo = True)
-
-      Execute(format("yes | {sudo} cp {jars_path_in_archive} {hive_lib}"))
-
-      Directory(params.jdbc_libs_dir,
-                create_parents = True)
-
-      Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}"))
-
-      Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
-
-    else:
-      Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target),
-            #creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
-            path=["/bin", "/usr/bin/"],
-            sudo = True)
-
-  else:
-    #for default hive db (Mysql)
-    Execute(('cp', '--remove-destination', format('/usr/share/java/{jdbc_jar_name}'), target),
-            #creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
-            path=["/bin", "/usr/bin/"],
-            sudo=True
-    )
-  pass
-
-  File(target,
-       mode = 0644,
-  )

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_client.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_client.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_client.py
deleted file mode 100755
index 3d9bfd7..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_client.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-import sys
-from resource_management import *
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from hive import hive
-from ambari_commons.os_family_impl import OsFamilyImpl
-from ambari_commons import OSConst
-from resource_management.core.exceptions import ClientComponentHasNoStatus
-
-class HiveClient(Script):
-  def install(self, env):
-    import params
-    self.install_packages(env)
-    self.configure(env)
-
-  def status(self, env):
-    raise ClientComponentHasNoStatus()
-
-  def configure(self, env):
-    import params
-    env.set_params(params)
-    hive(name='client')
-
-
-@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
-class HiveClientWindows(HiveClient):
-  pass
-
-
-@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
-class HiveClientDefault(HiveClient):
-  def get_component_name(self):
-    return "hadoop-client"
-
-  def pre_upgrade_restart(self, env, upgrade_type=None):
-    Logger.info("Executing Hive client Stack Upgrade pre-restart")
-
-    import params
-    env.set_params(params)
-    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
-      conf_select.select(params.stack_name, "hive", params.version)
-      conf_select.select(params.stack_name, "hadoop", params.version)
-      stack_select.select("hadoop-client", params.version)
-
-
-if __name__ == "__main__":
-  HiveClient().execute()

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_interactive.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_interactive.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_interactive.py
deleted file mode 100755
index 74c67fc..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_interactive.py
+++ /dev/null
@@ -1,302 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-# Python Imports
-import os
-import glob
-from urlparse import urlparse
-
-# Resource Management and Common Imports
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.version import compare_versions
-from resource_management.core.resources.service import ServiceConfig
-from resource_management.core.resources.system import File, Execute, Directory
-from resource_management.core.source import StaticFile, Template, DownloadSource, InlineTemplate
-from resource_management.core.shell import as_user
-from resource_management.libraries.functions.is_empty import is_empty
-from resource_management.libraries.resources.xml_config import XmlConfig
-from resource_management.libraries.functions.format import format
-from resource_management.core.exceptions import Fail
-from resource_management.core.shell import as_sudo
-from resource_management.core.shell import quote_bash_args
-from resource_management.core.logger import Logger
-from resource_management.core import utils
-
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from ambari_commons import OSConst
-from hive import fill_conf_dir, jdbc_connector
-
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def hive_interactive(name=None):
-  pass
-
-"""
-Sets up the configs, jdbc connection and tarball copy to HDFS for Hive Server Interactive.
-"""
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def hive_interactive(name=None):
-  import params
-
-  # list of properties that should be excluded from the config
-  # this approach is a compromise against adding a dedicated config
-  # type for hive_server_interactive or needed config groups on a
-  # per component basis
-  exclude_list = ['hive.enforce.bucketing',
-                  'hive.enforce.sorting']
-
-  # List of configs to be excluded from hive2 client, but present in Hive2 server.
-  exclude_list_for_hive2_client = ['javax.jdo.option.ConnectionPassword']
-
-  # Copy Tarballs in HDFS.
-  if params.stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted_major):
-    resource_created = copy_to_hdfs("tez_hive2",
-                 params.user_group,
-                 params.hdfs_user,
-                 file_mode=params.tarballs_mode,
-                 host_sys_prepped=params.host_sys_prepped)
-
-    if resource_created:
-      params.HdfsResource(None, action="execute")
-
-  Directory(params.hive_interactive_etc_dir_prefix,
-            mode=0755
-            )
-
-  Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list))
-  for conf_dir in params.hive_conf_dirs_list:
-    fill_conf_dir(conf_dir)
-
-  '''
-  As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
-  we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
-  '''
-  merged_hive_interactive_site = {}
-  merged_hive_interactive_site.update(params.config['configurations']['hive-site'])
-  merged_hive_interactive_site.update(params.config['configurations']['hive-interactive-site'])
-  for item in exclude_list:
-    if item in merged_hive_interactive_site.keys():
-      del merged_hive_interactive_site[item]
-
-  '''
-  Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
-  which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
-  '''
-  remove_atlas_hook_if_exists(merged_hive_interactive_site)
-
-  '''
-  As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
-  we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
-  '''
-  merged_tez_interactive_site = {}
-  if 'tez-site' in params.config['configurations']:
-    merged_tez_interactive_site.update(params.config['configurations']['tez-site'])
-    Logger.info("Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'.")
-  else:
-    Logger.error("Tez's 'tez-site' couldn't be retrieved from passed-in configurations.")
-
-  merged_tez_interactive_site.update(params.config['configurations']['tez-interactive-site'])
-  XmlConfig("tez-site.xml",
-            conf_dir = params.tez_interactive_config_dir,
-            configurations = merged_tez_interactive_site,
-            configuration_attributes=params.config['configuration_attributes']['tez-interactive-site'],
-            owner = params.tez_interactive_user,
-            group = params.user_group,
-            mode = 0664)
-
-  '''
-  Merge properties from hiveserver2-interactive-site into hiveserver2-site
-  '''
-  merged_hiveserver2_interactive_site = {}
-  if 'hiveserver2-site' in params.config['configurations']:
-    merged_hiveserver2_interactive_site.update(params.config['configurations']['hiveserver2-site'])
-    Logger.info("Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'.")
-  else:
-    Logger.error("'hiveserver2-site' couldn't be retrieved from passed-in configurations.")
-  merged_hiveserver2_interactive_site.update(params.config['configurations']['hiveserver2-interactive-site'])
-
-
-  # Create config files under /etc/hive2/conf and /etc/hive2/conf/conf.server:
-  #   hive-site.xml
-  #   hive-env.sh
-  #   llap-daemon-log4j2.properties
-  #   llap-cli-log4j2.properties
-  #   hive-log4j2.properties
-  #   hive-exec-log4j2.properties
-  #   beeline-log4j2.properties
-
-  hive2_conf_dirs_list = params.hive_conf_dirs_list
-  hive2_client_conf_path = format("{stack_root}/current/{component_directory}/conf")
-
-  # Making copy of 'merged_hive_interactive_site' in 'merged_hive_interactive_site_copy', and deleting 'javax.jdo.option.ConnectionPassword'
-  # config from there, as Hive2 client shouldn't have that config.
-  merged_hive_interactive_site_copy = merged_hive_interactive_site.copy()
-  for item in exclude_list_for_hive2_client:
-    if item in merged_hive_interactive_site.keys():
-      del merged_hive_interactive_site_copy[item]
-
-  for conf_dir in hive2_conf_dirs_list:
-      if conf_dir == hive2_client_conf_path:
-        XmlConfig("hive-site.xml",
-                  conf_dir=conf_dir,
-                  configurations=merged_hive_interactive_site_copy,
-                  configuration_attributes=params.config['configuration_attributes']['hive-interactive-site'],
-                  owner=params.hive_user,
-                  group=params.user_group,
-                  mode=0644)
-      else:
-        XmlConfig("hive-site.xml",
-                  conf_dir=conf_dir,
-                  configurations=merged_hive_interactive_site,
-                  configuration_attributes=params.config['configuration_attributes']['hive-interactive-site'],
-                  owner=params.hive_user,
-                  group=params.user_group,
-                  mode=0644)
-
-      XmlConfig("hiveserver2-site.xml",
-                conf_dir=conf_dir,
-                configurations=merged_hiveserver2_interactive_site,
-                configuration_attributes=params.config['configuration_attributes']['hiveserver2-interactive-site'],
-                owner=params.hive_user,
-                group=params.user_group,
-                mode=0644)
-
-      hive_server_interactive_conf_dir = conf_dir
-
-      File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
-           owner=params.hive_user,
-           group=params.user_group,
-           content=InlineTemplate(params.hive_interactive_env_sh_template))
-
-      llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
-      File(format("{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
-           mode=0644,
-           group=params.user_group,
-           owner=params.hive_user,
-           content=params.llap_daemon_log4j)
-
-      llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
-      File(format("{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
-           mode=0644,
-           group=params.user_group,
-           owner=params.hive_user,
-           content=params.llap_cli_log4j2)
-
-      hive_log4j2_filename = 'hive-log4j2.properties'
-      File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
-         mode=0644,
-         group=params.user_group,
-         owner=params.hive_user,
-         content=params.hive_log4j2)
-
-      hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
-      File(format("{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
-         mode=0644,
-         group=params.user_group,
-         owner=params.hive_user,
-         content=params.hive_exec_log4j2)
-
-      beeline_log4j2_filename = 'beeline-log4j2.properties'
-      File(format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
-         mode=0644,
-         group=params.user_group,
-         owner=params.hive_user,
-         content=params.beeline_log4j2)
-
-      File(os.path.join(hive_server_interactive_conf_dir, "hadoop-metrics2-hiveserver2.properties"),
-           owner=params.hive_user,
-           group=params.user_group,
-           content=Template("hadoop-metrics2-hiveserver2.properties.j2")
-           )
-
-      File(format("{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"),
-           owner=params.hive_user,
-           group=params.user_group,
-           content=Template("hadoop-metrics2-llapdaemon.j2"))
-
-      File(format("{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"),
-           owner=params.hive_user,
-           group=params.user_group,
-           content=Template("hadoop-metrics2-llaptaskscheduler.j2"))
-
-
-  # On some OS this folder could be not exists, so we will create it before pushing there files
-  Directory(params.limits_conf_dir,
-            create_parents = True,
-            owner='root',
-            group='root')
-
-  File(os.path.join(params.limits_conf_dir, 'hive.conf'),
-       owner='root',
-       group='root',
-       mode=0644,
-       content=Template("hive.conf.j2"))
-
-  if not os.path.exists(params.target_hive_interactive):
-    jdbc_connector(params.target_hive_interactive, params.hive_intaractive_previous_jdbc_jar)
-
-  File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
-       content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
-       mode = 0644)
-  File(params.start_hiveserver2_interactive_path,
-       mode=0755,
-       content=Template(format('{start_hiveserver2_interactive_script}')))
-
-  Directory(params.hive_pid_dir,
-            create_parents=True,
-            cd_access='a',
-            owner=params.hive_user,
-            group=params.user_group,
-            mode=0755)
-  Directory(params.hive_log_dir,
-            create_parents=True,
-            cd_access='a',
-            owner=params.hive_user,
-            group=params.user_group,
-            mode=0755)
-  Directory(params.hive_interactive_var_lib,
-            create_parents=True,
-            cd_access='a',
-            owner=params.hive_user,
-            group=params.user_group,
-            mode=0755)
-
-"""
-Remove 'org.apache.atlas.hive.hook.HiveHook' value from Hive2/hive-site.xml config 'hive.exec.post.hooks', if exists.
-"""
-def remove_atlas_hook_if_exists(merged_hive_interactive_site):
-  if 'hive.exec.post.hooks' in merged_hive_interactive_site.keys():
-    existing_hive_exec_post_hooks = merged_hive_interactive_site.get('hive.exec.post.hooks')
-    if existing_hive_exec_post_hooks:
-      hook_splits = existing_hive_exec_post_hooks.split(",")
-      updated_hook_splits = [hook for hook in hook_splits if not hook.strip() == 'org.apache.atlas.hive.hook.HiveHook']
-      updated_hooks_str = ",".join((str(hook)).strip() for hook in updated_hook_splits)
-      if updated_hooks_str != existing_hive_exec_post_hooks:
-        merged_hive_interactive_site['hive.exec.post.hooks'] = updated_hooks_str
-        Logger.info("Updated Hive2/hive-site.xml 'hive.exec.post.hooks' value from : '{0}' to : '{1}'"
-                    .format(existing_hive_exec_post_hooks, updated_hooks_str))
-      else:
-        Logger.info("No change done to Hive2/hive-site.xml 'hive.exec.post.hooks' value.")
-  else:
-      Logger.debug("'hive.exec.post.hooks' doesn't exist in Hive2/hive-site.xml")

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_metastore.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_metastore.py
deleted file mode 100755
index 17bf581..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_metastore.py
+++ /dev/null
@@ -1,259 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-import os
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, Directory
-from resource_management.libraries.script import Script
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.constants import Direction
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions.version import format_stack_version
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.security_commons import build_expectations
-from resource_management.libraries.functions.security_commons import cached_kinit_executor
-from resource_management.libraries.functions.security_commons import get_params_from_filesystem
-from resource_management.libraries.functions.security_commons import validate_security_config_properties
-from resource_management.libraries.functions.security_commons import FILE_TYPE_XML
-from resource_management.core.resources.system import File
-
-from hive import hive
-from hive import jdbc_connector
-from hive_service import hive_service
-from ambari_commons.os_family_impl import OsFamilyImpl
-from ambari_commons import OSConst
-
-# the legacy conf.server location in previous stack versions
-LEGACY_HIVE_SERVER_CONF = "/etc/hive/conf.server"
-
-class HiveMetastore(Script):
-  def install(self, env):
-    import params
-    self.install_packages(env)
-
-
-  def start(self, env, upgrade_type=None):
-    import params
-    env.set_params(params)
-
-    # writing configurations on start required for securtity
-    self.configure(env)
-
-    hive_service('metastore', action='start', upgrade_type=upgrade_type)
-
-
-  def stop(self, env, upgrade_type=None):
-    import params
-    env.set_params(params)
-    hive_service('metastore', action='stop', upgrade_type=upgrade_type)
-
-
-  def configure(self, env):
-    import params
-    env.set_params(params)
-    hive(name = 'metastore')
-
-
-@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
-class HiveMetastoreWindows(HiveMetastore):
-  def status(self, env):
-    import status_params
-    from resource_management.libraries.functions import check_windows_service_status
-    check_windows_service_status(status_params.hive_metastore_win_service_name)
-
-
-@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
-class HiveMetastoreDefault(HiveMetastore):
-  def get_component_name(self):
-    return "hive-metastore"
-
-
-  def status(self, env):
-    import status_params
-    from resource_management.libraries.functions import check_process_status
-
-    env.set_params(status_params)
-    pid_file = format("{hive_pid_dir}/{hive_metastore_pid}")
-    # Recursively check all existing gmetad pid files
-    check_process_status(pid_file)
-
-
-  def pre_upgrade_restart(self, env, upgrade_type=None):
-    Logger.info("Executing Metastore Stack Upgrade pre-restart")
-    import params
-
-    env.set_params(params)
-
-    is_upgrade = params.upgrade_direction == Direction.UPGRADE
-
-    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
-      conf_select.select(params.stack_name, "hive", params.version)
-      stack_select.select("hive-metastore", params.version)
-
-    if is_upgrade and params.stack_version_formatted_major and \
-            check_stack_feature(StackFeature.HIVE_METASTORE_UPGRADE_SCHEMA, params.stack_version_formatted_major):
-      self.upgrade_schema(env)
-
-
-  def security_status(self, env):
-    import status_params
-    env.set_params(status_params)
-    if status_params.security_enabled:
-      props_value_check = {"hive.server2.authentication": "KERBEROS",
-                           "hive.metastore.sasl.enabled": "true",
-                           "hive.security.authorization.enabled": "true"}
-      props_empty_check = ["hive.metastore.kerberos.keytab.file",
-                           "hive.metastore.kerberos.principal"]
-
-      props_read_check = ["hive.metastore.kerberos.keytab.file"]
-      hive_site_props = build_expectations('hive-site', props_value_check, props_empty_check,
-                                            props_read_check)
-
-      hive_expectations ={}
-      hive_expectations.update(hive_site_props)
-
-      security_params = get_params_from_filesystem(status_params.hive_conf_dir,
-                                                   {'hive-site.xml': FILE_TYPE_XML})
-      result_issues = validate_security_config_properties(security_params, hive_expectations)
-      if not result_issues: # If all validations passed successfully
-        try:
-          # Double check the dict before calling execute
-          if 'hive-site' not in security_params \
-            or 'hive.metastore.kerberos.keytab.file' not in security_params['hive-site'] \
-            or 'hive.metastore.kerberos.principal' not in security_params['hive-site']:
-            self.put_structured_out({"securityState": "UNSECURED"})
-            self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."})
-            return
-
-          cached_kinit_executor(status_params.kinit_path_local,
-                                status_params.hive_user,
-                                security_params['hive-site']['hive.metastore.kerberos.keytab.file'],
-                                security_params['hive-site']['hive.metastore.kerberos.principal'],
-                                status_params.hostname,
-                                status_params.tmp_dir)
-
-          self.put_structured_out({"securityState": "SECURED_KERBEROS"})
-        except Exception as e:
-          self.put_structured_out({"securityState": "ERROR"})
-          self.put_structured_out({"securityStateErrorInfo": str(e)})
-      else:
-        issues = []
-        for cf in result_issues:
-          issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
-        self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
-        self.put_structured_out({"securityState": "UNSECURED"})
-    else:
-      self.put_structured_out({"securityState": "UNSECURED"})
-
-
-  def upgrade_schema(self, env):
-    """
-    Executes the schema upgrade binary.  This is its own function because it could
-    be called as a standalone task from the upgrade pack, but is safe to run it for each
-    metastore instance. The schema upgrade on an already upgraded metastore is a NOOP.
-
-    The metastore schema upgrade requires a database driver library for most
-    databases. During an upgrade, it's possible that the library is not present,
-    so this will also attempt to copy/download the appropriate driver.
-
-    This function will also ensure that configurations are written out to disk before running
-    since the new configs will most likely not yet exist on an upgrade.
-
-    Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades.
-    """
-    Logger.info("Upgrading Hive Metastore Schema")
-    import status_params
-    import params
-    env.set_params(params)
-
-    # ensure that configurations are written out before trying to upgrade the schema
-    # since the schematool needs configs and doesn't know how to use the hive conf override
-    self.configure(env)
-
-    if params.security_enabled:
-      cached_kinit_executor(status_params.kinit_path_local,
-        status_params.hive_user,
-        params.hive_metastore_keytab_path,
-        params.hive_metastore_principal,
-        status_params.hostname,
-        status_params.tmp_dir)
-      
-    # ensure that the JDBC drive is present for the schema tool; if it's not
-    # present, then download it first
-    if params.hive_jdbc_driver in params.hive_jdbc_drivers_list:
-      target_directory = format("{stack_root}/{version}/hive/lib")
-
-      # download it if it does not exist
-      if not os.path.exists(params.source_jdbc_file):
-        jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar)
-
-      target_directory_and_filename = os.path.join(target_directory, os.path.basename(params.source_jdbc_file))
-
-      if params.sqla_db_used:
-        target_native_libs_directory = format("{target_directory}/native/lib64")
-
-        Execute(format("yes | {sudo} cp {jars_in_hive_lib} {target_directory}"))
-
-        Directory(target_native_libs_directory, create_parents = True)
-
-        Execute(format("yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}"))
-
-        Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
-      else:
-        # copy the JDBC driver from the older metastore location to the new location only
-        # if it does not already exist
-        if not os.path.exists(target_directory_and_filename):
-          Execute(('cp', params.source_jdbc_file, target_directory),
-            path=["/bin", "/usr/bin/"], sudo = True)
-
-      File(target_directory_and_filename, mode = 0644)
-
-    # build the schema tool command
-    binary = format("{hive_schematool_ver_bin}/schematool")
-
-    # the conf.server directory changed locations between stack versions
-    # since the configurations have not been written out yet during an upgrade
-    # we need to choose the original legacy location
-    schematool_hive_server_conf_dir = params.hive_server_conf_dir
-    if params.current_version is not None:
-      current_version = format_stack_version(params.current_version)
-      if not(check_stack_feature(StackFeature.CONFIG_VERSIONING, current_version)):
-        schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF
-
-    env_dict = {
-      'HIVE_CONF_DIR': schematool_hive_server_conf_dir
-    }
-
-    command = format("{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
-    Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True)
-    
-  def get_log_folder(self):
-    import params
-    return params.hive_log_dir
-
-  def get_user(self):
-    import params
-    return params.hive_user
-
-
-if __name__ == "__main__":
-  HiveMetastore().execute()

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server.py
deleted file mode 100755
index 31b083b..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server.py
+++ /dev/null
@@ -1,211 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
-from resource_management.libraries.functions.get_stack_version import get_stack_version
-from resource_management.libraries.functions.check_process_status import check_process_status
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.security_commons import build_expectations, \
-  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
-  FILE_TYPE_XML
-from ambari_commons import OSCheck, OSConst
-if OSCheck.is_windows_family():
-  from resource_management.libraries.functions.windows_service_utils import check_windows_service_status
-from setup_ranger_hive import setup_ranger_hive
-from ambari_commons.os_family_impl import OsFamilyImpl
-from ambari_commons.constants import UPGRADE_TYPE_ROLLING
-from resource_management.core.logger import Logger
-
-import hive_server_upgrade
-from hive import hive
-from hive_service import hive_service
-
-
-class HiveServer(Script):
-  def install(self, env):
-    import params
-    self.install_packages(env)
-
-  def configure(self, env):
-    import params
-    env.set_params(params)
-    hive(name='hiveserver2')
-
-
-@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
-class HiveServerWindows(HiveServer):
-  def start(self, env):
-    import params
-    env.set_params(params)
-    self.configure(env) # FOR SECURITY
-    hive_service('hiveserver2', action='start')
-
-  def stop(self, env):
-    import params
-    env.set_params(params)
-    hive_service('hiveserver2', action='stop')
-
-  def status(self, env):
-    import status_params
-    check_windows_service_status(status_params.hive_server_win_service_name)
-
-
-@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
-class HiveServerDefault(HiveServer):
-  def get_component_name(self):
-    return "hive-server2"
-
-  def start(self, env, upgrade_type=None):
-    import params
-    env.set_params(params)
-    self.configure(env) # FOR SECURITY
-
-    setup_ranger_hive(upgrade_type=upgrade_type)
-    hive_service('hiveserver2', action = 'start', upgrade_type=upgrade_type)
-
-    # only perform this if upgrading and rolling; a non-rolling upgrade doesn't need
-    # to do this since hive is already down
-    if upgrade_type == UPGRADE_TYPE_ROLLING:
-      hive_server_upgrade.post_upgrade_deregister()
-
-
-  def stop(self, env, upgrade_type=None):
-    import params
-    env.set_params(params)
-
-    # During rolling upgrade, HiveServer2 should not be stopped before new server is available.
-    # Once new server is started, old one is stopped by the --deregister command which is 
-    # invoked by the 'hive_server_upgrade.post_upgrade_deregister()' method
-    if upgrade_type != UPGRADE_TYPE_ROLLING:
-      hive_service( 'hiveserver2', action = 'stop' )
-
-
-  def status(self, env):
-    import status_params
-    env.set_params(status_params)
-    pid_file = format("{hive_pid_dir}/{hive_pid}")
-
-    # Recursively check all existing gmetad pid files
-    check_process_status(pid_file)
-
-
-  def pre_upgrade_restart(self, env, upgrade_type=None):
-    Logger.info("Executing Hive Server Stack Upgrade pre-restart")
-    import params
-    env.set_params(params)
-
-    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
-      conf_select.select(params.stack_name, "hive", params.version)
-      stack_select.select("hive-server2", params.version)
-
-      # Copy mapreduce.tar.gz and tez.tar.gz to HDFS
-      resource_created = copy_to_hdfs(
-        "mapreduce",
-        params.user_group,
-        params.hdfs_user,
-        host_sys_prepped=params.host_sys_prepped)
-
-      resource_created = copy_to_hdfs(
-        "tez",
-        params.user_group,
-        params.hdfs_user,
-        host_sys_prepped=params.host_sys_prepped) or resource_created
-
-      if resource_created:
-        params.HdfsResource(None, action="execute")
-
-
-  def security_status(self, env):
-    import status_params
-    env.set_params(status_params)
-    if status_params.security_enabled:
-      props_value_check = {"hive.server2.authentication": "KERBEROS",
-                           "hive.metastore.sasl.enabled": "true",
-                           "hive.security.authorization.enabled": "true"}
-      props_empty_check = ["hive.server2.authentication.kerberos.keytab",
-                           "hive.server2.authentication.kerberos.principal",
-                           "hive.server2.authentication.spnego.principal",
-                           "hive.server2.authentication.spnego.keytab"]
-
-      props_read_check = ["hive.server2.authentication.kerberos.keytab",
-                          "hive.server2.authentication.spnego.keytab"]
-      hive_site_props = build_expectations('hive-site', props_value_check, props_empty_check,
-                                            props_read_check)
-
-      hive_expectations ={}
-      hive_expectations.update(hive_site_props)
-
-      security_params = get_params_from_filesystem(status_params.hive_conf_dir,
-                                                   {'hive-site.xml': FILE_TYPE_XML})
-      result_issues = validate_security_config_properties(security_params, hive_expectations)
-      if not result_issues: # If all validations passed successfully
-        try:
-          # Double check the dict before calling execute
-          if 'hive-site' not in security_params \
-            or 'hive.server2.authentication.kerberos.keytab' not in security_params['hive-site'] \
-            or 'hive.server2.authentication.kerberos.principal' not in security_params['hive-site']  \
-            or 'hive.server2.authentication.spnego.keytab' not in security_params['hive-site'] \
-            or 'hive.server2.authentication.spnego.principal' not in security_params['hive-site']:
-            self.put_structured_out({"securityState": "UNSECURED"})
-            self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."})
-            return
-
-          cached_kinit_executor(status_params.kinit_path_local,
-                                status_params.hive_user,
-                                security_params['hive-site']['hive.server2.authentication.kerberos.keytab'],
-                                security_params['hive-site']['hive.server2.authentication.kerberos.principal'],
-                                status_params.hostname,
-                                status_params.tmp_dir)
-          cached_kinit_executor(status_params.kinit_path_local,
-                                status_params.hive_user,
-                                security_params['hive-site']['hive.server2.authentication.spnego.keytab'],
-                                security_params['hive-site']['hive.server2.authentication.spnego.principal'],
-                                status_params.hostname,
-                                status_params.tmp_dir)
-          self.put_structured_out({"securityState": "SECURED_KERBEROS"})
-        except Exception as e:
-          self.put_structured_out({"securityState": "ERROR"})
-          self.put_structured_out({"securityStateErrorInfo": str(e)})
-      else:
-        issues = []
-        for cf in result_issues:
-          issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
-        self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
-        self.put_structured_out({"securityState": "UNSECURED"})
-    else:
-      self.put_structured_out({"securityState": "UNSECURED"})
-
-  def get_log_folder(self):
-    import params
-    return params.hive_log_dir
-  
-  def get_user(self):
-    import params
-    return params.hive_user
-
-if __name__ == "__main__":
-  HiveServer().execute()

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d3448b8/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server_interactive.py b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server_interactive.py
deleted file mode 100755
index 2df001c..0000000
--- a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/package/scripts/hive_server_interactive.py
+++ /dev/null
@@ -1,535 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-# Python Imports
-import subprocess
-import os
-import re
-import time
-import shutil
-from datetime import datetime
-import json
-
-# Ambari Commons & Resource Management imports
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions.check_process_status import check_process_status
-from resource_management.core.source import InlineTemplate
-from resource_management.core.resources.system import Execute
-
-# Imports needed for Rolling/Express Upgrade
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
-
-from resource_management.core import shell
-from resource_management.core.exceptions import Fail
-from resource_management.core.logger import Logger
-from ambari_commons import OSCheck, OSConst
-from ambari_commons.os_family_impl import OsFamilyImpl
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.libraries.functions.decorator import retry
-from resource_management.libraries.functions.security_commons import build_expectations, \
-  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
-  FILE_TYPE_XML
-
-# Local Imports
-from setup_ranger_hive import setup_ranger_hive
-from hive_service_interactive import hive_service_interactive
-from hive_interactive import hive_interactive
-from hive_server import HiveServerDefault
-from setup_ranger_hive_interactive import setup_ranger_hive_interactive
-
-import traceback
-
-class HiveServerInteractive(Script):
-  pass
-
-
-@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
-class HiveServerInteractiveDefault(HiveServerInteractive):
-
-    def get_component_name(self):
-      return "hive-server2-hive2"
-
-    def install(self, env):
-      import params
-      self.install_packages(env)
-
-    def configure(self, env):
-      import params
-      env.set_params(params)
-      hive_interactive(name='hiveserver2')
-
-    def pre_upgrade_restart(self, env, upgrade_type=None):
-      Logger.info("Executing Hive Server Interactive Stack Upgrade pre-restart")
-      import params
-      env.set_params(params)
-
-      if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
-        stack_select.select("hive-server2-hive2", params.version)
-        conf_select.select(params.stack_name, "hive2", params.version)
-
-        # Copy hive.tar.gz and tez.tar.gz used by Hive Interactive to HDFS
-        resource_created = copy_to_hdfs(
-          "hive2",
-          params.user_group,
-          params.hdfs_user,
-          host_sys_prepped=params.host_sys_prepped)
-
-        resource_created = copy_to_hdfs(
-          "tez_hive2",
-          params.user_group,
-          params.hdfs_user,
-          host_sys_prepped=params.host_sys_prepped) or resource_created
-
-        if resource_created:
-          params.HdfsResource(None, action="execute")
-
-    def start(self, env, upgrade_type=None):
-      import params
-      env.set_params(params)
-      self.configure(env)
-
-      if params.security_enabled:
-        # Do the security setup, internally calls do_kinit()
-        self.setup_security()
-
-      # TODO : We need have conditional [re]start of LLAP once "status check command" for LLAP is ready.
-      # Check status and based on that decide on [re]starting.
-
-      # Start LLAP before Hive Server Interactive start.
-      status = self._llap_start(env)
-      if not status:
-        raise Fail("Skipping START of Hive Server Interactive since LLAP app couldn't be STARTED.")
-
-      # TODO : test the workability of Ranger and Hive2 during upgrade
-      setup_ranger_hive_interactive(upgrade_type=upgrade_type)
-      hive_service_interactive('hiveserver2', action='start', upgrade_type=upgrade_type)
-
-
-    def stop(self, env, upgrade_type=None):
-      import params
-      env.set_params(params)
-
-      if params.security_enabled:
-        self.do_kinit()
-
-      # Stop Hive Interactive Server first
-      hive_service_interactive('hiveserver2', action='stop')
-
-      self._llap_stop(env)
-
-    def status(self, env):
-      import status_params
-      env.set_params(status_params)
-
-      # We are not doing 'llap' status check done here as part of status check for 'HSI', as 'llap' status
-      # check is a heavy weight operation.
-
-      pid_file = format("{hive_pid_dir}/{hive_interactive_pid}")
-      # Recursively check all existing gmetad pid files
-      check_process_status(pid_file)
-
-    def security_status(self, env):
-      import status_params
-      env.set_params(status_params)
-
-      if status_params.security_enabled:
-        props_value_check = {"hive.server2.authentication": "KERBEROS",
-                             "hive.metastore.sasl.enabled": "true",
-                             "hive.security.authorization.enabled": "true"}
-        props_empty_check = ["hive.server2.authentication.kerberos.keytab",
-                             "hive.server2.authentication.kerberos.principal",
-                             "hive.server2.authentication.spnego.principal",
-                             "hive.server2.authentication.spnego.keytab"]
-
-        props_read_check = ["hive.server2.authentication.kerberos.keytab",
-                            "hive.server2.authentication.spnego.keytab"]
-        hive_site_props = build_expectations('hive-site', props_value_check, props_empty_check,
-                                             props_read_check)
-
-        hive_expectations ={}
-        hive_expectations.update(hive_site_props)
-
-        security_params = get_params_from_filesystem(status_params.hive_server_interactive_conf_dir,
-                                                     {'hive-site.xml': FILE_TYPE_XML})
-        result_issues = validate_security_config_properties(security_params, hive_expectations)
-        if not result_issues: # If all validations passed successfully
-          try:
-            # Double check the dict before calling execute
-            if 'hive-site' not in security_params \
-              or 'hive.server2.authentication.kerberos.keytab' not in security_params['hive-site'] \
-              or 'hive.server2.authentication.kerberos.principal' not in security_params['hive-site'] \
-              or 'hive.server2.authentication.spnego.keytab' not in security_params['hive-site'] \
-              or 'hive.server2.authentication.spnego.principal' not in security_params['hive-site']:
-              self.put_structured_out({"securityState": "UNSECURED"})
-              self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."})
-              return
-
-            cached_kinit_executor(status_params.kinit_path_local,
-                                  status_params.hive_user,
-                                  security_params['hive-site']['hive.server2.authentication.kerberos.keytab'],
-                                  security_params['hive-site']['hive.server2.authentication.kerberos.principal'],
-                                  status_params.hostname,
-                                  status_params.tmp_dir)
-            cached_kinit_executor(status_params.kinit_path_local,
-                                  status_params.hive_user,
-                                  security_params['hive-site']['hive.server2.authentication.spnego.keytab'],
-                                  security_params['hive-site']['hive.server2.authentication.spnego.principal'],
-                                  status_params.hostname,
-                                  status_params.tmp_dir)
-            self.put_structured_out({"securityState": "SECURED_KERBEROS"})
-          except Exception as e:
-            self.put_structured_out({"securityState": "ERROR"})
-            self.put_structured_out({"securityStateErrorInfo": str(e)})
-        else:
-          issues = []
-          for cf in result_issues:
-            issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
-          self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
-          self.put_structured_out({"securityState": "UNSECURED"})
-      else:
-        self.put_structured_out({"securityState": "UNSECURED"})
-
-    def restart_llap(self, env):
-      """
-      Custom command to Restart LLAP
-      """
-      Logger.info("Custom Command to retart LLAP")
-      import params
-      env.set_params(params)
-
-      if params.security_enabled:
-        self.do_kinit()
-
-      self._llap_stop(env)
-      self._llap_start(env)
-
-    def _llap_stop(self, env):
-      import params
-      Logger.info("Stopping LLAP")
-      SLIDER_APP_NAME = "llap0"
-
-      stop_cmd = ["slider", "stop", SLIDER_APP_NAME]
-
-      code, output, error = shell.call(stop_cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True)
-      if code == 0:
-        Logger.info(format("Stopped {SLIDER_APP_NAME} application on Slider successfully"))
-      elif code == 69 and output is not None and "Unknown application instance" in output:
-        Logger.info(format("Application {SLIDER_APP_NAME} was already stopped on Slider"))
-      else:
-        raise Fail(format("Could not stop application {SLIDER_APP_NAME} on Slider. {error}\n{output}"))
-
-      # Will exit with code 4 if need to run with "--force" to delete directories and registries.
-      Execute(('slider', 'destroy', SLIDER_APP_NAME, "--force"),
-              user=params.hive_user,
-              timeout=30,
-              ignore_failures=True,
-      )
-
-    """
-    Controls the start of LLAP.
-    """
-    def _llap_start(self, env, cleanup=False):
-      import params
-      env.set_params(params)
-      Logger.info("Starting LLAP")
-      LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
-      LLAP_APP_NAME = 'llap0'
-
-      unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
-
-      cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
-                   " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m "
-                   " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
-                   " --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
-      if params.security_enabled:
-        llap_keytab_splits = params.hive_llap_keytab_file.split("/")
-        Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
-        cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
-                      "{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}")
-
-      # Append args.
-      llap_java_args = InlineTemplate(params.llap_app_java_opts).get_content()
-      cmd += format(" --args \" {llap_java_args}\"")
-
-      run_file_path = None
-      try:
-        Logger.info(format("Command: {cmd}"))
-        code, output, error = shell.checked_call(cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True)
-
-        if code != 0 or output is None:
-          raise Fail("Command failed with either non-zero return code or no output.")
-
-        # E.g., output:
-        # Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
-        exp = r"Prepared (.*?run.sh) for running LLAP"
-        run_file_path = None
-        out_splits = output.split("\n")
-        for line in out_splits:
-          line = line.strip()
-          m = re.match(exp, line, re.I)
-          if m and len(m.groups()) == 1:
-            run_file_name = m.group(1)
-            run_file_path = os.path.join(params.hive_user_home_dir, run_file_name)
-            break
-        if not run_file_path:
-          raise Fail("Did not find run.sh file in output: " + str(output))
-
-        Logger.info(format("Run file path: {run_file_path}"))
-        Execute(run_file_path, user=params.hive_user)
-        Logger.info("Submitted LLAP app name : {0}".format(LLAP_APP_NAME))
-
-        # We need to check the status of LLAP app to figure out it got
-        # launched properly and is in running state. Then go ahead with Hive Interactive Server start.
-        status = self.check_llap_app_status(LLAP_APP_NAME, params.num_retries_for_checking_llap_status)
-        if status:
-          Logger.info("LLAP app '{0}' deployed successfully.".format(LLAP_APP_NAME))
-          return True
-        else:
-          Logger.error("LLAP app '{0}' deployment unsuccessful.".format(LLAP_APP_NAME))
-          return False
-      except:
-        # Attempt to clean up the packaged application, or potentially rename it with a .bak
-        if run_file_path is not None and cleanup:
-          try:
-            parent_dir = os.path.dirname(run_file_path)
-            if os.path.isdir(parent_dir):
-              shutil.rmtree(parent_dir)
-          except Exception, e:
-            Logger.error("Could not cleanup LLAP app package. Error: " + str(e))
-
-        # throw the original exception
-        raise
-
-    """
-    Does kinit and copies keytab for Hive/LLAP to HDFS.
-    """
-    def setup_security(self):
-      import params
-
-      self.do_kinit()
-
-      # Copy params.hive_llap_keytab_file to hdfs://<host>:<port>/user/<hive_user>/.slider/keytabs/<hive_user> , required by LLAP
-      slider_keytab_install_cmd = format("slider install-keytab --keytab {params.hive_llap_keytab_file} --folder {params.hive_user} --overwrite")
-      Execute(slider_keytab_install_cmd, user=params.hive_user)
-
-    def do_kinit(self):
-      import params
-
-      hive_interactive_kinit_cmd = format("{kinit_path_local} -kt {params.hive_server2_keytab} {params.hive_principal}; ")
-      Execute(hive_interactive_kinit_cmd, user=params.hive_user)
-
-      llap_kinit_cmd = format("{kinit_path_local} -kt {params.hive_llap_keytab_file} {params.hive_llap_principal}; ")
-      Execute(llap_kinit_cmd, user=params.hive_user)
-
-    """
-    Get llap app status data.
-    """
-    def _get_llap_app_status_info(self, app_name):
-      import status_params
-      LLAP_APP_STATUS_CMD_TIMEOUT = 0
-
-      llap_status_cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llapstatus --name {app_name} --findAppTimeout {LLAP_APP_STATUS_CMD_TIMEOUT}")
-      code, output, error = shell.checked_call(llap_status_cmd, user=status_params.hive_user, stderr=subprocess.PIPE,
-                                               logoutput=False)
-      Logger.info("Received 'llapstatus' command 'output' : {0}".format(output))
-      return self._make_valid_json(output)
-
-
-    """
-    Remove extra lines from 'llapstatus' status output (eg: because of MOTD logging) so as to have a valid JSON data to be passed in
-    to JSON converter.
-    """
-    def _make_valid_json(self, output):
-      '''
-
-      Note: It is assumed right now that extra lines will be only at the start and not at the end.
-
-      Sample expected JSON to be passed for 'loads' is either of the form :
-
-      Case 'A':
-      {
-          "amInfo" : {
-          "appName" : "llap0",
-          "appType" : "org-apache-slider",
-          "appId" : "APP1",
-          "containerId" : "container_1466036628595_0010_01_000001",
-          "hostname" : "hostName",
-          "amWebUrl" : "http://hostName:port/"
-        },
-        "state" : "LAUNCHING",
-        ....
-        "desiredInstances" : 1,
-        "liveInstances" : 0,
-        ....
-        ....
-      }
-
-      or
-
-      Case 'B':
-      {
-        "state" : "APP_NOT_FOUND"
-      }
-
-      '''
-      splits = output.split("\n")
-
-      len_splits = len(splits)
-      if (len_splits < 3):
-        raise Fail ("Malformed JSON data received from 'llapstatus' command. Exiting ....")
-
-      marker_idx = None # To detect where from to start reading for JSON data
-      for idx, split in enumerate(splits):
-        curr_elem = split.strip()
-        if idx+2 > len_splits:
-          raise Fail("Iterated over the received 'llapstatus' comamnd. Couldn't validate the received output for JSON parsing.")
-        next_elem = (splits[(idx + 1)]).strip()
-        if curr_elem == "{":
-          if next_elem == "\"amInfo\" : {" and (splits[len_splits-1]).strip() == '}':
-            # For Case 'A'
-            marker_idx = idx
-            break;
-          elif idx+3 == len_splits and next_elem.startswith('"state" : ') and (splits[idx + 2]).strip() == '}':
-              # For Case 'B'
-              marker_idx = idx
-              break;
-
-      Logger.info("Marker index for start of JSON data for 'llapsrtatus' comamnd : {0}".format(marker_idx))
-
-      # Remove extra logging from possible JSON output
-      if marker_idx is None:
-        raise Fail("Couldn't validate the received output for JSON parsing.")
-      else:
-        if marker_idx != 0:
-          del splits[0:marker_idx]
-          Logger.info("Removed lines: '1-{0}' from the received 'llapstatus' output to make it valid for JSON parsing.".format(marker_idx))
-
-      scanned_output = '\n'.join(splits)
-      llap_app_info = json.loads(scanned_output)
-      return llap_app_info
-
-
-    """
-    Checks llap app status. The states can be : 'COMPLETE', 'APP_NOT_FOUND', 'RUNNING_PARTIAL', 'RUNNING_ALL' & 'LAUNCHING'.
-
-    if app is in 'APP_NOT_FOUND', 'RUNNING_PARTIAL' and 'LAUNCHING' state:
-       we wait for 'num_times_to_wait' to have app in (1). 'RUNNING_ALL' or (2). 'RUNNING_PARTIAL'
-       state with 80% or more 'desiredInstances' running and Return True
-    else :
-       Return False
-
-    Parameters: llap_app_name : deployed llap app name.
-                num_retries :   Number of retries to check the LLAP app status.
-    """
-    def check_llap_app_status(self, llap_app_name, num_retries):
-      # counters based on various states.
-      curr_time = time.time()
-
-      if num_retries <= 0:
-        num_retries = 2
-      if num_retries > 20:
-        num_retries = 20
-      @retry(times=num_retries, sleep_time=2, err_class=Fail)
-      def do_retries():
-        live_instances = 0
-        desired_instances = 0
-
-        percent_desired_instances_to_be_up = 80 # Used in 'RUNNING_PARTIAL' state.
-        llap_app_info = self._get_llap_app_status_info(llap_app_name)
-        if llap_app_info is None or 'state' not in llap_app_info:
-          Logger.error("Malformed JSON data received for LLAP app. Exiting ....")
-          return False
-
-        if llap_app_info['state'].upper() == 'RUNNING_ALL':
-          Logger.info(
-            "LLAP app '{0}' in '{1}' state.".format(llap_app_name, llap_app_info['state']))
-          return True
-        elif llap_app_info['state'].upper() == 'RUNNING_PARTIAL':
-          # Check how many instances were up.
-          if 'liveInstances' in llap_app_info and 'desiredInstances' in llap_app_info:
-            live_instances = llap_app_info['liveInstances']
-            desired_instances = llap_app_info['desiredInstances']
-          else:
-            Logger.info(
-              "LLAP app '{0}' is in '{1}' state, but 'instances' information not available in JSON received. " \
-              "Exiting ....".format(llap_app_name, llap_app_info['state']))
-            Logger.info(llap_app_info)
-            return False
-          if desired_instances == 0:
-            Logger.info("LLAP app '{0}' desired instance are set to 0. Exiting ....".format(llap_app_name))
-            return False
-
-          percentInstancesUp = 0
-          if live_instances > 0:
-            percentInstancesUp = float(live_instances) / desired_instances * 100
-          if percentInstancesUp >= percent_desired_instances_to_be_up:
-            Logger.info("LLAP app '{0}' in '{1}' state. Live Instances : '{2}'  >= {3}% of Desired Instances : " \
-                        "'{4}'.".format(llap_app_name, llap_app_info['state'],
-                                       llap_app_info['liveInstances'],
-                                       percent_desired_instances_to_be_up,
-                                       llap_app_info['desiredInstances']))
-            return True
-          else:
-            Logger.info("LLAP app '{0}' in '{1}' state. Live Instances : '{2}'. Desired Instances : " \
-                        "'{3}' after {4} secs.".format(llap_app_name, llap_app_info['state'],
-                                                       llap_app_info['liveInstances'],
-                                                       llap_app_info['desiredInstances'],
-                                                       time.time() - curr_time))
-            raise Fail("App state is RUNNING_PARTIAL. Live Instances : '{0}', Desired Instance : '{1}'".format(llap_app_info['liveInstances'],
-                                                                                                           llap_app_info['desiredInstances']))
-        elif llap_app_info['state'].upper() in ['APP_NOT_FOUND', 'LAUNCHING', 'COMPLETE']:
-          status_str = format("LLAP app '{0}' current state is {1}.".format(llap_app_name, llap_app_info['state']))
-          Logger.info(status_str)
-          raise Fail(status_str)
-        else:  # Covers any unknown that we get.
-          Logger.info(
-            "LLAP app '{0}' current state is '{1}'. Expected : 'RUNNING'.".format(llap_app_name, llap_app_info['state']))
-          return False
-
-      try:
-        status = do_retries()
-        return status
-      except Exception, e:
-        Logger.info("LLAP app '{0}' did not come up after a wait of {1} seconds.".format(llap_app_name,
-                                                                                          time.time() - curr_time))
-        traceback.print_exc()
-        return False
-
-    def get_log_folder(self):
-      import params
-      return params.hive_log_dir
-
-    def get_user(self):
-      import params
-      return params.hive_user
-
-@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
-class HiveServerInteractiveWindows(HiveServerInteractive):
-
-  def status(self, env):
-    pass
-
-if __name__ == "__main__":
-  HiveServerInteractive().execute()
\ No newline at end of file


Mime
View raw message