ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-17983. (1). Revert AMBARI-16031 (Create /hadoop/llap/local on each host and disk in Kerberized cluster for LLAP), and (2). Remove the config 'hive.llap.daemon.work.dirs' as HIVE will manage the work directories itself.
Date Wed, 03 Aug 2016 19:31:42 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 9bb772392 -> da1701de9


AMBARI-17983. (1). Revert AMBARI-16031 (Create /hadoop/llap/local on each host and disk in
Kerberized cluster for LLAP), and (2). Remove the config 'hive.llap.daemon.work.dirs' as HIVE
will manage the work directories itself.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/da1701de
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/da1701de
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/da1701de

Branch: refs/heads/trunk
Commit: da1701de99e55997e39e0f644c14e7ff2f927fb7
Parents: 9bb7723
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Mon Aug 1 16:04:40 2016 -0700
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Wed Aug 3 12:31:22 2016 -0700

----------------------------------------------------------------------
 .../common-services/YARN/2.1.0.2.0/metainfo.xml | 10 ------
 .../2.1.0.2.0/package/scripts/nodemanager.py    | 12 +------
 .../2.1.0.2.0/package/scripts/params_linux.py   | 34 ++----------------
 .../YARN/2.1.0.2.0/package/scripts/yarn.py      | 33 +----------------
 .../configuration/hive-interactive-site.xml     | 10 ------
 .../stacks/HDP/2.5/services/HIVE/kerberos.json  |  5 ---
 ambari-web/app/controllers/main/service/item.js | 38 --------------------
 ambari-web/app/messages.js                      |  3 --
 ambari-web/app/models/host_component.js         |  7 ----
 ambari-web/app/views/main/service/item.js       | 13 +------
 ambari-web/test/views/main/service/item_test.js |  2 --
 11 files changed, 6 insertions(+), 161 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
index 35329bc..f80fa8a 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
@@ -91,16 +91,6 @@
              <!-- Used by decommission and recommission -->
              <masterComponent>RESOURCEMANAGER</masterComponent>
            </bulkCommands>
-          <customCommands>
-            <customCommand>
-              <name>CREATE_YARN_DIRECTORIES</name>
-              <commandScript>
-                <script>scripts/nodemanager.py</script>
-                <scriptType>PYTHON</scriptType>
-                <timeout>600</timeout>
-              </commandScript>
-            </customCommand>
-          </customCommands>
           <logs>
             <log>
               <logId>yarn_nodemanager</logId>

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
index 106b734..bc8947a 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
@@ -31,7 +31,7 @@ from resource_management.libraries.functions.security_commons import build_expec
   cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties,
\
   FILE_TYPE_XML
 from resource_management.core.logger import Logger
-from yarn import yarn, create_hive_llap_work_dir
+from yarn import yarn
 from service import service
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
@@ -90,16 +90,6 @@ class NodemanagerDefault(Nodemanager):
     env.set_params(status_params)
     check_process_status(status_params.nodemanager_pid_file)
 
-  def create_yarn_directories(self, env):
-    """
-    Custom command to Create Directories, e.g., needed by YARN Apps on Slider
-    """
-    Logger.info("Custom Command to Create Directories")
-    import params
-    env.set_params(params)
-    create_hive_llap_work_dir(params)
-
-
   def security_status(self, env):
     import status_params
     env.set_params(status_params)

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 154e433..b2c635d 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -19,7 +19,6 @@ Ambari Agent
 
 """
 import os
-import re
 
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.resources.hdfs_resource import HdfsResource
@@ -204,39 +203,12 @@ if hostname and nm_address and nm_address.startswith("0.0.0.0:"):
 nm_local_dirs = default("/configurations/yarn-site/yarn.nodemanager.local-dirs", "")
 nm_log_dirs = default("/configurations/yarn-site/yarn.nodemanager.log-dirs", "")
 
+nm_local_dirs_list = nm_local_dirs.split(',')
+nm_log_dirs_list = nm_log_dirs.split(',')
+
 nm_log_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_log_dir_mount.hist"
 nm_local_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_local_dir_mount.hist"
 
-# If Hive Interactive Server is present, then default value will be ${yarn.nodemanager.local-dirs}
-# During kerberization, kerberos.json changes the value to "/hadoop/llap/local".
-# This is a comma separated list of directories.
-hive_llap_work_dirs = default("/configurations/hive-interactive-site/hive.llap.daemon.work.dirs",
"")
-
-# It can also be a dynamic variable, such as "${yarn.nodemanager.local-dirs}"
-p = re.compile(r"\$\{(.*?)\}")
-m = p.search(hive_llap_work_dirs)
-if m and len(m.groups()) == 1:
-  hive_llap_work_dirs = default("/configurations/yarn-site/" + m.group(1), "")
-
-nm_local_dirs_list = nm_local_dirs.split(",")
-nm_log_dirs_list = nm_log_dirs.split(",")
-hive_llap_work_dirs_list = hive_llap_work_dirs.split(",")
-
-
-def cleanse_list(l):
-  if l is None:
-    return None
-  l = [e.strip() for e in l if e.strip() != ""]
-  return l if len(l) > 0 else None
-
-nm_local_dirs_list = cleanse_list(nm_local_dirs_list)
-nm_log_dirs_list = cleanse_list(nm_log_dirs_list)
-hive_llap_work_dirs_list = cleanse_list(hive_llap_work_dirs_list)
-
-# Needed for LLAP to determine if should create Hive LLAP Work Dirs.
-hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts',
[])
-hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts)
> 0 else None
-
 distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar"
 hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index cf5736e..93c3aaa 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -201,8 +201,6 @@ def yarn(name=None, config_dir=None):
            mode=0644,
            content=nm_local_dir_to_mount_file_content
       )
-
-    create_hive_llap_work_dir(params)
   #</editor-fold>
 
   if params.yarn_nodemanager_recovery_dir:
@@ -497,33 +495,4 @@ def yarn(name=None, config_dir=None):
     File(os.path.join(config_dir, 'ssl-server.xml.example'),
          owner=params.mapred_user,
          group=params.user_group
-    )
-
-
-def create_hive_llap_work_dir(params):
-  """
-  Create the work directory needed by LLAP, which is required on all NodeManagers.
-  This needs to be whenever NodeManagers are restarted, or after Hive Server Interactive
and LLAP are added and started
-  via a custom command
-  :param params: Command parameters dictionary.
-  """
-  if params.hive_llap_work_dirs_list is None or params.hive_server_interactive_host is None
or not params.security_enabled:
-    Logger.info("Skip creating any Hive LLAP work dir since either it's empty, Hive Interactive
is not present, "
-                "or cluster is not kerberized.")
-    return
-
-  # If we already created this dir list because it has the same value as NM Local Dirs, then
skip it
-  skip = False
-  if params.nm_local_dirs_list is not None and set(params.nm_local_dirs_list) == set(params.hive_llap_work_dirs_list):
-    skip = True
-
-  if skip:
-    Logger.info(format("Skip creating Hive LLAP Work Dirs since it is equivalent to NM Local
Dirs: {nm_local_dirs_list}"))
-  else:
-    Logger.info(format("Hive Server Interactive is present on the cluster, ensure that the
Hive LLAP work dirs exist: {hive_llap_work_dirs_list}"))
-    Directory(params.hive_llap_work_dirs_list,
-              owner=params.yarn_user,
-              group=params.user_group,
-              create_parents=True,
-              ignore_failures=False,
-              mode=0775)
+    )
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 776c2b6..14ab470 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -607,16 +607,6 @@ limitations under the License.
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
-    <name>hive.llap.daemon.work.dirs</name>
-    <value>${yarn.nodemanager.local-dirs}</value>
-    <description>
-      Working directories for the daemon. Needs to be set for a secure cluster, since LLAP
may
-      not have access to the default YARN working directories. yarn.nodemanager.local-dirs
is
-      used if this is not set
-    </description>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
     <name>hive.llap.zk.sm.connectionString</name>
     <value>localhost:2181</value>
     <description>ZooKeeper connection string for ZooKeeper SecretManager.</description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
index 4fae665..37a7e82 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
@@ -29,11 +29,6 @@
           }
         },
         {
-          "hive-interactive-site": {
-            "hive.llap.daemon.work.dirs": "/hadoop/llap/local"
-          }
-        },
-        {
           "ranger-hive-audit": {
             "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",
             "xasecure.audit.jaas.Client.loginModuleControlFlag": "required",

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/app/controllers/main/service/item.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/item.js b/ambari-web/app/controllers/main/service/item.js
index e3df70b..6ce811d 100644
--- a/ambari-web/app/controllers/main/service/item.js
+++ b/ambari-web/app/controllers/main/service/item.js
@@ -693,44 +693,6 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow
     App.showAlertPopup(Em.I18n.t('common.error'), error);
   },
 
-  createYARNDirectories: function(event) {
-    var context = Em.I18n.t('services.service.actions.run.createYARNDirectories');
-    var command = "CREATE_YARN_DIRECTORIES";
-    var component = "NODEMANAGER";
-    var controller = this;
-    var hosts = App.Service.find('YARN').get('hostComponents').filterProperty('componentName',
component).mapProperty('hostName');
-    return App.showConfirmationPopup(function() {
-      App.ajax.send({
-        name: 'service.item.executeCustomCommand',
-        sender: controller,
-        data: {
-          command: command,
-          context: context,
-          hosts: hosts.join(','),
-          serviceName: "YARN",
-          componentName: component
-        },
-        success: 'createYARNDirectoriesSuccessCallback',
-        error: 'createYARNDirectoriesErrorCallback'
-      });
-    }, Em.I18n.t('services.service.actions.run.createYARNDirectories.confirmation'), null,
Em.I18n.t('popup.confirmation.commonHeader'), Em.I18n.t('ok'), false);
-  },
-  createYARNDirectoriesSuccessCallback : function(data, ajaxOptions, params) {
-    if (data.Requests.id) {
-      App.router.get('backgroundOperationsController').showPopup();
-    }
-  },
-  createYARNDirectoriesErrorCallback : function(data) {
-    var error = Em.I18n.t('services.service.actions.run.executeCustomCommand.error');
-    if (data && data.responseText) {
-      try {
-        var json = $.parseJSON(data.responseText);
-        error += json.message;
-      } catch (err) {}
-    }
-    App.showAlertPopup(Em.I18n.t('common.error'), error);
-  },
-
   /**
    * On click handler for rebalance Hdfs command from items menu
    */

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index 2d7ed6e..1f42685 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -1847,9 +1847,6 @@ Em.I18n.translations = {
 
   // Hive Server Interactive custom command to restart LLAP
   'services.service.actions.run.restartLLAP':'Restart LLAP',
-  // Node Manager custom command to refresh YARN Apps on Slider
-  'services.service.actions.run.createYARNDirectories':'Create YARN Directories',
-  'services.service.actions.run.createYARNDirectories.confirmation':'Please confirm. This
will create necessary folders on the NodeManagers required for running YARN Apps on Slider,
such as LLAP. This does not need to restart NodeManagers.',
   'services.service.actions.run.immediateStopHawqService.context':'Stop HAWQ Service (Immediate
Mode)',
   'services.service.actions.run.immediateStopHawqService.label':'Stop HAWQ Service (Immediate
Mode)',
   'services.service.actions.run.immediateStopHawqSegment.label':'Stop (Immediate Mode)',

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/app/models/host_component.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/host_component.js b/ambari-web/app/models/host_component.js
index 91642c9..dca7ff0 100644
--- a/ambari-web/app/models/host_component.js
+++ b/ambari-web/app/models/host_component.js
@@ -372,13 +372,6 @@ App.HostComponentActionMap = {
         label: Em.I18n.t('services.service.actions.run.restartLLAP') + ' ∞',
         cssClass: 'icon-refresh'
       },
-      CREATE_YARN_DIRECTORIES: {
-        action: 'createYARNDirectories',
-        customCommand: 'CREATE_YARN_DIRECTORIES',
-        context: Em.I18n.t('services.service.actions.run.createYARNDirectories'),
-        label: Em.I18n.t('services.service.actions.run.createYARNDirectories'),
-        cssClass: 'icon-refresh'
-      },
       REBALANCEHDFS: {
         action: 'rebalanceHdfsNodes',
         customCommand: 'REBALANCEHDFS',

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/app/views/main/service/item.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/main/service/item.js b/ambari-web/app/views/main/service/item.js
index 02ac426..a007e17 100644
--- a/ambari-web/app/views/main/service/item.js
+++ b/ambari-web/app/views/main/service/item.js
@@ -40,8 +40,7 @@ App.MainServiceItemView = Em.View.extend({
     'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE', 'REMOVE_HAWQ_STANDBY',
'RESYNC_HAWQ_STANDBY'],
     'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'],
     'HAWQSTANDBY': ['ACTIVATE_HAWQ_STANDBY'],
-    'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"],
-    'NODEMANAGER' : ["CREATE_YARN_DIRECTORIES"]
+    'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"]
   },
 
    addActionMap: function() {
@@ -142,16 +141,6 @@ App.MainServiceItemView = Em.View.extend({
       }
       if (this.get('serviceName') === 'YARN') {
         options.push(actionMap.REFRESHQUEUES);
-        var nodeManagerComponent = App.StackServiceComponent.find().findProperty('componentName',
'NODEMANAGER');
-        var isNodeManagerPresent = allSlaves.contains('NODEMANAGER');
-        if (nodeManagerComponent && isNodeManagerPresent) {
-          var nodeManagerCustomCommands = nodeManagerComponent.get('customCommands');
-          nodeManagerCustomCommands.forEach(function (command) {
-            if (actionMap[command]) {
-              options.push(actionMap[command]);
-            }
-          });
-        }
       }
       options.push(actionMap.RESTART_ALL);
       allSlaves.concat(allMasters).filter(function (_component) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/test/views/main/service/item_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/main/service/item_test.js b/ambari-web/test/views/main/service/item_test.js
index ecfa7de..f318185 100644
--- a/ambari-web/test/views/main/service/item_test.js
+++ b/ambari-web/test/views/main/service/item_test.js
@@ -494,8 +494,6 @@ describe('App.MainServiceItemView', function () {
             return Em.Object.create({ customCommands: ["DECOMMISSION", "REBALANCEHDFS"] });
           case 'RESOURCEMANAGER':
             return Em.Object.create({ customCommands: ["DECOMMISSION", "REFRESHQUEUES"] });
-          case 'NODEMANAGER':
-            return Em.Object.create({ customCommands: ["REFRESHQUEUES", "CREATE_YARN_DIRECTORIES"]
});
           case 'HBASE_MASTER':
             return Em.Object.create({ customCommands: ["DECOMMISSION"] });
           case 'KNOX_GATEWAY':


Mime
View raw message