ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ababiic...@apache.org
Subject [19/32] ambari git commit: AMBARI-21729 Owner of yarn.include stays to be root:root where as dfs.include gets changed to the correct one upon restart services (dsen)
Date Thu, 17 Aug 2017 10:16:25 GMT
AMBARI-21729 Owner of yarn.include stays to be root:root where as dfs.include gets changed
to the correct one upon restart services (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5c78bd75
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5c78bd75
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5c78bd75

Branch: refs/heads/branch-feature-logsearch-ui
Commit: 5c78bd752aa5a064966ecee9fcee374cf12eb7a8
Parents: ab8d58b
Author: Dmytro Sen <dsen@apache.org>
Authored: Wed Aug 16 12:34:02 2017 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Wed Aug 16 12:34:02 2017 +0300

----------------------------------------------------------------------
 .../2.1.0.2.0/package/scripts/params_linux.py   |    5 +-
 .../YARN/2.1.0.2.0/package/scripts/yarn.py      |   14 +-
 .../3.0.0.3.0/package/scripts/params_linux.py   |    5 +-
 .../YARN/3.0.0.3.0/package/scripts/yarn.py      |   14 +-
 .../stacks/2.0.6/YARN/test_resourcemanager.py   |  160 ++-
 .../default_yarn_include_file_dont_manage.json  | 1260 ++++++++++++++++++
 .../default_yarn_include_file_manage.json       | 1260 ++++++++++++++++++
 .../secured_yarn_include_file_dont_manage.json  | 1078 +++++++++++++++
 .../secured_yarn_include_file_manage.json       | 1078 +++++++++++++++
 .../YARN/package/scripts/params_linux.py        |    5 +-
 .../2.0/services/YARN/package/scripts/yarn.py   |   16 +-
 11 files changed, 4880 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 0b1215b..2ab779f 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -169,9 +169,6 @@ rm_hosts = config['clusterHostInfo']['rm_host']
 rm_host = rm_hosts[0]
 rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1]
 rm_https_port = default('/configurations/yarn-site/yarn.resourcemanager.webapp.https.address',
":8090").split(':')[-1]
-# TODO UPGRADE default, update site during upgrade
-rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
-rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path)
 
 java64_home = config['hostLevelParams']['java_home']
 java_exec = format("{java64_home}/bin/java")
@@ -241,6 +238,7 @@ user_group = config['configurations']['cluster-env']['user_group']
 #exclude file
 exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
 exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
+rm_nodes_exclude_dir = os.path.dirname(exclude_file_path)
 
 nm_hosts = default("/clusterHostInfo/nm_hosts", [])
 #incude file
@@ -248,6 +246,7 @@ include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.node
 include_hosts = None
 manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
 if include_file_path and manage_include_files:
+  rm_nodes_include_dir = os.path.dirname(include_file_path)
   include_hosts = list(set(nm_hosts) - set(exclude_hosts))
 
 ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index 1177107..363d464 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -390,10 +390,22 @@ def setup_resourcemanager():
        create_parents=True,
        cd_access='a',
   )
-  File(params.rm_nodes_exclude_path,
+  File(params.exclude_file_path,
+       content=Template("exclude_hosts_list.j2"),
        owner=params.yarn_user,
        group=params.user_group
   )
+  if params.include_hosts:
+    Directory(params.rm_nodes_include_dir,
+      mode=0755,
+      create_parents=True,
+      cd_access='a',
+      )
+    File(params.include_file_path,
+      content=Template("include_hosts_list.j2"),
+      owner=params.yarn_user,
+      group=params.user_group
+    )
   File(params.yarn_job_summary_log,
      owner=params.yarn_user,
      group=params.user_group

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
index 68d17f0..e66ec3c 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
@@ -166,9 +166,6 @@ rm_hosts = config['clusterHostInfo']['rm_host']
 rm_host = rm_hosts[0]
 rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1]
 rm_https_port = default('/configurations/yarn-site/yarn.resourcemanager.webapp.https.address',
":8090").split(':')[-1]
-# TODO UPGRADE default, update site during upgrade
-rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
-rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path)
 
 java64_home = config['hostLevelParams']['java_home']
 java_exec = format("{java64_home}/bin/java")
@@ -238,6 +235,7 @@ user_group = config['configurations']['cluster-env']['user_group']
 #exclude file
 exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
 exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
+rm_nodes_exclude_dir = os.path.dirname(exclude_file_path)
 
 nm_hosts = default("/clusterHostInfo/nm_hosts", [])
 #incude file
@@ -245,6 +243,7 @@ include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.node
 include_hosts = None
 manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
 if include_file_path and manage_include_files:
+  rm_nodes_include_dir = os.path.dirname(include_file_path)
   include_hosts = list(set(nm_hosts) - set(exclude_hosts))
 
 ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
index 0591511..ef314f3 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
@@ -391,10 +391,22 @@ def setup_resourcemanager():
        create_parents=True,
        cd_access='a',
   )
-  File(params.rm_nodes_exclude_path,
+  File(params.exclude_file_path,
+       content=Template("exclude_hosts_list.j2"),
        owner=params.yarn_user,
        group=params.user_group
   )
+  if params.include_hosts:
+    Directory(params.rm_nodes_include_dir,
+      mode=0755,
+      create_parents=True,
+      cd_access='a',
+    )
+    File(params.include_file_path,
+      content=Template("include_hosts_list.j2"),
+      owner=params.yarn_user,
+      group=params.user_group
+    )
   # This depends on the parent directory already existing.
   File(params.yarn_job_summary_log,
      owner=params.yarn_user,

http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
index 8e92116..8a703df 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
@@ -50,6 +50,28 @@ class TestResourceManager(RMFTestCase):
     self.assert_configure_default()
     self.assertNoMoreResources()
 
+  def test_configure_default_with_include_file_dont_manage(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+                       classname="Resourcemanager",
+                       command="configure",
+                       config_file="default_yarn_include_file_dont_manage.json",
+                       stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assert_configure_default(is_include_file_configured=True, manage_include_files=False)
+    self.assertNoMoreResources()
+
+  def test_configure_default_with_include_file_manage(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+                       classname="Resourcemanager",
+                       command="configure",
+                       config_file="default_yarn_include_file_manage.json",
+                       stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assert_configure_default(is_include_file_configured=True, manage_include_files=True)
+    self.assertNoMoreResources()
+
   def test_start_default(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
                        classname="Resourcemanager",
@@ -100,6 +122,28 @@ class TestResourceManager(RMFTestCase):
     )
     self.assert_configure_secured()
 
+  def test_configure_secured_with_include_file_dont_manage(self):
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+                       classname="Resourcemanager",
+                       command="configure",
+                       config_file="secured_yarn_include_file_dont_manage.json",
+                       stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assert_configure_secured(is_include_file_configured=True, manage_include_files=False)
+
+  def test_configure_secured_with_include_file_manage(self):
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+                       classname="Resourcemanager",
+                       command="configure",
+                       config_file="secured_yarn_include_file_manage.json",
+                       stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assert_configure_secured(is_include_file_configured=True, manage_include_files=True)
+
   def test_start_secured(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
                        classname="Resourcemanager",
@@ -179,7 +223,95 @@ class TestResourceManager(RMFTestCase):
 
     self.assertNoMoreResources()
 
-  def assert_configure_default(self):
+  def test_decommission_default_with_include_file_dont_manage(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+                       classname = "Resourcemanager",
+                       command = "decommission",
+                       config_file="default_yarn_include_file_dont_manage.json",
+                       stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+        owner = 'yarn',
+        content = Template('exclude_hosts_list.j2'),
+        group = 'hadoop',
+    )
+    self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
+        environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
+        user = 'yarn',
+    )
+    self.assertNoMoreResources()
+
+  def test_decommission_default_with_include_file_manage(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+                       classname = "Resourcemanager",
+                       command = "decommission",
+                       config_file="default_yarn_include_file_manage.json",
+                       stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+                              owner = 'yarn',
+                              content = Template('exclude_hosts_list.j2'),
+                              group = 'hadoop',
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
+                              owner = 'yarn',
+                              content = Template('include_hosts_list.j2'),
+                              group = 'hadoop',
+                              )
+    self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
+                              environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
+                              user = 'yarn',
+                              )
+    self.assertNoMoreResources()
+
+  def test_decommission_secured_with_include_file_dont_manage(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+                       classname = "Resourcemanager",
+                       command = "decommission",
+                       config_file="secured_yarn_include_file_dont_manage.json",
+                       stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+        owner = 'yarn',
+        content = Template('exclude_hosts_list.j2'),
+        group = 'hadoop',
+    )
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab
rm/c6401.ambari.apache.org@EXAMPLE.COM; yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
+        environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
+        user = 'yarn',
+    )
+
+    self.assertNoMoreResources()
+
+  def test_decommission_secured_with_include_file_manage(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+                       classname = "Resourcemanager",
+                       command = "decommission",
+                       config_file="secured_yarn_include_file_manage.json",
+                       stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+        owner = 'yarn',
+        content = Template('exclude_hosts_list.j2'),
+        group = 'hadoop',
+    )
+    self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
+        owner = 'yarn',
+        content = Template('include_hosts_list.j2'),
+        group = 'hadoop',
+    )
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab
rm/c6401.ambari.apache.org@EXAMPLE.COM; yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
+        environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
+        user = 'yarn',
+    )
+
+    self.assertNoMoreResources()
+
+  def assert_configure_default(self, is_include_file_configured = False, manage_include_files=False):
     self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
       owner = 'yarn',
       group = 'hadoop',
@@ -235,9 +367,21 @@ class TestResourceManager(RMFTestCase):
         cd_access = 'a',
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+        content = Template('exclude_hosts_list.j2'),
         owner = 'yarn',
         group = 'hadoop',
     )
+    if is_include_file_configured and manage_include_files:
+      self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include',
+        mode = 0755,
+        create_parents = True,
+        cd_access = 'a',
+      )
+      self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
+        content = Template('include_hosts_list.j2'),
+        owner = 'yarn',
+        group = 'hadoop',
+      )
     self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
       owner = 'yarn',
       group = 'hadoop',
@@ -347,7 +491,7 @@ class TestResourceManager(RMFTestCase):
                               group = 'hadoop',
                               )
 
-  def assert_configure_secured(self):
+  def assert_configure_secured(self, is_include_file_configured = False, manage_include_files=False):
     self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
       owner = 'yarn',
       group = 'hadoop',
@@ -403,9 +547,21 @@ class TestResourceManager(RMFTestCase):
         cd_access = 'a',
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+        content = Template('exclude_hosts_list.j2'),
         owner = 'yarn',
         group = 'hadoop',
     )
+    if is_include_file_configured and manage_include_files:
+      self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include',
+        mode = 0755,
+        create_parents = True,
+        cd_access = 'a',
+      )
+      self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
+        content = Template('include_hosts_list.j2'),
+        owner = 'yarn',
+        group = 'hadoop',
+      )
     self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
       owner = 'yarn',
       group = 'hadoop',


Mime
View raw message