Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 269FD18439 for ; Wed, 19 Aug 2015 17:55:21 +0000 (UTC) Received: (qmail 18619 invoked by uid 500); 19 Aug 2015 17:55:17 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 18588 invoked by uid 500); 19 Aug 2015 17:55:17 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 18579 invoked by uid 99); 19 Aug 2015 17:55:17 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 19 Aug 2015 17:55:17 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B2B38E35DB; Wed, 19 Aug 2015 17:55:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dsen@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-12817 YARN ownership is not set correctly on container-executor (dsen) Date: Wed, 19 Aug 2015 17:55:17 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/branch-2.1 938c276d7 -> 67a05f3c4 AMBARI-12817 YARN ownership is not set correctly on container-executor (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/67a05f3c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/67a05f3c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/67a05f3c Branch: refs/heads/branch-2.1 Commit: 67a05f3c45b8067c42efd38a664eb6fae05736c0 Parents: 938c276 Author: Dmytro Sen Authored: Wed Aug 19 20:39:35 2015 +0300 Committer: Dmytro Sen Committed: Wed Aug 19 20:55:06 2015 +0300 ---------------------------------------------------------------------- ...ackLevelConfigurationResourceDefinition.java | 11 +++---- .../server/api/services/AmbariMetaInfo.java | 5 +++- .../apache/ambari/server/stack/StackModule.java | 31 ++++++++++++++------ .../YARN/2.1.0.2.0/configuration/yarn-site.xml | 6 ++++ .../src/main/resources/key_properties.json | 1 + .../stacks/HDP/2.0.6/services/stack_advisor.py | 6 ++++ .../services/YARN/configuration/yarn-site.xml | 4 +++ .../stacks/2.0.6/common/test_stack_advisor.py | 5 +++- .../stacks/2.2/common/test_stack_advisor.py | 22 ++++++++++++-- .../app/controllers/wizard/step7_controller.js | 12 ++++++-- ambari-web/app/mixins/common/serverValidator.js | 10 +++++++ 11 files changed, 92 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java index ce6a5b3..5a6ff7e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java @@ -27,7 +27,7 @@ import org.apache.ambari.server.controller.spi.Resource.Type; public class StackLevelConfigurationResourceDefinition extends BaseResourceDefinition { public StackLevelConfigurationResourceDefinition(Type resourceType) { - super(Resource.Type.StackLevelConfiguration); + super(resourceType); } public StackLevelConfigurationResourceDefinition() { @@ -43,11 +43,12 @@ public class StackLevelConfigurationResourceDefinition extends BaseResourceDefin public String getSingularName() { return "configuration"; } - /* + @Override public Set getSubResourceDefinitions() { - Set setChildren = new HashSet(); - return setChildren; + Set subs = new HashSet(); + subs.add(new SubResourceDefinition(Resource.Type.StackConfigurationDependency)); + + return subs; } - */ } http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 4afa9b0..561b3f4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -631,7 +631,10 @@ public class AmbariMetaInfo { public Set getPropertiesByName(String stackName, String version, String serviceName, String propertyName) throws AmbariException { - Set properties = getServiceProperties(stackName, version, serviceName); + + Set properties = serviceName == null ? + getStackProperties(stackName, version) + : getServiceProperties(stackName, version, serviceName); if (properties.size() == 0) { throw new StackAccessException("stackName=" + stackName http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index 4b88aff..2df6e04 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -590,7 +590,8 @@ public class StackModule extends BaseModule implements V // Stack-definition has 'depends-on' relationship specified. // We have a map to construct the 'depended-by' relationship. - Map> dependedByMap = new HashMap>(); + Map> dependedByMap = + new HashMap>(); // Go through all service-configs and gather the reversed 'depended-by' // relationship into map. Since we do not have the reverse {@link PropertyInfo}, @@ -616,14 +617,26 @@ public class StackModule extends BaseModule implements V // Go through all service-configs again and set their 'depended-by' if necessary. for (ServiceModule serviceModule : serviceModules.values()) { - for (PropertyInfo pi : serviceModule.getModuleInfo().getProperties()) { - String type = ConfigHelper.fileNameToConfigType(pi.getFilename()); - String name = pi.getName(); - Set set = - dependedByMap.remove(new PropertyDependencyInfo(type, name)); - if (set != null) { - pi.getDependedByProperties().addAll(set); - } + addDependedByProperties(dependedByMap, serviceModule.getModuleInfo().getProperties()); + } + // Go through all stack-configs again and set their 'depended-by' if necessary. + addDependedByProperties(dependedByMap, stackInfo.getProperties()); + } + + /** + * Add dependendByProperties to property info's + * @param dependedByMap Map containing the 'depended-by' relationships + * @param properties properties to check against dependedByMap + */ + private void addDependedByProperties(Map> dependedByMap, + Collection properties) { + for (PropertyInfo pi : properties) { + String type = ConfigHelper.fileNameToConfigType(pi.getFilename()); + String name = pi.getName(); + Set set = + dependedByMap.remove(new PropertyDependencyInfo(type, name)); + if (set != null) { + pi.getDependedByProperties().addAll(set); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml index 686e40c..26a74df 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml @@ -178,6 +178,12 @@ yarn.nodemanager.linux-container-executor.group hadoop Unix group of the NodeManager + + + cluster-env + user_group + + http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-server/src/main/resources/key_properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/key_properties.json b/ambari-server/src/main/resources/key_properties.json index ff20b8b..9d71981 100644 --- a/ambari-server/src/main/resources/key_properties.json +++ b/ambari-server/src/main/resources/key_properties.json @@ -70,6 +70,7 @@ "StackVersion": "StackConfigurationDependency/stack_version", "StackService": "StackConfigurationDependency/service_name", "StackConfiguration": "StackConfigurationDependency/property_name", + "StackLevelConfiguration": "StackConfigurationDependency/property_name", "StackConfigurationDependency": "StackConfigurationDependency/dependency_name" }, "StackServiceComponent": { http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 3fd010a..5fa9f41 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -130,6 +130,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) putYarnEnvProperty('min_user_id', self.get_system_min_uid()) + containerExecutorGroup = 'hadoop' + if 'cluster-env' in services['configurations'] and 'user_group' in services['configurations']['cluster-env']['properties']: + containerExecutorGroup = services['configurations']['cluster-env']['properties']['user_group'] + putYarnProperty("yarn.nodemanager.linux-container-executor.group", containerExecutorGroup) def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts): putMapredProperty = self.putProperty(configurations, "mapred-site", services) @@ -722,8 +726,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return self.toConfigurationValidationProblems(validationItems, "mapred-site") def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + clusterEnv = getSiteProperties(configurations, "cluster-env") validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')}, {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')}, + {"config-name": 'yarn.nodemanager.linux-container-executor.group', "item": self.validatorEqualsPropertyItem(properties, "yarn.nodemanager.linux-container-executor.group", clusterEnv, "user_group")}, {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ] return self.toConfigurationValidationProblems(validationItems, "yarn-site") http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml index 049eeb2..c69fd39 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml @@ -452,6 +452,10 @@ yarn-env yarn_cgroups_enabled + + cluster-env + user_group + http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 8fee562..86a34d9 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -202,6 +202,7 @@ class TestHDP206StackAdvisor(TestCase): expectedItems = [ {"message": "Value is less than the recommended default of 512", "level": "WARN"}, + {'message': 'Value should be set for yarn.nodemanager.linux-container-executor.group', 'level': 'ERROR'}, {"message": "Value should be integer", "level": "ERROR"}, {"message": "Value should be set", "level": "ERROR"} ] @@ -466,6 +467,7 @@ class TestHDP206StackAdvisor(TestCase): def test_recommendYARNConfigurations(self): configurations = {} + services = {"configurations": configurations} clusterData = { "containers" : 5, "ramPerContainer": 256 @@ -478,6 +480,7 @@ class TestHDP206StackAdvisor(TestCase): }, "yarn-site": { "properties": { + "yarn.nodemanager.linux-container-executor.group": "hadoop", "yarn.nodemanager.resource.memory-mb": "1280", "yarn.scheduler.minimum-allocation-mb": "256", "yarn.scheduler.maximum-allocation-mb": "1280" @@ -485,7 +488,7 @@ class TestHDP206StackAdvisor(TestCase): } } - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, None, None) + self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None) self.assertEquals(configurations, expected) def test_recommendMapReduce2Configurations_mapMemoryLessThan2560(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index e764739..851c8b5 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -690,6 +690,7 @@ class TestHDP22StackAdvisor(TestCase): def test_recommendYARNConfigurations(self): configurations = {} + services = {"configurations": configurations} clusterData = { "cpu": 4, "containers" : 5, @@ -703,6 +704,7 @@ class TestHDP22StackAdvisor(TestCase): }, "yarn-site": { "properties": { + "yarn.nodemanager.linux-container-executor.group": "hadoop", "yarn.nodemanager.resource.memory-mb": "1280", "yarn.scheduler.minimum-allocation-mb": "256", "yarn.scheduler.maximum-allocation-mb": "1280", @@ -713,7 +715,7 @@ class TestHDP22StackAdvisor(TestCase): } } - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, None, None) + self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None) self.assertEquals(configurations, expected) def test_recommendYARNConfigurationAttributes(self): @@ -745,6 +747,7 @@ class TestHDP22StackAdvisor(TestCase): }, "yarn-site": { "properties": { + "yarn.nodemanager.linux-container-executor.group": "hadoop", "yarn.nodemanager.resource.memory-mb": "1280", "yarn.scheduler.minimum-allocation-mb": "256", "yarn.scheduler.maximum-allocation-vcores": "2", @@ -1303,7 +1306,12 @@ class TestHDP22StackAdvisor(TestCase): "yarn.scheduler.maximum-allocation-mb": "2048", "yarn.nodemanager.resource.cpu-vcores": "2" }, - } + }, + "cluster-env": { + "properties": { + "user_group": "hadoopcustom", + } + } } clusterData = { "cpu": 4, @@ -1312,6 +1320,11 @@ class TestHDP22StackAdvisor(TestCase): "totalAvailableRam": 4096, } expected = { + "cluster-env": { + "properties": { + "user_group": "hadoopcustom" + } + }, "yarn-env": { "properties": { "min_user_id": "500" @@ -1339,6 +1352,7 @@ class TestHDP22StackAdvisor(TestCase): }, "yarn-site": { "properties": { + "yarn.nodemanager.linux-container-executor.group": "hadoopcustom", "yarn.nodemanager.resource.memory-mb": "1792", "yarn.scheduler.minimum-allocation-mb": "100", "yarn.scheduler.maximum-allocation-vcores": "1", @@ -1597,6 +1611,7 @@ class TestHDP22StackAdvisor(TestCase): }, "yarn-site": { "properties": { + "yarn.nodemanager.linux-container-executor.group": "hadoop", "yarn.nodemanager.resource.memory-mb": "1280", "yarn.scheduler.minimum-allocation-mb": "100", "yarn.scheduler.maximum-allocation-vcores": "1", @@ -1807,6 +1822,7 @@ class TestHDP22StackAdvisor(TestCase): }, "yarn-site": { "properties": { + "yarn.nodemanager.linux-container-executor.group": "hadoop", "yarn.nodemanager.resource.memory-mb": "1280", "yarn.scheduler.minimum-allocation-mb": "700", "yarn.scheduler.maximum-allocation-vcores": "1", @@ -2687,6 +2703,7 @@ class TestHDP22StackAdvisor(TestCase): }, "yarn-site": { "properties": { + "yarn.nodemanager.linux-container-executor.group": "hadoop", "yarn.nodemanager.container-executor.group": "hadoop", "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor", "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "/cgroup", @@ -2743,6 +2760,7 @@ class TestHDP22StackAdvisor(TestCase): "yarn.nodemanager.container-executor.group": "hadoop", "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor", "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "/cgroup", + "yarn.nodemanager.linux-container-executor.group": "hadoop", "yarn.nodemanager.container-executor.cgroups.mount": "true", "yarn.nodemanager.resource.memory-mb": "39424", "yarn.scheduler.minimum-allocation-mb": "3584", http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-web/app/controllers/wizard/step7_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/wizard/step7_controller.js b/ambari-web/app/controllers/wizard/step7_controller.js index 1d3eea4..293ea03 100644 --- a/ambari-web/app/controllers/wizard/step7_controller.js +++ b/ambari-web/app/controllers/wizard/step7_controller.js @@ -882,7 +882,7 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, App.E var serviceConfigTags = []; for (var site in data.Clusters.desired_configs) { if (data.Clusters.desired_configs.hasOwnProperty(site)) { - if (installedServiceSites.contains(site)) { + if (installedServiceSites.contains(site) || site == 'cluster-env') { serviceConfigTags.push({ siteName: site, tagName: data.Clusters.desired_configs[site].tag, @@ -1182,8 +1182,14 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, App.E var serviceToShow = this.get('selectedServiceNames').concat('MISC'); var miscConfigs = this.get('stepConfigs').findProperty('serviceName', 'MISC').configs; if (this.get('wizardController.name') == "addServiceController") { - miscConfigs.findProperty('name', 'smokeuser').set('value', this.get('content.smokeuser')).set('isEditable', false); - miscConfigs.findProperty('name', 'user_group').set('value', this.get('content.group')).set('isEditable', false); + miscConfigs.findProperty('name', 'smokeuser').set('isEditable', false); + miscConfigs.findProperty('name', 'user_group').set('isEditable', false); + if (this.get('content.smokeuser')) { + miscConfigs.findProperty('name', 'smokeuser').set('value', this.get('content.smokeuser')); + } + if (this.get('content.group')) { + miscConfigs.findProperty('name', 'user_group').set('value', this.get('content.group')); + } } App.config.miscConfigVisibleProperty(miscConfigs, serviceToShow); } http://git-wip-us.apache.org/repos/asf/ambari/blob/67a05f3c/ambari-web/app/mixins/common/serverValidator.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/mixins/common/serverValidator.js b/ambari-web/app/mixins/common/serverValidator.js index b0c0772..9d696c2 100644 --- a/ambari-web/app/mixins/common/serverValidator.js +++ b/ambari-web/app/mixins/common/serverValidator.js @@ -129,7 +129,17 @@ App.ServerValidatorMixin = Em.Mixin.create({ recommendations.blueprint.configurations = blueprintUtils.buildConfigsJSON(this.get('services'), this.get('stepConfigs').filter(function(serviceConfigs) { return self.get('installedServiceNames').contains(serviceConfigs.get('serviceName')); })); + // include cluster-env site to recommendations call + var miscService = this.get('services').findProperty('serviceName', 'MISC'); + if (miscService) { + var miscConfigs = blueprintUtils.buildConfigsJSON([miscService], [this.get('stepConfigs').findProperty('serviceName', 'MISC')]); + var clusterEnv = App.permit(miscConfigs, 'cluster-env'); + if (!App.isEmptyObject(clusterEnv)) { + $.extend(recommendations.blueprint.configurations, clusterEnv); + } + } } + return App.ajax.send({ 'name': 'config.recommendations', 'sender': this,