Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F378A200BF3 for ; Thu, 5 Jan 2017 13:26:45 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id F1D9A160B27; Thu, 5 Jan 2017 12:26:45 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C9A59160B26 for ; Thu, 5 Jan 2017 13:26:44 +0100 (CET) Received: (qmail 78282 invoked by uid 500); 5 Jan 2017 12:26:44 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 78272 invoked by uid 99); 5 Jan 2017 12:26:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 Jan 2017 12:26:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D53DFDFC2F; Thu, 5 Jan 2017 12:26:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ababiichuk@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-19383 Configs changes after adding/deleting host component: value shouldn't be changed if it differs in host names order only. (ababiichuk) Date: Thu, 5 Jan 2017 12:26:43 +0000 (UTC) archived-at: Thu, 05 Jan 2017 12:26:46 -0000 Repository: ambari Updated Branches: refs/heads/trunk 6b198cc71 -> 1527bd15c AMBARI-19383 Configs changes after adding/deleting host component: value shouldn't be changed if it differs in host names order only. (ababiichuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1527bd15 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1527bd15 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1527bd15 Branch: refs/heads/trunk Commit: 1527bd15c6226fbb8e690c64f68b593ac5088c2b Parents: 6b198cc Author: ababiichuk Authored: Thu Jan 5 13:57:21 2017 +0200 Committer: ababiichuk Committed: Thu Jan 5 14:34:46 2017 +0200 ---------------------------------------------------------------------- ambari-web/app/controllers/main/host/details.js | 94 ++++++------ ambari-web/app/utils/config.js | 42 +++++- .../configs/add_component_config_initializer.js | 5 +- .../configs/hosts_based_initializer_mixin.js | 2 +- .../test/controllers/main/host/details_test.js | 12 +- ambari-web/test/utils/config_test.js | 147 ++++++++++++++++++- 6 files changed, 244 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/controllers/main/host/details.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/host/details.js b/ambari-web/app/controllers/main/host/details.js index 4a7cce4..7df2c4e 100644 --- a/ambari-web/app/controllers/main/host/details.js +++ b/ambari-web/app/controllers/main/host/details.js @@ -923,10 +923,10 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow fileName: fileName, name: propertyName, value: currentValue - }; - var configProperty = initializer.initialValue(propertyDef, hostComponentsTopology, dependencies); + }, + configProperty = initializer.initialValue(propertyDef, hostComponentsTopology, dependencies); initializer.updateSiteObj(configs[fileName], configProperty); - if (this.get('isReconfigureRequired') && currentValue !== propertyDef.value) { + if (this.get('isReconfigureRequired') && currentValue !== configs[fileName][propertyName]) { var service = App.config.get('serviceByConfigTypeMap')[fileName]; propertiesToChange.pushObject({ propertyFileName: fileName, @@ -1101,9 +1101,10 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow fileName: fileName, name: propertyName, value: currentValue - }; - configs[fileName][propertyName] = Em.get(initializer.initialValue(propertyDef, localDB, dependencies), 'value'); - if (this.get('isReconfigureRequired') && propertyDef.value !== currentValue) { + }, + configProperty = initializer.initialValue(propertyDef, localDB, dependencies); + initializer.updateSiteObj(configs[fileName], configProperty); + if (this.get('isReconfigureRequired') && currentValue !== configs[fileName][propertyName]) { var service = App.config.get('serviceByConfigTypeMap')[fileName]; propertiesToChange.pushObject({ propertyFileName: fileName, @@ -1286,50 +1287,55 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow */ onLoadRangerConfigs: function (data) { var properties = [ - { - type: 'core-site', - name: 'hadoop.security.key.provider.path' - }, - { - type: 'hdfs-site', - name: 'dfs.encryption.key.provider.uri' - } - ]; - var hostToInstall = this.get('rangerKMSServerHost'); - var rkmsHosts = this.getRangerKMSServerHosts(); - var rkmsPort = data.items.findProperty('type', 'kms-env').properties['kms_port']; - var newValue = 'kms://http@' + rkmsHosts.join(';') + ':' + rkmsPort + '/kms'; - var coreSiteConfigs = data.items.findProperty('type', 'core-site'); - var hdfsSiteConfigs = data.items.findProperty('type', 'hdfs-site'); - var groups = [ - { - properties: { - 'core-site': coreSiteConfigs.properties, - 'hdfs-site': hdfsSiteConfigs.properties + { + type: 'core-site', + name: 'hadoop.security.key.provider.path' }, - properties_attributes: { - 'core-site': coreSiteConfigs.properties_attributes, - 'hdfs-site': hdfsSiteConfigs.properties_attributes + { + type: 'hdfs-site', + name: 'dfs.encryption.key.provider.uri' } - } - ]; - var propertiesToChange = this.get('allPropertiesToChange'); + ], + hostToInstall = this.get('rangerKMSServerHost'), + rkmsHosts = this.getRangerKMSServerHosts().join(';'), + rkmsPort = data.items.findProperty('type', 'kms-env').properties['kms_port'], + newValue = 'kms://http@' + rkmsHosts + ':' + rkmsPort + '/kms', + coreSiteConfigs = data.items.findProperty('type', 'core-site'), + hdfsSiteConfigs = data.items.findProperty('type', 'hdfs-site'), + groups = [ + { + properties: { + 'core-site': coreSiteConfigs.properties, + 'hdfs-site': hdfsSiteConfigs.properties + }, + properties_attributes: { + 'core-site': coreSiteConfigs.properties_attributes, + 'hdfs-site': hdfsSiteConfigs.properties_attributes + } + } + ], + propertiesToChange = this.get('allPropertiesToChange'); properties.forEach(function (property) { var typeConfigs = data.items.findProperty('type', property.type).properties, - currentValue = typeConfigs[property.name]; - if (this.get('isReconfigureRequired') && currentValue !== newValue) { - var service = App.config.get('serviceByConfigTypeMap')[property.type]; - propertiesToChange.pushObject({ - propertyFileName: property.type, - propertyName: property.name, - serviceDisplayName: service && service.get('displayName'), - initialValue: currentValue, - recommendedValue: newValue, - saveRecommended: true - }); + currentValue = typeConfigs[property.name], + pattern = new RegExp('^kms:\\/\\/http@(.+):' + rkmsPort + '\\/kms$'), + patternMatch = currentValue && currentValue.match(pattern), + currentHostsList = patternMatch && patternMatch[1].split(';').sort().join(';'); + if (currentHostsList !== rkmsHosts) { + typeConfigs[property.name] = newValue; + if (this.get('isReconfigureRequired')) { + var service = App.config.get('serviceByConfigTypeMap')[property.type]; + propertiesToChange.pushObject({ + propertyFileName: property.type, + propertyName: property.name, + serviceDisplayName: service && service.get('displayName'), + initialValue: currentValue, + recommendedValue: newValue, + saveRecommended: true + }); + } } - typeConfigs[property.name] = newValue; }, this); if (this.get('isReconfigureRequired')) { this.setConfigsChanges(groups); http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/utils/config.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/config.js b/ambari-web/app/utils/config.js index d0dbb69..8eeca5f 100644 --- a/ambari-web/app/utils/config.js +++ b/ambari-web/app/utils/config.js @@ -37,6 +37,17 @@ App.config = Em.Object.create({ preDefinedServiceConfigs: [], /** + * Map for methods used to parse hosts lists from certain config properties + */ + uniqueHostsListParsers: [ + { + propertyName: 'templeton.hive.properties', + type: 'webhcat-site', + method: 'getTempletonHiveHosts' + } + ], + + /** * * Returns file name version that stored on server. * @@ -1234,6 +1245,12 @@ App.config = Em.Object.create({ return false; }, + getTempletonHiveHosts: function (value) { + var pattern = /thrift:\/\/.+:\d+/, + patternMatch = value.match(pattern); + return patternMatch ? patternMatch[0].split('\\,') : value; + }, + /** * Update config property value based on its current value and list of zookeeper server hosts. * Used to prevent sort order issues. @@ -1251,14 +1268,31 @@ App.config = Em.Object.create({ * * @method updateHostsListValue * @param {Object} siteConfigs - prepared site config object to store + * @param {String} propertyType - type of the property to update * @param {String} propertyName - name of the property to update * @param {String} hostsList - list of ZooKeeper Server names to set as config property value + * @param {Boolean} isArray - determines whether value string is formatted as array * @return {String} - result value */ - updateHostsListValue: function(siteConfigs, propertyName, hostsList) { - var value = hostsList; - var propertyHosts = (siteConfigs[propertyName] || '').split(','); - var hostsToSet = hostsList.split(','); + updateHostsListValue: function(siteConfigs, propertyType, propertyName, hostsList, isArray) { + var value = hostsList, + propertyHosts = (siteConfigs[propertyName] || ''), + hostsToSet = hostsList, + parser = this.get('uniqueHostsListParsers').find(function (property) { + return property.type === propertyType && property.propertyName === propertyName; + }); + if (parser) { + propertyHosts = this.get(parser.method)(propertyHosts); + hostsToSet = this.get(parser.method)(hostsToSet); + } else { + if (isArray) { + var pattern = /(^\[|]$)/g; + propertyHosts = propertyHosts.replace(pattern, ''); + hostsToSet = hostsToSet.replace(pattern, ''); + } + propertyHosts = propertyHosts.split(','); + hostsToSet = hostsToSet.split(','); + } if (!Em.isEmpty(siteConfigs[propertyName])) { var diffLength = propertyHosts.filter(function(hostName) { http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/utils/configs/add_component_config_initializer.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/configs/add_component_config_initializer.js b/ambari-web/app/utils/configs/add_component_config_initializer.js index 7e3fedf..21fb6b4 100644 --- a/ambari-web/app/utils/configs/add_component_config_initializer.js +++ b/ambari-web/app/utils/configs/add_component_config_initializer.js @@ -146,7 +146,10 @@ App.AddComponentConfigInitializer = App.HaConfigInitializerClass.extend(App.Host */ updateSiteObj: function(siteConfigs, configProperty) { if (!siteConfigs || !configProperty) return false; - App.config.updateHostsListValue(siteConfigs, configProperty.name, configProperty.value); + var initializer = this.get('initializers')[configProperty.name], + isArray = !!(initializer && (initializer.type === 'json_stringified_value' + || Em.isArray(initializer) && initializer.someProperty('type', 'json_stringified_value'))); + App.config.updateHostsListValue(siteConfigs, configProperty.fileName, configProperty.name, configProperty.value, isArray); return true; }, http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js b/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js index d0d2503..086ad4a 100644 --- a/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js +++ b/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js @@ -371,7 +371,7 @@ App.HostsBasedInitializerMixin = Em.Mixin.create({ component: component, componentExists: componentExists, modifier: { - delmitier: Em.isNone(delimiter) ? ',' : delimiter + delimiter: Em.isNone(delimiter) ? ',' : delimiter } }; }, http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/test/controllers/main/host/details_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/controllers/main/host/details_test.js b/ambari-web/test/controllers/main/host/details_test.js index a4cb177..03dadc6 100644 --- a/ambari-web/test/controllers/main/host/details_test.js +++ b/ambari-web/test/controllers/main/host/details_test.js @@ -3247,7 +3247,7 @@ describe('App.MainHostDetailsController', function () { 'result': [ { properties: { - 'core-site': {'hadoop.security.key.provider.path': 'kms://http@host1;host2:port/kms'}, + 'core-site': {'hadoop.security.key.provider.path': 'kms://http@host2;host1:port/kms'}, 'hdfs-site': {'dfs.encryption.key.provider.uri': 'kms://http@host1;host2:port/kms'} }, properties_attributes: { @@ -3278,11 +3278,15 @@ describe('App.MainHostDetailsController', function () { }, { type: 'core-site', - properties: {} + properties: { + 'hadoop.security.key.provider.path': 'kms://http@host2;host1:port/kms' + } }, { type: 'hdfs-site', - properties: {} + properties: { + 'dfs.encryption.key.provider.uri': 'kms://http@host2:port/kms' + } } ] }; @@ -3290,7 +3294,7 @@ describe('App.MainHostDetailsController', function () { beforeEach(function () { controller.set('rangerKMSServerHost', item.hostToInstall); sinon.stub(controller, 'getRangerKMSServerHosts').returns(item.kmsHosts); - controller.onLoadRangerConfigs(data, null, {}); + controller.onLoadRangerConfigs(data); }); it('saveConfigsBatch is called with valid arguments', function () { http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/test/utils/config_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/utils/config_test.js b/ambari-web/test/utils/config_test.js index 8c450ed..7c2515a 100644 --- a/ambari-web/test/utils/config_test.js +++ b/ambari-web/test/utils/config_test.js @@ -1060,6 +1060,7 @@ describe('App.config', function() { 'hadoop.registry.zk.quorum': 'host1,host2' }, propertyName: 'hadoop.registry.zk.quorum', + propertyType: 'yarn-site', hostsList: 'host1', e: 'host1' }, @@ -1068,6 +1069,7 @@ describe('App.config', function() { 'hadoop.registry.zk.quorum': 'host1:10,host2:10' }, propertyName: 'hadoop.registry.zk.quorum', + propertyType: 'yarn-site', hostsList: 'host2:10,host1:10', e: 'host1:10,host2:10' }, @@ -1076,6 +1078,7 @@ describe('App.config', function() { 'hadoop.registry.zk.quorum': 'host1:10,host2:10,host3:10' }, propertyName: 'hadoop.registry.zk.quorum', + propertyType: 'yarn-site', hostsList: 'host2:10,host1:10', e: 'host2:10,host1:10' }, @@ -1084,6 +1087,7 @@ describe('App.config', function() { 'hadoop.registry.zk.quorum': 'host1:10,host2:10,host3:10' }, propertyName: 'hadoop.registry.zk.quorum', + propertyType: 'yarn-site', hostsList: 'host2:10,host1:10,host3:10,host4:11', e: 'host2:10,host1:10,host3:10,host4:11' }, @@ -1092,15 +1096,114 @@ describe('App.config', function() { 'hive.zookeeper.quorum': 'host1' }, propertyName: 'some.new.property', + propertyType: 'hive-site', hostsList: 'host2,host1:10', e: 'host2,host1:10' + }, + { + siteConfigs: { + 'some.new.property': '[\'host1\',\'host2\']' + }, + propertyName: 'some.new.property', + propertyType: 'property-type', + hostsList: '[\'host1\',\'host2\']', + isArray: true, + e: '[\'host1\',\'host2\']', + message: 'array-formatted property value with no changes' + }, + { + siteConfigs: { + 'some.new.property': '[\'host2\',\'host1\']' + }, + propertyName: 'some.new.property', + propertyType: 'property-type', + hostsList: '[\'host1\',\'host2\']', + isArray: true, + e: '[\'host2\',\'host1\']', + message: 'array-formatted property value with different hosts order' + }, + { + siteConfigs: { + 'some.new.property': '[\'host1\',\'host2\']' + }, + propertyName: 'some.new.property', + propertyType: 'property-type', + hostsList: '[\'host3\',\'host4\']', + isArray: true, + e: '[\'host3\',\'host4\']', + message: 'array-formatted property value with changes' + }, + { + siteConfigs: { + 'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true' + }, + propertyName: 'templeton.hive.properties', + propertyType: 'webhcat-site', + hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + message: 'templeton.hive.properties, no changes' + }, + { + siteConfigs: { + 'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host2:9083\\,thrift://host1:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true' + }, + propertyName: 'templeton.hive.properties', + propertyType: 'webhcat-site', + hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host2:9083\\,thrift://host1:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + message: 'templeton.hive.properties, different hosts order' + }, + { + siteConfigs: { + 'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9082\\,thrift://host2:9082,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true' + }, + propertyName: 'templeton.hive.properties', + propertyType: 'webhcat-site', + hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + message: 'templeton.hive.properties, different ports' + }, + { + siteConfigs: { + 'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true' + }, + propertyName: 'templeton.hive.properties', + propertyType: 'webhcat-site', + hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host3:9083\\,thrift://host4:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host3:9083\\,thrift://host4:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + message: 'templeton.hive.properties, different hosts' + }, + { + siteConfigs: { + 'templeton.hive.properties': 'hive.metastore.local=false' + }, + propertyName: 'templeton.hive.properties', + propertyType: 'hive-site', + hostsList: 'hive.metastore.local=true', + e: 'hive.metastore.local=true', + message: 'custom templeton.hive.properties' } ]; + tests.forEach(function(test) { - it('ZK located on {0}, current prop value is "{1}" "{2}" value should be "{3}"'.format(test.hostsList, ''+test.siteConfigs[test.propertyName], test.propertyName, test.e), function() { - var result = App.config.updateHostsListValue(test.siteConfigs, test.propertyName, test.hostsList); - expect(result).to.be.eql(test.e); - expect(test.siteConfigs[test.propertyName]).to.be.eql(test.e); + var message = test.message + || 'ZK located on {0}, current prop value is "{1}" "{2}" value should be "{3}"' + .format(test.hostsList, ''+test.siteConfigs[test.propertyName], test.propertyName, test.e); + + describe(message, function () { + var result; + + beforeEach(function () { + result = App.config.updateHostsListValue(test.siteConfigs, test.propertyType, test.propertyName, test.hostsList, test.isArray); + }); + + it('returned value', function() { + expect(result).to.be.eql(test.e); + }); + + it('value in configs object', function() { + expect(test.siteConfigs[test.propertyName]).to.be.eql(test.e); + }); }); }); }); @@ -1312,6 +1415,42 @@ describe('App.config', function() { }); }); + describe('#getTempletonHiveHosts', function () { + var testCases = [ + { + value: 'hive.metastore.local=false,hive.metastore.uris=thrift://host0:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + result: ['thrift://host0:9083'], + message: 'one host' + }, + { + value: 'hive.metastore.local=false,hive.metastore.uris=thrift://host0:9083\\,thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + result: ['thrift://host0:9083', 'thrift://host1:9083', 'thrift://host2:9083'], + message: 'several hosts' + }, + { + value: 'thrift://host0:9083\\,thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true', + result: ['thrift://host0:9083', 'thrift://host1:9083', 'thrift://host2:9083'], + message: 'no leading text' + }, + { + value: 'hive.metastore.local=false,hive.metastore.uris=thrift://host0:9083\\,thrift://host1:9083\\,thrift://host2:9083', + result: ['thrift://host0:9083', 'thrift://host1:9083', 'thrift://host2:9083'], + message: 'no trailing text' + }, + { + value: 'hive.metastore.local=false', + result: 'hive.metastore.local=false', + message: 'no hosts list' + } + ]; + + testCases.forEach(function (test) { + it(test.message, function () { + expect(App.config.getTempletonHiveHosts(test.value)).to.eql(test.result); + }); + }); + }); + describe('#isDirHeterogeneous', function () { it ('retruns true for dfs.datanode.data.dir', function () { expect(App.config.isDirHeterogeneous('dfs.datanode.data.dir')).to.be.true;