ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ababiic...@apache.org
Subject ambari git commit: AMBARI-19383 Configs changes after adding/deleting host component: value shouldn't be changed if it differs in host names order only. (ababiichuk)
Date Thu, 05 Jan 2017 12:26:43 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 6b198cc71 -> 1527bd15c


AMBARI-19383 Configs changes after adding/deleting host component: value shouldn't be changed
if it differs in host names order only. (ababiichuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1527bd15
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1527bd15
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1527bd15

Branch: refs/heads/trunk
Commit: 1527bd15c6226fbb8e690c64f68b593ac5088c2b
Parents: 6b198cc
Author: ababiichuk <ababiichuk@hortonworks.com>
Authored: Thu Jan 5 13:57:21 2017 +0200
Committer: ababiichuk <ababiichuk@hortonworks.com>
Committed: Thu Jan 5 14:34:46 2017 +0200

----------------------------------------------------------------------
 ambari-web/app/controllers/main/host/details.js |  94 ++++++------
 ambari-web/app/utils/config.js                  |  42 +++++-
 .../configs/add_component_config_initializer.js |   5 +-
 .../configs/hosts_based_initializer_mixin.js    |   2 +-
 .../test/controllers/main/host/details_test.js  |  12 +-
 ambari-web/test/utils/config_test.js            | 147 ++++++++++++++++++-
 6 files changed, 244 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/controllers/main/host/details.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/host/details.js b/ambari-web/app/controllers/main/host/details.js
index 4a7cce4..7df2c4e 100644
--- a/ambari-web/app/controllers/main/host/details.js
+++ b/ambari-web/app/controllers/main/host/details.js
@@ -923,10 +923,10 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow
             fileName: fileName,
             name: propertyName,
             value: currentValue
-          };
-        var configProperty = initializer.initialValue(propertyDef, hostComponentsTopology,
dependencies);
+          },
+          configProperty = initializer.initialValue(propertyDef, hostComponentsTopology,
dependencies);
         initializer.updateSiteObj(configs[fileName], configProperty);
-        if (this.get('isReconfigureRequired') && currentValue !== propertyDef.value)
{
+        if (this.get('isReconfigureRequired') && currentValue !== configs[fileName][propertyName])
{
           var service = App.config.get('serviceByConfigTypeMap')[fileName];
           propertiesToChange.pushObject({
             propertyFileName: fileName,
@@ -1101,9 +1101,10 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow
               fileName: fileName,
               name: propertyName,
               value: currentValue
-            };
-          configs[fileName][propertyName] = Em.get(initializer.initialValue(propertyDef,
localDB, dependencies), 'value');
-          if (this.get('isReconfigureRequired') && propertyDef.value !== currentValue)
{
+            },
+            configProperty = initializer.initialValue(propertyDef, localDB, dependencies);
+          initializer.updateSiteObj(configs[fileName], configProperty);
+          if (this.get('isReconfigureRequired') && currentValue !== configs[fileName][propertyName])
{
             var service = App.config.get('serviceByConfigTypeMap')[fileName];
             propertiesToChange.pushObject({
               propertyFileName: fileName,
@@ -1286,50 +1287,55 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow
    */
   onLoadRangerConfigs: function (data) {
     var properties = [
-      {
-        type: 'core-site',
-        name: 'hadoop.security.key.provider.path'
-      },
-      {
-        type: 'hdfs-site',
-        name: 'dfs.encryption.key.provider.uri'
-      }
-    ];
-    var hostToInstall = this.get('rangerKMSServerHost');
-    var rkmsHosts = this.getRangerKMSServerHosts();
-    var rkmsPort = data.items.findProperty('type', 'kms-env').properties['kms_port'];
-    var newValue = 'kms://http@' + rkmsHosts.join(';') + ':' + rkmsPort + '/kms';
-    var coreSiteConfigs = data.items.findProperty('type', 'core-site');
-    var hdfsSiteConfigs = data.items.findProperty('type', 'hdfs-site');
-    var groups = [
-      {
-        properties: {
-          'core-site': coreSiteConfigs.properties,
-          'hdfs-site': hdfsSiteConfigs.properties
+        {
+          type: 'core-site',
+          name: 'hadoop.security.key.provider.path'
         },
-        properties_attributes: {
-          'core-site': coreSiteConfigs.properties_attributes,
-          'hdfs-site': hdfsSiteConfigs.properties_attributes
+        {
+          type: 'hdfs-site',
+          name: 'dfs.encryption.key.provider.uri'
         }
-      }
-    ];
-    var propertiesToChange = this.get('allPropertiesToChange');
+      ],
+      hostToInstall = this.get('rangerKMSServerHost'),
+      rkmsHosts = this.getRangerKMSServerHosts().join(';'),
+      rkmsPort = data.items.findProperty('type', 'kms-env').properties['kms_port'],
+      newValue = 'kms://http@' + rkmsHosts + ':' + rkmsPort + '/kms',
+      coreSiteConfigs = data.items.findProperty('type', 'core-site'),
+      hdfsSiteConfigs = data.items.findProperty('type', 'hdfs-site'),
+      groups = [
+        {
+          properties: {
+            'core-site': coreSiteConfigs.properties,
+            'hdfs-site': hdfsSiteConfigs.properties
+          },
+          properties_attributes: {
+            'core-site': coreSiteConfigs.properties_attributes,
+            'hdfs-site': hdfsSiteConfigs.properties_attributes
+          }
+        }
+      ],
+      propertiesToChange = this.get('allPropertiesToChange');
 
     properties.forEach(function (property) {
       var typeConfigs = data.items.findProperty('type', property.type).properties,
-        currentValue = typeConfigs[property.name];
-      if (this.get('isReconfigureRequired') && currentValue !== newValue) {
-        var service = App.config.get('serviceByConfigTypeMap')[property.type];
-        propertiesToChange.pushObject({
-          propertyFileName: property.type,
-          propertyName: property.name,
-          serviceDisplayName: service && service.get('displayName'),
-          initialValue: currentValue,
-          recommendedValue: newValue,
-          saveRecommended: true
-        });
+        currentValue = typeConfigs[property.name],
+        pattern = new RegExp('^kms:\\/\\/http@(.+):' + rkmsPort + '\\/kms$'),
+        patternMatch = currentValue && currentValue.match(pattern),
+        currentHostsList = patternMatch && patternMatch[1].split(';').sort().join(';');
+      if (currentHostsList !== rkmsHosts) {
+        typeConfigs[property.name] = newValue;
+        if (this.get('isReconfigureRequired')) {
+          var service = App.config.get('serviceByConfigTypeMap')[property.type];
+          propertiesToChange.pushObject({
+            propertyFileName: property.type,
+            propertyName: property.name,
+            serviceDisplayName: service && service.get('displayName'),
+            initialValue: currentValue,
+            recommendedValue: newValue,
+            saveRecommended: true
+          });
+        }
       }
-      typeConfigs[property.name] = newValue;
     }, this);
     if (this.get('isReconfigureRequired')) {
       this.setConfigsChanges(groups);

http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/utils/config.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/config.js b/ambari-web/app/utils/config.js
index d0dbb69..8eeca5f 100644
--- a/ambari-web/app/utils/config.js
+++ b/ambari-web/app/utils/config.js
@@ -37,6 +37,17 @@ App.config = Em.Object.create({
   preDefinedServiceConfigs: [],
 
   /**
+   * Map for methods used to parse hosts lists from certain config properties
+   */
+  uniqueHostsListParsers: [
+    {
+      propertyName: 'templeton.hive.properties',
+      type: 'webhcat-site',
+      method: 'getTempletonHiveHosts'
+    }
+  ],
+
+  /**
    *
    * Returns file name version that stored on server.
    *
@@ -1234,6 +1245,12 @@ App.config = Em.Object.create({
     return false;
   },
 
+  getTempletonHiveHosts: function (value) {
+    var pattern = /thrift:\/\/.+:\d+/,
+      patternMatch = value.match(pattern);
+    return patternMatch ? patternMatch[0].split('\\,') : value;
+  },
+
   /**
    * Update config property value based on its current value and list of zookeeper server
hosts.
    * Used to prevent sort order issues.
@@ -1251,14 +1268,31 @@ App.config = Em.Object.create({
    *
    * @method updateHostsListValue
    * @param {Object} siteConfigs - prepared site config object to store
+   * @param {String} propertyType - type of the property to update
    * @param {String} propertyName - name of the property to update
    * @param {String} hostsList - list of ZooKeeper Server names to set as config property
value
+   * @param {Boolean} isArray - determines whether value string is formatted as array
    * @return {String} - result value
    */
-  updateHostsListValue: function(siteConfigs, propertyName, hostsList) {
-    var value = hostsList;
-    var propertyHosts = (siteConfigs[propertyName] || '').split(',');
-    var hostsToSet = hostsList.split(',');
+  updateHostsListValue: function(siteConfigs, propertyType, propertyName, hostsList, isArray)
{
+    var value = hostsList,
+      propertyHosts = (siteConfigs[propertyName] || ''),
+      hostsToSet = hostsList,
+      parser = this.get('uniqueHostsListParsers').find(function (property) {
+        return property.type === propertyType && property.propertyName === propertyName;
+      });
+    if (parser) {
+      propertyHosts = this.get(parser.method)(propertyHosts);
+      hostsToSet = this.get(parser.method)(hostsToSet);
+    } else {
+      if (isArray) {
+        var pattern = /(^\[|]$)/g;
+        propertyHosts = propertyHosts.replace(pattern, '');
+        hostsToSet = hostsToSet.replace(pattern, '');
+      }
+      propertyHosts = propertyHosts.split(',');
+      hostsToSet = hostsToSet.split(',');
+    }
 
     if (!Em.isEmpty(siteConfigs[propertyName])) {
       var diffLength = propertyHosts.filter(function(hostName) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/utils/configs/add_component_config_initializer.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/add_component_config_initializer.js b/ambari-web/app/utils/configs/add_component_config_initializer.js
index 7e3fedf..21fb6b4 100644
--- a/ambari-web/app/utils/configs/add_component_config_initializer.js
+++ b/ambari-web/app/utils/configs/add_component_config_initializer.js
@@ -146,7 +146,10 @@ App.AddComponentConfigInitializer = App.HaConfigInitializerClass.extend(App.Host
    */
   updateSiteObj: function(siteConfigs, configProperty) {
     if (!siteConfigs || !configProperty) return false;
-    App.config.updateHostsListValue(siteConfigs, configProperty.name, configProperty.value);
+    var initializer = this.get('initializers')[configProperty.name],
+      isArray = !!(initializer && (initializer.type === 'json_stringified_value'
+        || Em.isArray(initializer) && initializer.someProperty('type', 'json_stringified_value')));
+    App.config.updateHostsListValue(siteConfigs, configProperty.fileName, configProperty.name,
configProperty.value, isArray);
     return true;
   },
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js b/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js
index d0d2503..086ad4a 100644
--- a/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js
+++ b/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js
@@ -371,7 +371,7 @@ App.HostsBasedInitializerMixin = Em.Mixin.create({
       component: component,
       componentExists: componentExists,
       modifier: {
-        delmitier: Em.isNone(delimiter) ? ',' : delimiter
+        delimiter: Em.isNone(delimiter) ? ',' : delimiter
       }
     };
   },

http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/test/controllers/main/host/details_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/main/host/details_test.js b/ambari-web/test/controllers/main/host/details_test.js
index a4cb177..03dadc6 100644
--- a/ambari-web/test/controllers/main/host/details_test.js
+++ b/ambari-web/test/controllers/main/host/details_test.js
@@ -3247,7 +3247,7 @@ describe('App.MainHostDetailsController', function () {
         'result': [
           {
             properties: {
-              'core-site': {'hadoop.security.key.provider.path': 'kms://http@host1;host2:port/kms'},
+              'core-site': {'hadoop.security.key.provider.path': 'kms://http@host2;host1:port/kms'},
               'hdfs-site': {'dfs.encryption.key.provider.uri': 'kms://http@host1;host2:port/kms'}
             },
             properties_attributes: {
@@ -3278,11 +3278,15 @@ describe('App.MainHostDetailsController', function () {
             },
             {
               type: 'core-site',
-              properties: {}
+              properties: {
+                'hadoop.security.key.provider.path': 'kms://http@host2;host1:port/kms'
+              }
             },
             {
               type: 'hdfs-site',
-              properties: {}
+              properties: {
+                'dfs.encryption.key.provider.uri': 'kms://http@host2:port/kms'
+              }
             }
           ]
         };
@@ -3290,7 +3294,7 @@ describe('App.MainHostDetailsController', function () {
         beforeEach(function () {
           controller.set('rangerKMSServerHost', item.hostToInstall);
           sinon.stub(controller, 'getRangerKMSServerHosts').returns(item.kmsHosts);
-          controller.onLoadRangerConfigs(data, null, {});
+          controller.onLoadRangerConfigs(data);
         });
 
         it('saveConfigsBatch is called with valid arguments', function () {

http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/test/utils/config_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/utils/config_test.js b/ambari-web/test/utils/config_test.js
index 8c450ed..7c2515a 100644
--- a/ambari-web/test/utils/config_test.js
+++ b/ambari-web/test/utils/config_test.js
@@ -1060,6 +1060,7 @@ describe('App.config', function() {
           'hadoop.registry.zk.quorum': 'host1,host2'
         },
         propertyName: 'hadoop.registry.zk.quorum',
+        propertyType: 'yarn-site',
         hostsList: 'host1',
         e: 'host1'
       },
@@ -1068,6 +1069,7 @@ describe('App.config', function() {
           'hadoop.registry.zk.quorum': 'host1:10,host2:10'
         },
         propertyName: 'hadoop.registry.zk.quorum',
+        propertyType: 'yarn-site',
         hostsList: 'host2:10,host1:10',
         e: 'host1:10,host2:10'
       },
@@ -1076,6 +1078,7 @@ describe('App.config', function() {
           'hadoop.registry.zk.quorum': 'host1:10,host2:10,host3:10'
         },
         propertyName: 'hadoop.registry.zk.quorum',
+        propertyType: 'yarn-site',
         hostsList: 'host2:10,host1:10',
         e: 'host2:10,host1:10'
       },
@@ -1084,6 +1087,7 @@ describe('App.config', function() {
           'hadoop.registry.zk.quorum': 'host1:10,host2:10,host3:10'
         },
         propertyName: 'hadoop.registry.zk.quorum',
+        propertyType: 'yarn-site',
         hostsList: 'host2:10,host1:10,host3:10,host4:11',
         e: 'host2:10,host1:10,host3:10,host4:11'
       },
@@ -1092,15 +1096,114 @@ describe('App.config', function() {
           'hive.zookeeper.quorum': 'host1'
         },
         propertyName: 'some.new.property',
+        propertyType: 'hive-site',
         hostsList: 'host2,host1:10',
         e: 'host2,host1:10'
+      },
+      {
+        siteConfigs: {
+          'some.new.property': '[\'host1\',\'host2\']'
+        },
+        propertyName: 'some.new.property',
+        propertyType: 'property-type',
+        hostsList: '[\'host1\',\'host2\']',
+        isArray: true,
+        e: '[\'host1\',\'host2\']',
+        message: 'array-formatted property value with no changes'
+      },
+      {
+        siteConfigs: {
+          'some.new.property': '[\'host2\',\'host1\']'
+        },
+        propertyName: 'some.new.property',
+        propertyType: 'property-type',
+        hostsList: '[\'host1\',\'host2\']',
+        isArray: true,
+        e: '[\'host2\',\'host1\']',
+        message: 'array-formatted property value with different hosts order'
+      },
+      {
+        siteConfigs: {
+          'some.new.property': '[\'host1\',\'host2\']'
+        },
+        propertyName: 'some.new.property',
+        propertyType: 'property-type',
+        hostsList: '[\'host3\',\'host4\']',
+        isArray: true,
+        e: '[\'host3\',\'host4\']',
+        message: 'array-formatted property value with changes'
+      },
+      {
+        siteConfigs: {
+          'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true'
+        },
+        propertyName: 'templeton.hive.properties',
+        propertyType: 'webhcat-site',
+        hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        message: 'templeton.hive.properties, no changes'
+      },
+      {
+        siteConfigs: {
+          'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host2:9083\\,thrift://host1:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true'
+        },
+        propertyName: 'templeton.hive.properties',
+        propertyType: 'webhcat-site',
+        hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host2:9083\\,thrift://host1:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        message: 'templeton.hive.properties, different hosts order'
+      },
+      {
+        siteConfigs: {
+          'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9082\\,thrift://host2:9082,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true'
+        },
+        propertyName: 'templeton.hive.properties',
+        propertyType: 'webhcat-site',
+        hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        message: 'templeton.hive.properties, different ports'
+      },
+      {
+        siteConfigs: {
+          'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true'
+        },
+        propertyName: 'templeton.hive.properties',
+        propertyType: 'webhcat-site',
+        hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host3:9083\\,thrift://host4:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host3:9083\\,thrift://host4:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        message: 'templeton.hive.properties, different hosts'
+      },
+      {
+        siteConfigs: {
+          'templeton.hive.properties': 'hive.metastore.local=false'
+        },
+        propertyName: 'templeton.hive.properties',
+        propertyType: 'hive-site',
+        hostsList: 'hive.metastore.local=true',
+        e: 'hive.metastore.local=true',
+        message: 'custom templeton.hive.properties'
       }
     ];
+
     tests.forEach(function(test) {
-      it('ZK located on {0}, current prop value is "{1}" "{2}" value should be "{3}"'.format(test.hostsList,
''+test.siteConfigs[test.propertyName], test.propertyName, test.e), function() {
-        var result = App.config.updateHostsListValue(test.siteConfigs, test.propertyName,
test.hostsList);
-        expect(result).to.be.eql(test.e);
-        expect(test.siteConfigs[test.propertyName]).to.be.eql(test.e);
+      var message = test.message
+        || 'ZK located on {0}, current prop value is "{1}" "{2}" value should be "{3}"'
+          .format(test.hostsList, ''+test.siteConfigs[test.propertyName], test.propertyName,
test.e);
+
+      describe(message, function () {
+        var result;
+
+        beforeEach(function () {
+          result = App.config.updateHostsListValue(test.siteConfigs, test.propertyType, test.propertyName,
test.hostsList, test.isArray);
+        });
+
+        it('returned value', function() {
+          expect(result).to.be.eql(test.e);
+        });
+
+        it('value in configs object', function() {
+          expect(test.siteConfigs[test.propertyName]).to.be.eql(test.e);
+        });
       });
     });
   });
@@ -1312,6 +1415,42 @@ describe('App.config', function() {
     });
   });
 
+  describe('#getTempletonHiveHosts', function () {
+    var testCases = [
+      {
+        value: 'hive.metastore.local=false,hive.metastore.uris=thrift://host0:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        result: ['thrift://host0:9083'],
+        message: 'one host'
+      },
+      {
+        value: 'hive.metastore.local=false,hive.metastore.uris=thrift://host0:9083\\,thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        result: ['thrift://host0:9083', 'thrift://host1:9083', 'thrift://host2:9083'],
+        message: 'several hosts'
+      },
+      {
+        value: 'thrift://host0:9083\\,thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+        result: ['thrift://host0:9083', 'thrift://host1:9083', 'thrift://host2:9083'],
+        message: 'no leading text'
+      },
+      {
+        value: 'hive.metastore.local=false,hive.metastore.uris=thrift://host0:9083\\,thrift://host1:9083\\,thrift://host2:9083',
+        result: ['thrift://host0:9083', 'thrift://host1:9083', 'thrift://host2:9083'],
+        message: 'no trailing text'
+      },
+      {
+        value: 'hive.metastore.local=false',
+        result: 'hive.metastore.local=false',
+        message: 'no hosts list'
+      }
+    ];
+
+    testCases.forEach(function (test) {
+      it(test.message, function () {
+        expect(App.config.getTempletonHiveHosts(test.value)).to.eql(test.result);
+      });
+    });
+  });
+
   describe('#isDirHeterogeneous', function () {
     it ('retruns true for dfs.datanode.data.dir', function () {
       expect(App.config.isDirHeterogeneous('dfs.datanode.data.dir')).to.be.true;


Mime
View raw message