ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srima...@apache.org
Subject git commit: AMBARI-4943. Verify Tez configurations are according to recommended settings. (srimanth)
Date Tue, 04 Mar 2014 19:27:08 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 20fa2346b -> 01e59101c


AMBARI-4943. Verify Tez configurations are according to recommended settings. (srimanth)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/01e59101
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/01e59101
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/01e59101

Branch: refs/heads/trunk
Commit: 01e59101c2a70bc0eca94a5a5b2dfd1e31fa359d
Parents: 20fa234
Author: Srimanth Gunturi <sgunturi@hortonworks.com>
Authored: Tue Mar 4 11:16:07 2014 -0800
Committer: Srimanth Gunturi <sgunturi@hortonworks.com>
Committed: Tue Mar 4 11:26:56 2014 -0800

----------------------------------------------------------------------
 .../services/HIVE/configuration/hive-site.xml   |  2 +-
 .../services/TEZ/configuration/tez-site.xml     |  2 +-
 ambari-web/app/data/service_configs.js          | 10 +++--
 .../storm_defaults_provider.js                  |  2 +-
 .../defaults_providers/tez_defaults_provider.js | 40 ++++++++++++++++++++
 .../yarn_defaults_provider.js                   |  2 +-
 .../validators/service_configs_validator.js     |  2 +-
 .../configs/validators/tez_configs_validator.js | 38 +++++++++++++++++++
 8 files changed, 90 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
index f0be795..88c8e88 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
@@ -91,7 +91,7 @@ limitations under the License.
   <property>
     <name>hive.semantic.analyzer.factory.impl</name>
     <value>org.apache.hivealog.cli.HCatSemanticAnalyzerFactory</value>
-    <description>controls which SemanticAnalyzerFactory implemenation class is used
by CLI</description>
+    <description>controls which SemanticAnalyzerFactory implementation class is used
by CLI</description>
   </property>
 
   <property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
index 333a8ac..5a750bf 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
@@ -47,7 +47,7 @@
   <property>
     <name>tez.am.java.opts</name>
     <value>-server -Xmx1024m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC</value>
-    <description>Java options for the Tez AppMaster process</description>
+    <description>Java options for the Tez AppMaster process. The -Xmx parameter value
is generally 0.8 times tez.am.resource.memory.mb config.</description>
   </property>
 
   <property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/data/service_configs.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/service_configs.js b/ambari-web/app/data/service_configs.js
index b3266fb..0473134 100644
--- a/ambari-web/app/data/service_configs.js
+++ b/ambari-web/app/data/service_configs.js
@@ -19,8 +19,10 @@
 var App = require('app');
 require('models/service_config');
 require('utils/configs/defaults_providers/yarn_defaults_provider');
+require('utils/configs/defaults_providers/tez_defaults_provider');
 require('utils/configs/defaults_providers/storm_defaults_provider');
 require('utils/configs/validators/yarn_configs_validator');
+require('utils/configs/validators/tez_configs_validator');
 require('utils/configs/validators/mapreduce2_configs_validator');
 require('utils/configs/validators/storm_configs_validator');
 
@@ -74,7 +76,7 @@ module.exports = [
     serviceName: 'YARN',
     displayName: 'YARN',
     configsValidator: App.YARNConfigsValidator,
-    defaultsProviders: [App.YARNDefaultsProvider],
+    defaultsProviders: [App.YARNDefaultsProvider.create()],
     filename: 'yarn-site',
     configCategories: [
       App.ServiceConfigCategory.create({ name: 'ResourceManager', displayName : 'Resource
Manager', hostComponentNames : ['RESOURCEMANAGER']}),
@@ -95,7 +97,7 @@ module.exports = [
     displayName: 'MapReduce 2',
     filename: 'mapred-site',
     configsValidator: App.MapReduce2ConfigsValidator,
-    defaultsProviders: [App.YARNDefaultsProvider],
+    defaultsProviders: [App.YARNDefaultsProvider.create()],
     configCategories: [
       App.ServiceConfigCategory.create({ name: 'HistoryServer', displayName : 'History Server',
hostComponentNames : ['HISTORYSERVER']}),
       App.ServiceConfigCategory.create({ name: 'General', displayName : 'General'}),
@@ -240,7 +242,7 @@ module.exports = [
     serviceName: 'STORM',
     displayName: 'Storm',
     configsValidator: App.STORMConfigsValidator,
-    defaultsProviders: [App.STORMDefaultsProvider],
+    defaultsProviders: [App.STORMDefaultsProvider.create()],
     filename: 'storm-site',
     configCategories: [
       App.ServiceConfigCategory.create({ name: 'Nimbus', displayName : 'Nimbus'}),
@@ -260,6 +262,8 @@ module.exports = [
     serviceName: 'TEZ',
     displayName: 'Tez',
     filename: 'tez-site',
+    configsValidator: App.TezConfigsValidator,
+    defaultsProviders: [App.TezDefaultsProvider.create()],
     configCategories: [
       App.ServiceConfigCategory.create({ name: 'General', displayName : 'General'}),
       App.ServiceConfigCategory.create({ name: 'Advanced', displayName : 'Advanced'}),

http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js b/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js
index cdd8ab9..ce8704d 100644
--- a/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js
+++ b/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js
@@ -18,7 +18,7 @@
 var App = require('app');
 require('utils/configs/defaults_providers/defaultsProvider');
 
-App.STORMDefaultsProvider = App.DefaultsProvider.create({
+App.STORMDefaultsProvider = App.DefaultsProvider.extend({
 
   clusterData: null,
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/defaults_providers/tez_defaults_provider.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/defaults_providers/tez_defaults_provider.js b/ambari-web/app/utils/configs/defaults_providers/tez_defaults_provider.js
new file mode 100644
index 0000000..18a3109
--- /dev/null
+++ b/ambari-web/app/utils/configs/defaults_providers/tez_defaults_provider.js
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/defaults_providers/yarn_defaults_provider');
+
+App.TezDefaultsProvider = App.YARNDefaultsProvider.extend({
+
+  configsTemplate: {
+    'tez.am.resource.memory.mb': null,
+    'tez.am.java.opts': null
+  },
+
+  getDefaults : function(localDB) {
+    var configs = this._super(localDB);
+    if (configs['yarn.app.mapreduce.am.resource.mb'] != null) {
+      configs['tez.am.resource.memory.mb'] = configs['yarn.app.mapreduce.am.resource.mb'];
+      configs['tez.am.java.opts'] = '-server -Xmx' + Math.round(0.8 * configs['tez.am.resource.memory.mb'])
+          + 'm -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC';
+    } else {
+      jQuery.extend(configs, this.get('configsTemplate'));
+    }
+    return configs;
+  }
+
+});

http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js b/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js
index 0e6e312..da395e0 100644
--- a/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js
+++ b/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js
@@ -18,7 +18,7 @@
 var App = require('app');
 require('utils/configs/defaults_providers/defaultsProvider');
 
-App.YARNDefaultsProvider = App.DefaultsProvider.create({
+App.YARNDefaultsProvider = App.DefaultsProvider.extend({
 
   /**
    * List of the configs that should be calculated

http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/validators/service_configs_validator.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/validators/service_configs_validator.js b/ambari-web/app/utils/configs/validators/service_configs_validator.js
index d4dbd34..5507641 100644
--- a/ambari-web/app/utils/configs/validators/service_configs_validator.js
+++ b/ambari-web/app/utils/configs/validators/service_configs_validator.js
@@ -93,7 +93,7 @@ App.ServiceConfigsValidator = Em.Object.extend({
    * @private
    */
   _checkXmxValueFormat: function(value) {
-    var regex = /^\-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)?(\s+)?(\s.+)?$/;
+    var regex = /(^|\s)\-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)(\s|$)/;
     if (!regex.test(value)) {
       return false;
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/validators/tez_configs_validator.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/validators/tez_configs_validator.js b/ambari-web/app/utils/configs/validators/tez_configs_validator.js
new file mode 100644
index 0000000..14042fb
--- /dev/null
+++ b/ambari-web/app/utils/configs/validators/tez_configs_validator.js
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/validators/service_configs_validator');
+
+App.TezConfigsValidator = App.ServiceConfigsValidator.create({
+  /**
+   * List of the configs that should be validated
+   */
+  configValidators: {
+    'tez.am.resource.memory.mb': 'tezAMResourceMb',
+    'tez.am.java.opts': 'tezAMJavaOpts',
+  },
+
+  tezAMResourceMb: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  },
+
+  tezAMJavaOpts: function(config) {
+    return this.validateXmxValue(config);
+  }
+
+});


Mime
View raw message