Return-Path: X-Original-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DD63A108BB for ; Wed, 13 Nov 2013 18:54:59 +0000 (UTC) Received: (qmail 1512 invoked by uid 500); 13 Nov 2013 18:54:59 -0000 Delivered-To: apmail-incubator-ambari-commits-archive@incubator.apache.org Received: (qmail 1443 invoked by uid 500); 13 Nov 2013 18:54:59 -0000 Mailing-List: contact ambari-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@incubator.apache.org Delivered-To: mailing list ambari-commits@incubator.apache.org Received: (qmail 1172 invoked by uid 99); 13 Nov 2013 18:54:59 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 13 Nov 2013 18:54:59 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 294EC8A83EE; Wed, 13 Nov 2013 18:54:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: srimanth@apache.org To: ambari-commits@incubator.apache.org Date: Wed, 13 Nov 2013 18:55:06 -0000 Message-Id: <41394c12c94241a690796b3d5d799f2b@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [10/20] git commit: AMBARI-3675: Default value of 'Default virtual memory for a job's map-task' is not valid. [Merged] (jaimin) AMBARI-3675: Default value of 'Default virtual memory for a job's map-task' is not valid. [Merged] (jaimin) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/2204615e Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/2204615e Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/2204615e Branch: refs/heads/branch-1.4-bk Commit: 2204615e7632e0fd8e234db2fbbe2ed79ad98dfd Parents: e7c2b07 Author: Jaimin Jetly Authored: Mon Nov 4 01:19:01 2013 -0800 Committer: Srimanth Gunturi Committed: Tue Nov 5 12:52:17 2013 -0800 ---------------------------------------------------------------------- .../app/controllers/wizard/step7_controller.js | 2 +- ambari-web/app/utils/config.js | 7 +- .../yarn_defaults_provider.js | 192 ++++++++++--------- ambari-web/app/utils/helper.js | 3 + ambari-web/app/utils/string_utils.js | 2 +- 5 files changed, 114 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/2204615e/ambari-web/app/controllers/wizard/step7_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/wizard/step7_controller.js b/ambari-web/app/controllers/wizard/step7_controller.js index b46e6eb..5ac149b 100644 --- a/ambari-web/app/controllers/wizard/step7_controller.js +++ b/ambari-web/app/controllers/wizard/step7_controller.js @@ -93,7 +93,7 @@ App.WizardStep7Controller = Em.Controller.extend({ configs = App.config.fileConfigsIntoTextarea(configs, 'capacity-scheduler.xml'); } //STEP 6: Distribute configs by service and wrap each one in App.ServiceConfigProperty (configs -> serviceConfigs) - var serviceConfigs = App.config.renderConfigs(configs, storedConfigs, this.get('allInstalledServiceNames'), this.get('selectedServiceNames')); + var serviceConfigs = App.config.renderConfigs(configs, storedConfigs, this.get('allInstalledServiceNames'), this.get('selectedServiceNames'), localDB, storedConfigs); this.set('stepConfigs', serviceConfigs); this.activateSpecialConfigs(); this.set('selectedService', this.get('stepConfigs').filterProperty('showConfig', true).objectAt(0)); http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/2204615e/ambari-web/app/utils/config.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/config.js b/ambari-web/app/utils/config.js index 6897de1..288665d 100644 --- a/ambari-web/app/utils/config.js +++ b/ambari-web/app/utils/config.js @@ -365,6 +365,7 @@ App.config = Em.Object.create({ configData.filename = stored.filename; configData.description = stored.description; } else if (!preDefined && stored) { + configData = { id: stored.id, name: stored.name, @@ -465,9 +466,10 @@ App.config = Em.Object.create({ * @param allInstalledServiceNames * @param selectedServiceNames * @param localDB + * @param storedConfigs * @return {Array} */ - renderConfigs: function (configs, storedConfigs, allInstalledServiceNames, selectedServiceNames) { + renderConfigs: function (configs, storedConfigs, allInstalledServiceNames, selectedServiceNames, localDB, storedConfigs) { var renderedServiceConfigs = []; var localDB = { hosts: App.db.getHosts(), @@ -505,7 +507,8 @@ App.config = Em.Object.create({ serviceConfig.set('showConfig', service.showConfig); // Use calculated default values for some configs - if (service.defaultsProviders) { + + if (!storedConfigs && service.defaultsProviders) { service.defaultsProviders.forEach(function(defaultsProvider) { var defaults = defaultsProvider.getDefaults(localDB); for(var name in defaults) { http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/2204615e/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js b/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js index 0ae3e70..80732cb 100644 --- a/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js +++ b/ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js @@ -31,6 +31,7 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({ 'mapreduce.reduce.memory.mb': null, 'mapreduce.map.java.opts': null, 'mapreduce.reduce.java.opts': null, + 'mapreduce.task.io.sort.mb': null, 'yarn.app.mapreduce.am.resource.mb': null, 'yarn.app.mapreduce.am.command-opts': null }, @@ -61,110 +62,118 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({ */ hBaseRam: null, + GB: 1024, /** * Minimum container size (in RAM). * This value is dependent on the amount of RAM available, as in smaller memory nodes the minimum container size should also be smaller * * Value in MB! */ - recommendedMinimumContainerSize: function() { + recommendedMinimumContainerSize: function () { if (!this.clusterDataIsValid()) return null; var ram = this.get('clusterData.ram'); - switch(true) { - case (ram < 4*1024): return 256; - case (ram >= 4*1024 && ram < 8*1024): return 512; - case (ram >= 8*1024 && ram < 24*1024): return 1024; - case (ram >= 24*1024): - default: return 2048; + switch (true) { + case (ram <=4 ): + return 256; + case (ram <= 8): + return 512; + case (ram <= 24): + return 1024; + default: + return 2048; } }.property('clusterData.ram'), /** * Maximum number of containers allowed per node + * max (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))) * min (2*CORES, 1.8*DISKS, (Total available RAM) / MIN_CONTAINER_SIZE) */ - containers: function() { + containers: function () { if (!this.clusterDataIsValid()) return null; var cpu = this.get('clusterData.cpu'); var disk = this.get('clusterData.disk'); var ram = this.get('clusterData.ram'); var containerSize = this.get('recommendedMinimumContainerSize'); cpu *= 2; - disk *= 1.8; + disk = Math.ceil(disk * 1.8); ram = (ram - this.get('reservedRam')); if (this.get('clusterData.hBaseInstalled')) { ram -= this.get('hBaseRam') } + ram *= this.get('GB'); ram /= containerSize; - if (cpu < disk) { - if (cpu < ram) { - return cpu; - } - return ram; - } - else { - if (disk < ram) { - return parseInt(disk); - } - return ram; - } - }.property('clusterData.cpu', 'clusterData.ram', 'clusterData.disk', 'recommendedMinimumContainerSize'), + return Math.round(Math.max(cpu, Math.min(disk, ram))); + }.property('clusterData.cpu', 'clusterData.ram', 'clusterData.hBaseInstalled', 'clusterData.disk', 'reservedRam', 'hBaseRam', 'recommendedMinimumContainerSize'), /** * amount of RAM per container - * RAM-per-container = max(MIN_CONTAINER_SIZE, (Total Available RAM) / containers)) + * RAM-per-container = abs(MIN_CONTAINER_SIZE, (Total Available RAM) / containers)) * * Value in MB! */ - ramPerContainer: function() { - var containerSize = this.get('recommendedMinimumContainerSize'); + ramPerContainer: function () { var containers = this.get('containers'); - if (!containerSize || !containers) { - return null; - } - var s = this.get('clusterData.ram') - this.get('reservedRam'); + var ram = this.get('clusterData.ram'); + ram = (ram - this.get('reservedRam')); if (this.get('clusterData.hBaseInstalled')) { - s -= this.get('hBaseRam'); + ram -= this.get('hBaseRam') } - s /= containers; - return (containerSize > s) ? containerSize : s; - }.property('recommendedMinimumContainerSize', 'containers'), + ram *= this.get('GB'); + var container_ram = Math.abs(ram / containers); + return container_ram > this.get('GB') ? container_ram / (512 * 512) : container_ram; + }.property('recommendedMinimumContainerSize', 'containers', 'clusterData.ram', 'clusterData.hBaseInstalled', 'hBaseRam', 'reservedRam'), + + mapMemory: function () { + return this.get('ramPerContainer'); + }.property('ramPerContainer'), + + reduceMemory: function () { + var ramPerContainer = this.get('ramPerContainer'); + return ramPerContainer <= 2048 ? 2 * ramPerContainer : ramPerContainer; + }.property('ramPerContainer'), + + amMemeory: function () { + return Math.max(this.get('mapMemory'), this.get('reduceMemory')); + }.property('mapMemory', 'reduceMemory'), /** * Reserved for HBase and system memory is based on total available memory */ - reservedMemoryRecommendations: function() { - var table = [ - {os:1,hbase:1}, - {os:2,hbase:1}, - {os:2,hbase:2}, - {os:4,hbase:4}, - {os:6,hbase:8}, - {os:8,hbase:8}, - {os:8,hbase:8}, - {os:12,hbase:16}, - {os:24,hbase:24}, - {os:32,hbase:32}, - {os:64,hbase:64} - ]; - var ram = this.get('clusterData.ram') / 1024; - var index = 0; - switch (true) { - case (ram <= 4): index = 0; break; - case (ram > 4 && ram <= 8): index = 1; break; - case (ram > 8 && ram <= 16): index = 2; break; - case (ram > 16 && ram <= 24): index = 3; break; - case (ram > 24 && ram <= 48): index = 4; break; - case (ram > 48 && ram <= 64): index = 5; break; - case (ram > 64 && ram <= 72): index = 6; break; - case (ram > 72 && ram <= 96): index = 7; break; - case (ram > 96 && ram <= 128): index = 8; break; - case (ram > 128 && ram <= 256): index = 9; break; - case (ram > 256 && ram <= 512): index = 10; break; - default: index = 10; break; + + + + reservedStackRecommendations: function () { + var memory = this.get('clusterData.ram'); + var reservedStack = { 4: 1, 8: 2, 16: 2, 24: 4, 48: 6, 64: 8, 72: 8, 96: 12, + 128: 24, 256: 32, 512: 64}; + + if (memory in reservedStack) { + this.set('reservedRam', reservedStack[memory]); } - this.set('reservedRam', table[index].os * 1024); - this.set('hBaseRam', table[index].hbase * 1024); + if (memory <= 4) + this.set('reservedRam', 1); + else if (memory >= 512) + this.set('reservedRam', 64); + else + this.set('reservedRam', 1); + }.observes('clusterData.ram'), + + hbaseMemRecommendations: function () { + var memory = this.get('clusterData.ram'); + var reservedHBase = {4:1, 8:1, 16:2, 24:4, 48:8, 64:8, 72:8, 96:16, + 128:24, 256:32, 512:64}; + + if (memory in reservedHBase) { + this.set('reservedRam', reservedHBase[memory]); + } + if (memory <= 4) + this.set('hBaseRam', 1); + else if (memory >= 512) + this.set('hBaseRam', 64); + else + this.set('hBaseRam', 2); + }.observes('clusterData.ram'), /** @@ -198,20 +207,21 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({ * * @return {object} */ - getDefaults: function(localDB) { + getDefaults: function (localDB) { this._super(); this.getClusterData(localDB); var configs = {}; jQuery.extend(configs, this.get('configsTemplate')); - configs['yarn.nodemanager.resource.memory-mb'] = this.get('containers') * this.get('ramPerContainer'); - configs['yarn.scheduler.minimum-allocation-mb'] = this.get('ramPerContainer'); - configs['yarn.scheduler.maximum-allocation-mb'] = this.get('containers') * this.get('ramPerContainer'); - configs['mapreduce.map.memory.mb'] = this.get('ramPerContainer'); - configs['mapreduce.reduce.memory.mb'] = 2 * this.get('ramPerContainer'); - configs['mapreduce.map.java.opts'] = Math.round(0.8 * this.get('ramPerContainer')); - configs['mapreduce.reduce.java.opts'] = Math.round(0.8 *2 * this.get('ramPerContainer')); - configs['yarn.app.mapreduce.am.resource.mb'] = 2 * this.get('ramPerContainer'); - configs['yarn.app.mapreduce.am.command-opts'] = Math.round(0.8 * 2 * this.get('ramPerContainer')); + configs['yarn.nodemanager.resource.memory-mb'] = Math.round(this.get('containers') * this.get('ramPerContainer')); + configs['yarn.scheduler.minimum-allocation-mb'] = Math.round(this.get('ramPerContainer')); + configs['yarn.scheduler.maximum-allocation-mb'] = Math.round(this.get('containers') * this.get('ramPerContainer')); + configs['yarn.app.mapreduce.am.resource.mb'] = Math.round(this.get('amMemory')); + configs['yarn.app.mapreduce.am.command-opts'] = Math.round(0.8 * this.get('amMemory')); + configs['mapreduce.map.memory.mb'] = Math.round(this.get('mapMemory')); + configs['mapreduce.reduce.memory.mb'] = Math.round(this.get('reduceMemory')); + configs['mapreduce.map.java.opts'] = Math.round(0.8 * this.get('mapMemory')); + configs['mapreduce.reduce.java.opts'] = Math.round(0.8 * this.get('reduceMemory')); + configs['mapreduce.task.io.sort.mb'] = Math.round(0.4 * this.get('mapMemory')); return configs; }, @@ -219,13 +229,13 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({ * Calculate needed cluster data (like disk count, cpu count, ram (in MB!) and hbase availability) * @param {object} localDB Object with information about hosts and master/slave components */ - getClusterData: function(localDB) { + getClusterData: function (localDB) { this._super(); - var components = ['RESOURCEMANAGER', 'NODEMANAGER']; + var components = ['NODEMANAGER']; var hosts = []; if (!localDB.hosts || !(localDB.masterComponentHosts || localDB.slaveComponentHosts)) return; var hBaseInstalled = !!localDB.masterComponentHosts.filterProperty('component', 'HBASE_MASTER').length; - components.forEach(function(component) { + components.forEach(function (component) { var mc = localDB.masterComponentHosts.findProperty('component', component); if (mc) { if (!hosts.contains(mc.hostName)) { @@ -235,7 +245,7 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({ else { var sc = localDB.slaveComponentHosts.findProperty('componentName', component); if (sc) { - sc.hosts.map(function(host) { + sc.hosts.map(function (host) { if (!hosts.contains(host.hostName)) { hosts.push(host.hostName); } @@ -249,23 +259,29 @@ App.YARNDefaultsProvider = App.DefaultsProvider.create({ ram: 0, hBaseInstalled: hBaseInstalled }; - hosts.forEach(function(hostName) { - var host = localDB.hosts[hostName]; - if (host) { - clusterData.cpu += parseInt(host.cpu); - clusterData.disk += host.disk_info.length; - clusterData.ram += Math.round(parseFloat(host.memory) / 1024); - } - }); + var host = hosts[0] && localDB.hosts[hosts[0]]; + if (host) { + clusterData.cpu = parseInt(host.cpu); + invalidMountPoints= ['/homes/','/dev/','/tmp/']; + var length = 0; + host.disk_info.forEach(function(disk) { + //invalid mountpoints + if (!(disk.mountpoint.startsWith('/homes/') || disk.mountpoint.startsWith('/dev/') || disk.mountpoint.startsWith('/tmp/'))) { + length++; + } + },this); + clusterData.disk = length; + clusterData.ram = Math.round(parseFloat(host.memory) / (1024 * 1024)); + } this.set('clusterData', clusterData); }, /** * Verify clusterData - check if all properties are defined */ - clusterDataIsValid: function() { + clusterDataIsValid: function () { if (!this.get('clusterData')) return false; - if (this.get('clusterData.ram') == null || this.get('clusterData.cpu') == null || this.get('clusterData.disk') == null || this.get('clusterData.hBaseInstalled') == null) return false; + if (this.get('clusterData.ram') == null || this.get('clusterData.cpu') == null || this.get('clusterData.disk') == null || this.get('clusterData.hBaseInstalled') == null) return false; return true; } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/2204615e/ambari-web/app/utils/helper.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/helper.js b/ambari-web/app/utils/helper.js index 60961dc..f86a2be 100644 --- a/ambari-web/app/utils/helper.js +++ b/ambari-web/app/utils/helper.js @@ -23,6 +23,9 @@ String.prototype.trim = function () { String.prototype.endsWith = function(suffix) { return this.indexOf(suffix, this.length - suffix.length) !== -1; }; +String.prototype.startsWith = function (prefix){ + return this.indexOf(prefix) == 0; +}; /** * convert ip address string to long int http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/2204615e/ambari-web/app/utils/string_utils.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/string_utils.js b/ambari-web/app/utils/string_utils.js index 85068ad..92cb75f 100644 --- a/ambari-web/app/utils/string_utils.js +++ b/ambari-web/app/utils/string_utils.js @@ -107,7 +107,7 @@ module.exports = { }, isSingleLine: function(string){ - return string.trim().indexOf("\n") == -1; + return String(string).trim().indexOf("\n") == -1; }, /** * transform array of objects into CSV format content