Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B9A921759D for ; Fri, 19 Jun 2015 08:31:06 +0000 (UTC) Received: (qmail 83184 invoked by uid 500); 19 Jun 2015 08:31:06 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 83151 invoked by uid 500); 19 Jun 2015 08:31:06 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 83142 invoked by uid 99); 19 Jun 2015 08:31:06 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 19 Jun 2015 08:31:06 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 87ADCDFF96; Fri, 19 Jun 2015 08:31:06 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: atkach@apache.org To: commits@ambari.apache.org Message-Id: <85bb1364c47a43138b90ab46e2fee339@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-11992 Hosts page responsiveness is severely impacted when metrics API responses are slow. (atkach) Date: Fri, 19 Jun 2015 08:31:06 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/branch-2.1 4c8603887 -> a4780f657 AMBARI-11992 Hosts page responsiveness is severely impacted when metrics API responses are slow. (atkach) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a4780f65 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a4780f65 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a4780f65 Branch: refs/heads/branch-2.1 Commit: a4780f65723b5b3fcbfa13408e90f00829598801 Parents: 4c86038 Author: Andrii Tkach Authored: Thu Jun 18 15:33:46 2015 +0300 Committer: Andrii Tkach Committed: Fri Jun 19 11:30:31 2015 +0300 ---------------------------------------------------------------------- .../app/controllers/global/update_controller.js | 177 ++++++++++++------- .../main/dashboard/config_history_controller.js | 1 + ambari-web/app/mappers/hosts_mapper.js | 15 ++ ambari-web/app/utils/ajax/ajax.js | 16 ++ ambari-web/app/views/main/host.js | 4 +- ambari-web/app/views/main/host/details.js | 2 +- .../global/update_controller_test.js | 76 ++++++++ .../dashboard/config_history_controller_test.js | 2 +- ambari-web/test/mappers/hosts_mapper_test.js | 52 +++++- 9 files changed, 271 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a4780f65/ambari-web/app/controllers/global/update_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/global/update_controller.js b/ambari-web/app/controllers/global/update_controller.js index e265c3f..11d17ca 100644 --- a/ambari-web/app/controllers/global/update_controller.js +++ b/ambari-web/app/controllers/global/update_controller.js @@ -42,6 +42,11 @@ App.UpdateController = Em.Controller.extend({ paginationKeys: ['page_size', 'from'], /** + * @type {string} + */ + HOSTS_TEST_URL: '/data/hosts/HDP2/hosts.json', + + /** * map which track status of requests, whether it's running or completed * @type {object} */ @@ -55,23 +60,19 @@ App.UpdateController = Em.Controller.extend({ /** * construct URL from real URL and query parameters - * @param testUrl * @param realUrl * @param queryParams * @return {String} */ - getComplexUrl: function (testUrl, realUrl, queryParams) { + getComplexUrl: function (realUrl, queryParams) { var prefix = App.get('apiPrefix') + '/clusters/' + App.get('clusterName'), params = ''; - if (App.get('testMode')) { - return testUrl; - } else { - if (queryParams) { - params = this.computeParameters(queryParams); - } - return prefix + realUrl.replace('', params); + if (queryParams) { + params = this.computeParameters(queryParams); } + params = (params.length > 0) ? params + "&" : params; + return prefix + realUrl.replace('', params); }, /** @@ -111,7 +112,7 @@ App.UpdateController = Em.Controller.extend({ } params += '&'; }); - return params; + return params.substring(0, params.length - 1); }, /** @@ -172,28 +173,44 @@ App.UpdateController = Em.Controller.extend({ } }, - updateHost: function (callback, error) { - var testUrl = '/data/hosts/HDP2/hosts.json', + /** + * + * @param {Function} callback + * @param {Function} error + * @param {boolean} lazyLoadMetrics + */ + updateHost: function (callback, error, lazyLoadMetrics) { + var testUrl = this.get('HOSTS_TEST_URL'), self = this, - hostDetailsFilter = ''; - var realUrl = '/hosts?fields=Hosts/rack_info,Hosts/host_name,Hosts/maintenance_state,Hosts/public_host_name,Hosts/cpu_count,Hosts/ph_cpu_count,' + + hostDetailsFilter = '', + realUrl = '/hosts?fields=Hosts/rack_info,Hosts/host_name,Hosts/maintenance_state,Hosts/public_host_name,Hosts/cpu_count,Hosts/ph_cpu_count,' + 'alerts_summary,Hosts/host_status,Hosts/last_heartbeat_time,Hosts/ip,host_components/HostRoles/state,host_components/HostRoles/maintenance_state,' + 'host_components/HostRoles/stale_configs,host_components/HostRoles/service_name,host_components/HostRoles/desired_admin_state,' + - 'metrics/disk,metrics/load/load_one,Hosts/total_mem&minimal_response=true'; - var hostAuxiliaryInfo = ',Hosts/os_arch,Hosts/os_type,metrics/cpu/cpu_system,metrics/cpu/cpu_user,metrics/memory/mem_total,metrics/memory/mem_free'; - var stackVersionInfo = ',stack_versions/HostStackVersions,' + + 'Hosts/total_mem&minimal_response=true', + hostDetailsParams = ',Hosts/os_arch,Hosts/os_type,metrics/cpu/cpu_system,metrics/cpu/cpu_user,metrics/memory/mem_total,metrics/memory/mem_free', + stackVersionInfo = ',stack_versions/HostStackVersions,' + 'stack_versions/repository_versions/RepositoryVersions/repository_version,stack_versions/repository_versions/RepositoryVersions/id,' + - 'stack_versions/repository_versions/RepositoryVersions/display_name'; - realUrl = realUrl.replace("", (App.get('supports.stackUpgrade') ? stackVersionInfo : "")); + 'stack_versions/repository_versions/RepositoryVersions/display_name', + mainHostController = App.router.get('mainHostController'), + sortProperties = mainHostController.getSortProps(); - if (App.router.get('currentState.name') == 'index' && App.router.get('currentState.parentState.name') == 'hosts') { + if (App.router.get('currentState.parentState.name') == 'hosts') { App.updater.updateInterval('updateHost', App.get('contentUpdateInterval')); + hostDetailsParams = ''; + this.get('queryParams').set('Hosts', mainHostController.getQueryParameters(true)); } else { - if (App.router.get('currentState.parentState.name') == 'hostDetails' && - ['summary', 'alerts', 'stackVersions'].contains(App.router.get('currentState.name'))) { + if (App.router.get('currentState.parentState.name') == 'hostDetails') { hostDetailsFilter = App.router.get('location.lastSetURL').match(/\/hosts\/(.*)\/(summary|alerts|stackVersions)/)[1]; App.updater.updateInterval('updateHost', App.get('componentsUpdateInterval')); + //if host details page opened then request info only of one displayed host + this.get('queryParams').set('Hosts', [ + { + key: 'Hosts/host_name', + value: [hostDetailsFilter], + type: 'MULTIPLE' + } + ]); } else { callback(); @@ -203,24 +220,18 @@ App.UpdateController = Em.Controller.extend({ } } } - var mainHostController = App.router.get('mainHostController'), - sortProperties = mainHostController.getSortProps(); - if (hostDetailsFilter) { - //if host details page opened then request info only of one displayed host - this.get('queryParams').set('Hosts', [ - { - key: 'Hosts/host_name', - value: [hostDetailsFilter], - type: 'MULTIPLE' - } - ]); - } else { - hostAuxiliaryInfo = ''; - this.get('queryParams').set('Hosts', mainHostController.getQueryParameters(true)); - } - realUrl = realUrl.replace('', hostAuxiliaryInfo); + + realUrl = realUrl.replace("", (App.get('supports.stackUpgrade') ? stackVersionInfo : "")); + realUrl = realUrl.replace("", (lazyLoadMetrics ? "" : "metrics/disk,metrics/load/load_one,")); + realUrl = realUrl.replace('', hostDetailsParams); var clientCallback = function (skipCall, queryParams) { + var completeCallback = function () { + callback(); + if (lazyLoadMetrics) { + self.loadHostsMetric(queryParams); + } + }; if (skipCall) { //no hosts match filter by component App.hostsMapper.map({ @@ -230,35 +241,18 @@ App.UpdateController = Em.Controller.extend({ callback(); } else { - var params = self.computeParameters(queryParams), - paginationProps = self.computeParameters(queryParams.filter(function (param) { - return (this.get('paginationKeys').contains(param.key)); - }, self)), - sortProps = self.computeParameters(sortProperties); - - if ((params.length + paginationProps.length + sortProps.length) > 0) { - realUrl = App.get('apiPrefix') + '/clusters/' + App.get('clusterName') + - realUrl.replace('', '') + - (paginationProps.length > 0 ? '&' + paginationProps.substring(0, paginationProps.length - 1) : '') + - (sortProps.length > 0 ? '&' + sortProps.substring(0, sortProps.length - 1) : ''); - if (App.get('testMode')) { - realUrl = testUrl; - } - App.HttpClient.get(realUrl, App.hostsMapper, { - complete: callback, - doGetAsPost: true, - params: params.substring(0, params.length - 1), - error: error - }); - } - else { - var hostsUrl = self.getComplexUrl(testUrl, realUrl, queryParams); - App.HttpClient.get(hostsUrl, App.hostsMapper, { - complete: callback, - doGetAsPost: false, - error: error - }); + if (App.get('testMode')) { + realUrl = testUrl; + } else { + realUrl = self.addParamsToHostsUrl.call(self, queryParams, sortProperties, realUrl); } + + App.HttpClient.get(realUrl, App.hostsMapper, { + complete: completeCallback, + doGetAsPost: true, + params: self.computeParameters(queryParams), + error: error + }); } }; @@ -268,6 +262,54 @@ App.UpdateController = Em.Controller.extend({ }, /** + * + * @param {Array} queryParams + * @param {Array} sortProperties + * @param {string} realUrl + * @returns {string} + */ + addParamsToHostsUrl: function (queryParams, sortProperties, realUrl) { + var paginationProps = this.computeParameters(queryParams.filter(function (param) { + return (this.get('paginationKeys').contains(param.key)); + }, this)); + var sortProps = this.computeParameters(sortProperties); + + return App.get('apiPrefix') + '/clusters/' + App.get('clusterName') + realUrl + + (paginationProps.length > 0 ? '&' + paginationProps : '') + + (sortProps.length > 0 ? '&' + sortProps : ''); + }, + + /** + * lazy load metrics of hosts + * @param {Array} queryParams + * @returns {$.ajax|null} + */ + loadHostsMetric: function (queryParams) { + var realUrl = '/hosts?fields=metrics/disk/disk_free,metrics/disk/disk_total,metrics/load/load_one&minimal_response=true'; + + if (App.Service.find('AMBARI_METRICS').get('isStarted')) { + return App.ajax.send({ + name: 'hosts.metrics.lazy_load', + sender: this, + data: { + url: this.addParamsToHostsUrl(queryParams, [], realUrl), + parameters: this.computeParameters(queryParams) + }, + success: 'loadHostsMetricSuccessCallback' + }); + } + return null; + }, + + /** + * success callback of loadHostsMetric + * @param {object} data + */ + loadHostsMetricSuccessCallback: function (data) { + App.hostsMapper.setMetrics(data); + }, + + /** * identify if any filter by host-component is active * if so run @getHostByHostComponents * @@ -291,14 +333,13 @@ App.UpdateController = Em.Controller.extend({ * @param callback */ getHostByHostComponents: function (callback) { - var testUrl = '/data/hosts/HDP2/hosts.json'; var realUrl = '/hosts?minimal_response=true'; App.ajax.send({ name: 'hosts.host_components.pre_load', sender: this, data: { - url: this.getComplexUrl(testUrl, realUrl, this.get('queryParams.Hosts')), + url: this.getComplexUrl(realUrl, this.get('queryParams.Hosts')), callback: callback }, success: 'getHostByHostComponentsSuccessCallback', http://git-wip-us.apache.org/repos/asf/ambari/blob/a4780f65/ambari-web/app/controllers/main/dashboard/config_history_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/dashboard/config_history_controller.js b/ambari-web/app/controllers/main/dashboard/config_history_controller.js index 30b466b..2d58d95 100644 --- a/ambari-web/app/controllers/main/dashboard/config_history_controller.js +++ b/ambari-web/app/controllers/main/dashboard/config_history_controller.js @@ -226,6 +226,7 @@ App.MainConfigHistoryController = Em.ArrayController.extend(App.TableServerMixin if (queryParams) { params = App.router.get('updateController').computeParameters(queryParams); } + params = (params.length > 0) ? params + "&" : params; return this.get('realUrl').replace('', params); } }, http://git-wip-us.apache.org/repos/asf/ambari/blob/a4780f65/ambari-web/app/mappers/hosts_mapper.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/mappers/hosts_mapper.js b/ambari-web/app/mappers/hosts_mapper.js index 9a18dc2..6158ee9 100644 --- a/ambari-web/app/mappers/hosts_mapper.js +++ b/ambari-web/app/mappers/hosts_mapper.js @@ -178,5 +178,20 @@ App.hostsMapper = App.QuickDataMapper.create({ App.get('componentConfigMapper').addNewHostComponents(newHostComponentsMap, cacheServices); } console.timeEnd('App.hostsMapper execution time'); + }, + + /** + * set metric fields of hosts + * @param {object} data + */ + setMetrics: function (data) { + this.get('model').find().forEach(function (host) { + if (host.get('isRequested')) { + var hostMetrics = data.items.findProperty('Hosts.host_name', host.get('hostName')); + host.set('diskTotal', Em.get(hostMetrics, 'metrics.disk.disk_total')); + host.set('diskFree', Em.get(hostMetrics, 'metrics.disk.disk_free')); + host.set('loadOne', Em.get(hostMetrics, 'metrics.load.load_one')); + } + }, this); } }); http://git-wip-us.apache.org/repos/asf/ambari/blob/a4780f65/ambari-web/app/utils/ajax/ajax.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/ajax/ajax.js b/ambari-web/app/utils/ajax/ajax.js index b4e1ac9..c904e13 100644 --- a/ambari-web/app/utils/ajax/ajax.js +++ b/ambari-web/app/utils/ajax/ajax.js @@ -2293,6 +2293,22 @@ var urls = { } } }, + 'hosts.metrics.lazy_load': { + real: '', + mock: '/data/hosts/HDP2/hosts.json', + format: function (data) { + return { + url: data.url, + headers: { + 'X-Http-Method-Override': 'GET' + }, + type: 'POST', + data: JSON.stringify({ + "RequestInfo": {"query": data.parameters} + }) + } + } + }, 'hosts.bulk.operations': { real: '/clusters/{clusterName}/hosts?fields=Hosts/host_name,Hosts/maintenance_state,' + 'host_components/HostRoles/state,host_components/HostRoles/maintenance_state,' + http://git-wip-us.apache.org/repos/asf/ambari/blob/a4780f65/ambari-web/app/views/main/host.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/views/main/host.js b/ambari-web/app/views/main/host.js index 8c76699..51a372e 100644 --- a/ambari-web/app/views/main/host.js +++ b/ambari-web/app/views/main/host.js @@ -83,7 +83,7 @@ App.MainHostView = App.TableView.extend(App.TableServerViewMixin, { refresh: function () { this.set('filteringComplete', false); var updaterMethodName = this.get('updater.tableUpdaterMap')[this.get('tableName')]; - this.get('updater')[updaterMethodName](this.updaterSuccessCb.bind(this), this.updaterErrorCb.bind(this)); + this.get('updater')[updaterMethodName](this.updaterSuccessCb.bind(this), this.updaterErrorCb.bind(this), true); return true; }, @@ -384,7 +384,7 @@ App.MainHostView = App.TableView.extend(App.TableServerViewMixin, { name: 'hosts.bulk.operations', sender: this, data: { - parameters: params.substring(0, params.length - 1), + parameters: params, operationData: operationData, loadingPopup: loadingPopup }, http://git-wip-us.apache.org/repos/asf/ambari/blob/a4780f65/ambari-web/app/views/main/host/details.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/views/main/host/details.js b/ambari-web/app/views/main/host/details.js index efe4827..2c5cb22 100644 --- a/ambari-web/app/views/main/host/details.js +++ b/ambari-web/app/views/main/host/details.js @@ -59,7 +59,7 @@ App.MainHostDetailsView = Em.View.extend({ didInsertElement: function() { var self = this; - this.set('isLoaded', false); + this.set('isLoaded', App.Host.find(this.get('content.id')).get('isLoaded')); App.router.get('updateController').updateHost(function () { self.set('isLoaded', true); if (!self.get('content.isLoaded')) { http://git-wip-us.apache.org/repos/asf/ambari/blob/a4780f65/ambari-web/test/controllers/global/update_controller_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/controllers/global/update_controller_test.js b/ambari-web/test/controllers/global/update_controller_test.js index 87be07b..40b5c19 100644 --- a/ambari-web/test/controllers/global/update_controller_test.js +++ b/ambari-web/test/controllers/global/update_controller_test.js @@ -228,4 +228,80 @@ describe('App.UpdateController', function () { }); }); }); + + describe("#getComplexUrl()", function () { + beforeEach(function () { + sinon.stub(App, 'get').returns('mock'); + sinon.stub(controller, 'computeParameters').returns('params'); + }); + afterEach(function () { + App.get.restore(); + controller.computeParameters.restore(); + }); + it("queryParams is empty", function () { + expect(controller.getComplexUrl('')).to.equal('mock/clusters/mock'); + }); + it("queryParams is present", function () { + var queryParams = [ + { + type: "EQUAL", + key: "key", + value: "value" + } + ]; + expect(controller.getComplexUrl('', queryParams)).to.equal('mock/clusters/mockparams&'); + }); + }); + + describe("#addParamsToHostsUrl()", function () { + beforeEach(function () { + sinon.stub(App, 'get').returns('mock'); + sinon.stub(controller, 'computeParameters').returns('params'); + }); + afterEach(function () { + App.get.restore(); + controller.computeParameters.restore(); + }); + it("", function () { + expect(controller.addParamsToHostsUrl([], [], 'url')).to.equal('mock/clusters/mockurl¶ms¶ms'); + }); + }); + + describe("#loadHostsMetric()", function () { + beforeEach(function () { + this.mock = sinon.stub(App.Service, 'find'); + sinon.stub(controller, 'computeParameters'); + sinon.stub(controller, 'addParamsToHostsUrl'); + sinon.stub(App.ajax, 'send'); + }); + afterEach(function () { + App.Service.find.restore(); + controller.computeParameters.restore(); + controller.addParamsToHostsUrl.restore(); + App.ajax.send.restore(); + }); + it("AMBARI_METRICS is not started", function () { + this.mock.returns(Em.Object.create({isStarted: false})); + expect(controller.loadHostsMetric([])).to.be.null; + expect(App.ajax.send.called).to.be.false; + }); + it("AMBARI_METRICS is started", function () { + this.mock.returns(Em.Object.create({isStarted: true})); + expect(controller.loadHostsMetric([])).to.be.object; + expect(App.ajax.send.calledOnce).to.be.true; + }); + }); + + describe("#loadHostsMetricSuccessCallback()", function () { + beforeEach(function () { + sinon.stub(App.hostsMapper, 'setMetrics'); + }); + afterEach(function () { + App.hostsMapper.setMetrics.restore(); + }); + it("", function () { + controller.loadHostsMetricSuccessCallback({}); + expect(App.hostsMapper.setMetrics.calledWith({})).to.be.true; + }); + }); }); http://git-wip-us.apache.org/repos/asf/ambari/blob/a4780f65/ambari-web/test/controllers/main/dashboard/config_history_controller_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/controllers/main/dashboard/config_history_controller_test.js b/ambari-web/test/controllers/main/dashboard/config_history_controller_test.js index 48d03d5..68e99c1 100644 --- a/ambari-web/test/controllers/main/dashboard/config_history_controller_test.js +++ b/ambari-web/test/controllers/main/dashboard/config_history_controller_test.js @@ -117,7 +117,7 @@ describe('MainConfigHistoryController', function () { if ('testMode' === k) return false; return Em.get(App, k); }); - expect(controller.getUrl({})).to.equal('/api/v1/clusters/mycluster/configurations/service_config_versions?paramsfields=service_config_version,user,group_id,group_name,is_current,createtime,service_name,hosts,service_config_version_note,is_cluster_compatible&minimal_response=true'); + expect(controller.getUrl({})).to.equal('/api/v1/clusters/mycluster/configurations/service_config_versions?params&fields=service_config_version,user,group_id,group_name,is_current,createtime,service_name,hosts,service_config_version_note,is_cluster_compatible&minimal_response=true'); }); }); http://git-wip-us.apache.org/repos/asf/ambari/blob/a4780f65/ambari-web/test/mappers/hosts_mapper_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/mappers/hosts_mapper_test.js b/ambari-web/test/mappers/hosts_mapper_test.js index 2b8de9f..5e5c293 100644 --- a/ambari-web/test/mappers/hosts_mapper_test.js +++ b/ambari-web/test/mappers/hosts_mapper_test.js @@ -25,7 +25,55 @@ require('mappers/server_data_mapper'); require('mappers/hosts_mapper'); describe('App.hostsMapper', function () { + var mapper = App.hostsMapper; - - + describe("#setMetrics()", function() { + var data = { + items: [ + { + Hosts: { + host_name: 'host1' + }, + metrics: { + load: { + load_one: 1 + } + } + } + ] + }; + beforeEach(function(){ + this.mock = sinon.stub(App.Host, 'find') + }); + afterEach(function(){ + this.mock.restore(); + }); + it("Host not in the model", function() { + var host = Em.Object.create({ + hostName: 'host2', + isRequested: true + }); + this.mock.returns([host]); + mapper.setMetrics(data); + expect(host.get('loadOne')).to.be.undefined; + }); + it("Host not in the filter", function() { + var host = Em.Object.create({ + hostName: 'host1', + isRequested: false + }); + this.mock.returns([host]); + mapper.setMetrics(data); + expect(host.get('loadOne')).to.be.undefined; + }); + it("Host should have updated metrics", function() { + var host = Em.Object.create({ + hostName: 'host1', + isRequested: true + }); + this.mock.returns([host]); + mapper.setMetrics(data); + expect(host.get('loadOne')).to.equal(1); + }); + }); });