incubator-ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From atk...@apache.org
Subject [1/3] AMBARI-3714 [perf] UI should only request data from master host-components. (atkach)
Date Thu, 07 Nov 2013 16:01:29 GMT
Updated Branches:
  refs/heads/trunk 8944c1db3 -> 5820bd7c6


http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/5820bd7c/ambari-web/app/controllers/global/update_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/global/update_controller.js b/ambari-web/app/controllers/global/update_controller.js
index d3a46ed..381a4b5 100644
--- a/ambari-web/app/controllers/global/update_controller.js
+++ b/ambari-web/app/controllers/global/update_controller.js
@@ -38,7 +38,7 @@ App.UpdateController = Em.Controller.extend({
   updateAll:function(){
     if(this.get('isWorking')) {
       App.updater.run(this, 'updateHostConditionally', 'isWorking');
-      App.updater.run(this, 'updateServiceMetric', 'isWorking');
+      App.updater.run(this, 'updateServiceMetric', 'isWorking', App.componentsUpdateInterval);
       App.updater.run(this, 'graphsUpdate', 'isWorking');
       if (App.supports.hostOverrides) {
         App.updater.run(this, 'updateComponentConfig', 'isWorking');
@@ -100,7 +100,7 @@ App.UpdateController = Em.Controller.extend({
     self.set('isUpdated', false);
 
     var conditionalFields = [];
-    var initialFields = ['components/host_components/HostRoles/state'];
+    var initialFields = [];
     var services = [
         {
             name: 'FLUME',
@@ -117,7 +117,7 @@ App.UpdateController = Em.Controller.extend({
     ];
     services.forEach(function(service) {
         if (App.Service.find(service.name)) {
-            conditionalFields.push("components/host_components/metrics/" + service.urlParam);
+            conditionalFields.push("host_components/metrics/" + service.urlParam);
         }
     });
     var conditionalFieldsString = conditionalFields.length > 0 ? ',' + conditionalFields.join(',')
: '';
@@ -125,28 +125,30 @@ App.UpdateController = Em.Controller.extend({
     var methodStartTs = new Date().getTime();
     var testUrl = App.get('isHadoop2Stack') ? '/data/dashboard/HDP2/services.json':'/data/dashboard/services.json';
 
-    var realUrl = '/services?fields=components/ServiceComponentInfo,' +
-      'components/host_components/metrics/jvm/memHeapUsedM,' +
-      'components/host_components/metrics/jvm/memHeapCommittedM,' +
-      'components/host_components/metrics/mapred/jobtracker/trackers_decommissioned,' +
-      'components/host_components/metrics/cpu/cpu_wio,' +
-      'components/host_components/metrics/rpc/RpcQueueTime_avg_time,' +
-      'components/host_components/metrics/dfs/FSNamesystem/HAState,' +
-      'components/host_components/metrics/dfs/FSNamesystem/CapacityUsed,' +
-      'components/host_components/metrics/dfs/FSNamesystem/CapacityTotal,' +
-      'components/host_components/metrics/dfs/FSNamesystem/CapacityRemaining,' +
-      'components/host_components/metrics/dfs/FSNamesystem/BlocksTotal,' +
-      'components/host_components/metrics/dfs/FSNamesystem/CorruptBlocks,' +
-      'components/host_components/metrics/dfs/FSNamesystem/MissingBlocks,' +
-      'components/host_components/metrics/dfs/FSNamesystem/UnderReplicatedBlocks,' +
-      'components/host_components/metrics/dfs/namenode/Version,' +
-      'components/host_components/metrics/dfs/namenode/LiveNodes,' +
-      'components/host_components/metrics/dfs/namenode/DeadNodes,' +
-      'components/host_components/metrics/dfs/namenode/DecomNodes,' +
-      'components/host_components/metrics/dfs/namenode/TotalFiles,' +
-      'components/host_components/metrics/dfs/namenode/UpgradeFinalized,' +
-      'components/host_components/metrics/dfs/namenode/Safemode,' +
-      'components/host_components/metrics/runtime/StartTime' +
+    var realUrl = '/components/?ServiceComponentInfo/category=MASTER&fields=' +
+      'ServiceComponentInfo,' +
+      'host_components/HostRoles/state,' +
+      'host_components/metrics/jvm/memHeapUsedM,' +
+      'host_components/metrics/jvm/memHeapCommittedM,' +
+      'host_components/metrics/mapred/jobtracker/trackers_decommissioned,' +
+      'host_components/metrics/cpu/cpu_wio,' +
+      'host_components/metrics/rpc/RpcQueueTime_avg_time,' +
+      'host_components/metrics/dfs/FSNamesystem/HAState,' +
+      'host_components/metrics/dfs/FSNamesystem/CapacityUsed,' +
+      'host_components/metrics/dfs/FSNamesystem/CapacityTotal,' +
+      'host_components/metrics/dfs/FSNamesystem/CapacityRemaining,' +
+      'host_components/metrics/dfs/FSNamesystem/BlocksTotal,' +
+      'host_components/metrics/dfs/FSNamesystem/CorruptBlocks,' +
+      'host_components/metrics/dfs/FSNamesystem/MissingBlocks,' +
+      'host_components/metrics/dfs/FSNamesystem/UnderReplicatedBlocks,' +
+      'host_components/metrics/dfs/namenode/Version,' +
+      'host_components/metrics/dfs/namenode/LiveNodes,' +
+      'host_components/metrics/dfs/namenode/DeadNodes,' +
+      'host_components/metrics/dfs/namenode/DecomNodes,' +
+      'host_components/metrics/dfs/namenode/TotalFiles,' +
+      'host_components/metrics/dfs/namenode/UpgradeFinalized,' +
+      'host_components/metrics/dfs/namenode/Safemode,' +
+      'host_components/metrics/runtime/StartTime' +
       conditionalFieldsString;
 
     var servicesUrl = isInitialLoad ? this.getUrl(testUrl, realUrl + initialFieldsString)
: this.getUrl(testUrl, realUrl);

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/5820bd7c/ambari-web/app/mappers/hosts_mapper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mappers/hosts_mapper.js b/ambari-web/app/mappers/hosts_mapper.js
index 32171d0..e02d67b 100644
--- a/ambari-web/app/mappers/hosts_mapper.js
+++ b/ambari-web/app/mappers/hosts_mapper.js
@@ -55,6 +55,7 @@ App.hostsMapper = App.QuickDataMapper.create({
       var result = [];
       var hostIds = {};
       var cacheData = App.cache['Hosts'];
+      var currentHostStatuses = {};
 
       json.items.forEach(function (item) {
         //receive host_components when added hosts
@@ -64,8 +65,11 @@ App.hostsMapper = App.QuickDataMapper.create({
         }, this);
 
         hostIds[item.Hosts.host_name] = true;
+        currentHostStatuses[item.Hosts.host_name] = item.Hosts.host_status;
         result.push(this.parseIt(item, this.config));
       }, this);
+
+      App.cache['previousHostStatuses'] = currentHostStatuses;
       result = this.sortByPublicHostName(result);
 
       var clientHosts = App.Host.find();

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/5820bd7c/ambari-web/app/mappers/server_data_mapper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mappers/server_data_mapper.js b/ambari-web/app/mappers/server_data_mapper.js
index e77a261..0762a6b 100644
--- a/ambari-web/app/mappers/server_data_mapper.js
+++ b/ambari-web/app/mappers/server_data_mapper.js
@@ -20,7 +20,9 @@ var App = require('app');
 
 //initialize common cache container for mappers
 App.cache = {
-  'Hosts': {}
+  'Hosts': {},
+  'previousHostStatuses': {},
+  'previousComponentStatuses': {}
 };
 
 App.ServerDataMapper = Em.Object.extend({

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/5820bd7c/ambari-web/app/mappers/service_mapper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mappers/service_mapper.js b/ambari-web/app/mappers/service_mapper.js
index fe6e215..53bbf41 100644
--- a/ambari-web/app/mappers/service_mapper.js
+++ b/ambari-web/app/mappers/service_mapper.js
@@ -175,55 +175,136 @@ App.servicesMapper = App.QuickDataMapper.create({
     }
     console.time('App.servicesMapper execution time');
     if (json.items) {
+
+      // Host components
+      var hostComponents = [];
+      var services = [];
+      var previousComponentStatuses = App.cache['previousComponentStatuses'];
+      /**
+       * services contains constructed service-components structure from components array
+       */
+      json.items.forEach(function (component) {
+        var service = services.findProperty('ServiceInfo.service_name', component.ServiceComponentInfo.service_name);
+        var serviceName = component.ServiceComponentInfo.service_name;
+        if (!service) {
+          service = {
+            ServiceInfo: {
+              service_name: component.ServiceComponentInfo.service_name
+            },
+            host_components: [],
+            components: []
+          };
+          services.push(service);
+        }
+        service.components.push(component);
+        component.host_components.forEach(function (host_component) {
+          host_component.id = host_component.HostRoles.component_name + "_" + host_component.HostRoles.host_name;
+          previousComponentStatuses[host_component.id] = host_component.HostRoles.state;
+          if (host_component.HostRoles.component_name == "HBASE_MASTER") {
+            this.config3.ha_status = 'metrics.hbase.master.IsActiveMaster';
+          }
+          var comp = this.parseIt(host_component, this.config3);
+          comp.service_id = serviceName;
+          hostComponents.push(comp);
+        }, this);
+      }, this);
+
+      //load master components to model
+      App.store.loadMany(this.get('model3'), hostComponents);
+
       var result = [];
-      json.items.forEach(function (item) {
-        var finalConfig = jQuery.extend({}, this.config);
+
+      //establish relations between services and host-components
+      var serviceComponents = {},
+          masterComponents = [],
+          datanodeHosts = [],
+          taskTrackerHosts = [],
+          hbaseRegionserverHosts = [],
+          nodemanagerHosts = [],
+          yarnClientHosts = [],
+          mapreduce2ClientHosts = [];
+      App.HostComponent.find().forEach(function (hostComponent) {
+        if (serviceComponents[hostComponent.get('service.id')]) {
+          serviceComponents[hostComponent.get('service.id')].hostComponents.push(hostComponent.get('id'));
+        } else {
+          serviceComponents[hostComponent.get('service.id')] = {
+            hostComponents: [hostComponent.get('id')]
+          }
+        }
+
+        if (hostComponent.get('isMaster')) {
+          masterComponents.push(hostComponent);
+        }
+
+        switch (hostComponent.get('componentName')) {
+          case 'DATANODE':
+            datanodeHosts.push(hostComponent.get('host.id'));
+            break;
+          case 'TASKTRACKER':
+            taskTrackerHosts.push(hostComponent.get('host.id'));
+            break;
+          case 'HBASE_REGIONSERVER':
+            hbaseRegionserverHosts.push(hostComponent.get('host.id'));
+            break;
+          case 'NODEMANAGER':
+            nodemanagerHosts.push(hostComponent.get('host.id'));
+            break;
+          case 'YARN_CLIENT':
+            yarnClientHosts.push(hostComponent.get('host.id'));
+            break;
+          case 'MAPREDUCE2_CLIENT':
+            mapreduce2ClientHosts.push(hostComponent.get('host.id'));
+            break;
+        }
+      });
+      //parse service metrics from components
+      services.forEach(function (item) {
         var finalJson = [];
-        item.host_components = [];
-        item.components.forEach(function (component) {
-          component.host_components.forEach(function (host_component) {
-            host_component.id = host_component.HostRoles.component_name + "_" + host_component.HostRoles.host_name;
-            item.host_components.push(host_component.id);
-          }, this);
-        }, this);
+        item.host_components = serviceComponents[item.ServiceInfo.service_name].hostComponents;
         item.host_components.sort();
 
         if (item && item.ServiceInfo && item.ServiceInfo.service_name ==
"HDFS") {
           finalJson = this.hdfsMapper(item);
+          finalJson.data_nodes = datanodeHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.HDFSService, finalJson);
-        }else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "MAPREDUCE") {
+        } else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "MAPREDUCE") {
           finalJson = this.mapreduceMapper(item);
+          finalJson.task_trackers = taskTrackerHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.MapReduceService, finalJson);
-        }else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "HBASE") {
+        } else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "HBASE") {
           finalJson = this.hbaseMapper(item);
+          finalJson.region_servers = hbaseRegionserverHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.HBaseService, finalJson);
-        }else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "FLUME") {
+        } else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "FLUME") {
           finalJson = this.flumeMapper(item);
           finalJson.rand = Math.random();
           result.push(finalJson);
-          if(finalJson.nodeObjs){
-            finalJson.nodeObjs.forEach(function(no){
+          if (finalJson.nodeObjs) {
+            finalJson.nodeObjs.forEach(function (no) {
               App.store.load(App.FlumeNode, no);
             });
           }
           App.store.load(App.FlumeService, finalJson);
-        }else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "YARN") {
+        } else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "YARN") {
           finalJson = this.yarnMapper(item);
+          finalJson.node_manager_nodes = nodemanagerHosts;
+          finalJson.yarn_client_nodes = yarnClientHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.YARNService, finalJson);
-        }else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "MAPREDUCE2") {
+        } else if (item && item.ServiceInfo && item.ServiceInfo.service_name
== "MAPREDUCE2") {
           finalJson = this.mapreduce2Mapper(item);
+          finalJson.map_reduce2_clients = mapreduce2ClientHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.MapReduce2Service, finalJson);
-        }else {
+        } else {
           finalJson = this.parseIt(item, this.config);
           finalJson.rand = Math.random();
           this.mapQuickLinks(finalJson, item);
@@ -231,73 +312,19 @@ App.servicesMapper = App.QuickDataMapper.create({
         }
       }, this);
 
-
       result = this.sortByOrder(this.get('servicesSortOrder'), result);
-      App.store.loadMany(this.get('model'), result);
-
-      // Host components
-      result = [];
-      var hostComponentsMap = {};
-      json.items.forEach(function(item){
-        item.components.forEach(function(component){
-          var service = component.ServiceComponentInfo.service_name;
-          component.host_components.forEach(function(host_component){
-            if(host_component.HostRoles.component_name == "HBASE_MASTER"){
-              this.config3.ha_status = 'metrics.hbase.master.IsActiveMaster';
-            }
-            var comp = this.parseIt(host_component, this.config3);
-            comp.service_id = service;
-            this.calculateState(comp);
-            result.push(comp);
-            hostComponentsMap[comp.id] = comp;
-          }, this)
-        }, this)
-      }, this);
 
+      //load services to model
+      App.store.loadMany(this.get('model'), result);
 
-      var oldHostComponents = App.HostComponent.find();
-      var item;
-      var currentHCWithComponentNames = {};
-      var currentComponentNameHostNames = {};
-      var doHostComponentsLoad = false;
       var servicesMap = {};
-      oldHostComponents.forEach(function (item) {
+      //calculate service statuses according to their host-components
+      masterComponents.forEach(function (item) {
         if (item) {
-          if (!hostComponentsMap[item.get('id')]) {
-            item.deleteRecord();
-            item.get('stateManager').transitionTo('loading');
-          } else {
-            var componentName = item.get('componentName');
-            if (componentName) {
-              currentHCWithComponentNames[item.get('id')] = item.get('id');
-            }
-            if (!currentComponentNameHostNames[componentName]) {
-              currentComponentNameHostNames[componentName] = [];
-            }
-            currentComponentNameHostNames[componentName].pushObject(item.get('host.hostName'));
-          }
           this.countServiceComponents(item, servicesMap, servicesMap[item.get('service.id')]);
         }
       }, this);
-
       this.updateServicesStatus(App.Service.find(), servicesMap);
-
-      for(var i in hostComponentsMap) {
-        if(!currentHCWithComponentNames[i]) {
-          doHostComponentsLoad = true;
-          break;
-        }
-      }
-      result.forEach(function (item) {
-        if (currentHCWithComponentNames[item.id] != null && 
-            !currentComponentNameHostNames[item.component_name].contains(item.host_id)) {
-          item.id = (new Date).getTime();
-        }
-      });
-
-      if (!App.router.get('clusterController.isLoaded') || doHostComponentsLoad) {
-        App.store.loadMany(this.get('model3'), result);
-      }
     }
     console.timeEnd('App.servicesMapper execution time');
   },
@@ -600,16 +627,6 @@ App.servicesMapper = App.QuickDataMapper.create({
           });
         }
       }
-      if (component.ServiceComponentInfo && component.ServiceComponentInfo.component_name
== "DATANODE") {
-        if (!item.data_nodes) {
-          item.data_nodes = [];
-        }
-        if (component.host_components) {
-          component.host_components.forEach(function (hc) {
-            item.data_nodes.push(hc.HostRoles.host_name);
-          });
-        }
-      }
     });
     // Map
     var finalJson = this.parseIt(item, finalConfig);
@@ -650,26 +667,6 @@ App.servicesMapper = App.QuickDataMapper.create({
         // extend config
         finalConfig = jQuery.extend(finalConfig, yarnConfig);
       }
-      if (component.ServiceComponentInfo && component.ServiceComponentInfo.component_name
== "NODEMANAGER") {
-        if (!item.node_manager_nodes) {
-          item.node_manager_nodes = [];
-        }
-        if (component.host_components) {
-          component.host_components.forEach(function (hc) {
-            item.node_manager_nodes.push(hc.HostRoles.host_name);
-          });
-        }
-      }
-      if (component.ServiceComponentInfo && component.ServiceComponentInfo.component_name
== "YARN_CLIENT") {
-        if (!item.yarn_client_nodes) {
-          item.yarn_client_nodes = [];
-        }
-        if (component.host_components) {
-          component.host_components.forEach(function (hc) {
-            item.yarn_client_nodes.push(hc.HostRoles.host_name);
-          });
-        }
-      }
     });
     // Map
     var finalJson = this.parseIt(item, finalConfig);
@@ -700,16 +697,6 @@ App.servicesMapper = App.QuickDataMapper.create({
         item.jobHistoryServerComponent = component;
         finalConfig = jQuery.extend(finalConfig, mapReduce2Config);
       }
-      if (component.ServiceComponentInfo && component.ServiceComponentInfo.component_name
== "MAPREDUCE2_CLIENT") {
-        if (!item.map_reduce2_clients) {
-          item.map_reduce2_clients = [];
-        }
-        if (component.host_components) {
-          component.host_components.forEach(function (hc) {
-            item.map_reduce2_clients.push(hc.HostRoles.host_name);
-          });
-        }
-      }
     });
     // Map
     var finalJson = this.parseIt(item, finalConfig);
@@ -759,19 +746,9 @@ App.servicesMapper = App.QuickDataMapper.create({
           });
         }
       }
-      if (component.ServiceComponentInfo && component.ServiceComponentInfo.component_name
== "TASKTRACKER") {
-        if (!item.task_trackers) {
-          item.task_trackers = [];
-        }
-        if (component.host_components) {
-          component.host_components.forEach(function (hc) {
-            item.task_trackers.push(hc.HostRoles.host_name);
-          });
-        }
-      }
     });
     // Map
-    finalJson = this.parseIt(item, finalConfig);
+    var finalJson = this.parseIt(item, finalConfig);
     finalJson.quick_links = [5, 6, 7, 8, 9, 10, 11, 12];
     return finalJson;
   },
@@ -791,19 +768,9 @@ App.servicesMapper = App.QuickDataMapper.create({
           item.regions_in_transition = regionsArray == null ? 0 : regionsArray;
         }
       }
-      if (component.ServiceComponentInfo && component.ServiceComponentInfo.component_name
== "HBASE_REGIONSERVER") {
-        if (!item.region_servers) {
-          item.region_servers = [];
-        }
-        if (component.host_components) {
-          component.host_components.forEach(function (hc) {
-            item.region_servers.push(hc.HostRoles.host_name);
-          });
-        }
-      }
     });
     // Map
-    finalJson = this.parseIt(item, finalConfig);
+    var finalJson = this.parseIt(item, finalConfig);
     finalJson.average_load = parseFloat(finalJson.average_load).toFixed(2);
     finalJson.quick_links = [13, 14, 15, 16, 17, 18];
     return finalJson;

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/5820bd7c/ambari-web/app/mappers/status_mapper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mappers/status_mapper.js b/ambari-web/app/mappers/status_mapper.js
index 3ae39cd..08bdb19 100644
--- a/ambari-web/app/mappers/status_mapper.js
+++ b/ambari-web/app/mappers/status_mapper.js
@@ -17,9 +17,6 @@
 
 var App = require('app');
 
-var previousComponentStatuses = {};
-var previousHostStatuses = {};
-
 App.statusMapper = App.QuickDataMapper.create({
   model: App.HostComponent,
   componentServiceMap: {
@@ -71,6 +68,8 @@ App.statusMapper = App.QuickDataMapper.create({
       var componentServiceMap = this.get('componentServiceMap');
       var currentComponentStatuses = {};
       var currentHostStatuses = {};
+      var previousHostStatuses = App.cache['previousHostStatuses'];
+      var previousComponentStatuses = App.cache['previousComponentStatuses'];
 
       json.items.forEach(function (host) {
         //update hosts, which have status changed
@@ -130,8 +129,8 @@ App.statusMapper = App.QuickDataMapper.create({
         App.store.loadMany(this.get('model'), addedHostComponents);
       }
 
-      previousComponentStatuses = currentComponentStatuses;
-      previousHostStatuses = currentHostStatuses;
+      App.cache['previousHostStatuses'] = currentHostStatuses;
+      App.cache['previousComponentStatuses'] = currentComponentStatuses;
 
       hosts.forEach(function (host) {
         var status = hostStatuses[host.get('id')];


Mime
View raw message