ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jai...@apache.org
Subject ambari git commit: AMBARI-11195. Need to be able to create customized graphs based on YARN queue-specific metrics. (jaimin)
Date Sun, 17 May 2015 19:41:17 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 67c65aba9 -> 421770b27


AMBARI-11195. Need to be able to create customized graphs based on YARN queue-specific metrics.
(jaimin)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/421770b2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/421770b2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/421770b2

Branch: refs/heads/trunk
Commit: 421770b27048fedfdfbff7c9f659e5f1ade387b7
Parents: 67c65ab
Author: Jaimin Jetly <jaimin@hortonworks.com>
Authored: Sun May 17 12:40:57 2015 -0700
Committer: Jaimin Jetly <jaimin@hortonworks.com>
Committed: Sun May 17 12:41:05 2015 -0700

----------------------------------------------------------------------
 ambari-web/app/assets/test/tests.js             |   1 +
 .../service/widgets/create/step2_controller.js  |  13 +-
 .../service/widgets/create/step3_controller.js  |  10 +-
 .../service/widgets/create/wizard_controller.js |  67 +++++-
 .../widgets/create/wizard_controller_test.js    | 235 +++++++++++++++++++
 5 files changed, 303 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/assets/test/tests.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/assets/test/tests.js b/ambari-web/app/assets/test/tests.js
index 64456a9..95e9f35 100644
--- a/ambari-web/app/assets/test/tests.js
+++ b/ambari-web/app/assets/test/tests.js
@@ -87,6 +87,7 @@ var files = ['test/init_model_test',
   'test/controllers/main/service/reassign/step4_controller_test',
   'test/controllers/main/service/reassign/step6_controller_test',
   'test/controllers/main/service/reassign/step7_controller_test',
+  'test/controllers/main/service/widgets/create/wizard_controller_test',
   'test/controllers/main/service/widgets/create/step1_controller_test',
   'test/controllers/main/service/widgets/create/step2_controller_test',
   'test/controllers/main/dashboard_test',

http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js b/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js
index 9c48872..65af9cd 100644
--- a/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js
+++ b/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js
@@ -343,14 +343,17 @@ App.WidgetWizardStep2Controller = Em.Controller.extend({
       value = '${';
       expression.data.forEach(function (element) {
         if (element.isMetric) {
-          metrics.push({
+          var metricObj = {
             "name": element.name,
             "service_name": element.serviceName,
             "component_name": element.componentName,
-            "metric_path": element.metricPath,
-            "host_component_criteria": element.hostComponentCriteria,
-            "category": element.category
-          });
+            "metric_path": element.metricPath
+          };
+          if (element.hostComponentCriteria) {
+            metricObj.host_component_criteria = element.hostComponentCriteria;
+          }
+          metrics.push(metricObj);
+
         }
         value += element.name;
       }, this);

http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js b/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js
index f946b74..24b32f8 100644
--- a/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js
+++ b/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js
@@ -122,15 +122,7 @@ App.WidgetWizardStep3Controller = Em.Controller.extend({
         description: this.get('widgetDescription') || "",
         scope: this.get('widgetScope').toUpperCase(),
         author: this.get('widgetAuthor'),
-        "metrics": this.get('widgetMetrics').map(function (metric) {
-          return {
-            "name": metric.name,
-            "service_name": metric.serviceName,
-            "component_name": metric.componentName,
-            "metric_path": metric.metricPath,
-            "host_component_criteria": metric.hostComponentCriteria
-          }
-        }),
+        metrics: this.get('widgetMetrics'),
         values: this.get('widgetValues').map(function (value) {
           delete value.computedValue;
           return value;

http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js b/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js
index 08a5582..4c1dd06 100644
--- a/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js
+++ b/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js
@@ -180,17 +180,17 @@ App.WidgetWizardController = App.WizardController.extend({
    * @returns {$.Deferred}
    */
   loadAllMetrics: function () {
-    var widgetMetrics = this.getDBProperty('allMetrics');
+    var allMetrics = this.getDBProperty('allMetrics');
     var self = this;
     var dfd = $.Deferred();
 
-    if (widgetMetrics.length === 0) {
+    if (allMetrics.length === 0) {
       this.loadAllMetricsFromServer(function () {
         dfd.resolve(self.get('content.allMetrics'));
       });
     } else {
-      this.set('content.allMetrics', widgetMetrics);
-      dfd.resolve(widgetMetrics);
+      this.set('content.allMetrics', allMetrics);
+      dfd.resolve(allMetrics);
     }
     return dfd.promise();
   },
@@ -223,23 +223,25 @@ App.WidgetWizardController = App.WizardController.extend({
     var self = this;
     var result = [];
     var metrics = {};
-
+    var slaveComponents = App.StackServiceComponent.find().filterProperty('isSlave').mapProperty('componentName');
     if (json) {
       json.items.forEach(function (service) {
         var data = service.artifacts[0].artifact_data[service.StackServices.service_name];
         for (var componentName in data) {
+          var isSlave = slaveComponents.contains(componentName);
           for (var level in data[componentName]) {
             var metricTypes = data[componentName][level]; //Ganglia or JMX
             metricTypes.forEach(function (_metricType) {
               metrics = _metricType['metrics']['default'];
               var type = _metricType["type"].toUpperCase();
-              if (!(type === 'JMX' && level.toUpperCase() === 'COMPONENT')) {
+              if (!((type === 'JMX' && level.toUpperCase() === 'COMPONENT') || (isSlave
&& level.toUpperCase() === 'HOSTCOMPONENT'))) {
                 for (var widgetId in metrics) {
+                  var _metrics = metrics[widgetId];
                   var metricObj = {
                     widget_id: widgetId,
-                    point_in_time: metrics[widgetId].pointInTime,
-                    temporal: metrics[widgetId].temporal,
-                    name: metrics[widgetId].name,
+                    point_in_time: _metrics.pointInTime,
+                    temporal: _metrics.temporal,
+                    name: _metrics.name,
                     level: level.toUpperCase(),
                     type: type,
                     component_name: componentName,
@@ -256,9 +258,56 @@ App.WidgetWizardController = App.WizardController.extend({
         }
       }, this);
     }
+    if (!!App.YARNService.find("YARN")) {
+      result = this.substitueQueueMetrics(result);
+    }
     this.save('allMetrics', result);
   },
 
+
+  /**
+   * @name substitueQueueMetrics
+   * @param metrics
+   * @return {Array} array of metric objects with regex substituted with actual metrics names
+   */
+  substitueQueueMetrics: function (metrics) {
+    var result = [];
+    var queuePaths = App.YARNService.find("YARN").get("allQueueNames");
+    var queueNames = [];
+    var queueMetricName;
+    queuePaths.forEach(function (_queuePath) {
+      var queueName = _queuePath.replace(/\//g, ".");
+      queueNames.push(queueName);
+    }, this);
+    var regexpForAMS = new RegExp("^yarn.QueueMetrics.Queue=\\(\\.\\+\\).*$");
+    var regexpForJMX = new RegExp("\\(\\.\\+\\)", "g");
+    var replaceRegexForMetricName = regexpForJMX;
+    var replaceRegexForMetricPath = new RegExp("\\$\\d\\..*\\)(?=\\/)", "g");
+    metrics.forEach(function (_metric) {
+      var isAMSQueueMetric = regexpForAMS.test(_metric.name);
+      var isJMXQueueMetrics = regexpForJMX.test(_metric.name);
+      if ((_metric.type === 'GANGLIA' && isAMSQueueMetric) || (_metric.type === 'JMX'
&& isJMXQueueMetrics)) {
+        queuePaths.forEach(function (_queuePath) {
+          queueMetricName = '';
+          if (_metric.type === 'GANGLIA') {
+            queueMetricName = _queuePath.replace(/\//g, ".");
+          } else if (_metric.type === 'JMX') {
+            _queuePath.split("/").forEach(function(_metricName, index){
+              queueMetricName = queueMetricName + ',q' + index + '=' + _metricName;
+            }, this);
+          }
+          var metricName = _metric.name.replace(replaceRegexForMetricName, queueMetricName);
+          var newMetricPath = _metric.widget_id.replace(replaceRegexForMetricPath, _queuePath);
+          var newQueueMetric = $.extend(true, {}, _metric, {name: metricName, widget_id:
newMetricPath});
+          result.pushObject(newQueueMetric);
+        }, this);
+      } else {
+        result.pushObject(_metric);
+      }
+    }, this);
+    return result;
+  },
+
   /**
    *
    * @param metricObj {Object}

http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js
b/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js
new file mode 100644
index 0000000..83ce39f
--- /dev/null
+++ b/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App = require('app');
+
+require('controllers/main/service/widgets/create/wizard_controller');
+
+
+describe('App.WidgetWizardController', function () {
+  var controller;
+
+  /**
+   * tests the function with following hierarchical queue scenario
+   *                   root 
+   *                    |
+   *                  queue1
+   *                 /     \
+   *             queue2   queue3
+   *                  
+   */
+  describe("#substitueQueueMetrics", function () {
+    beforeEach(function () {
+      controller = App.WidgetWizardController.create();
+      sinon.stub(App.YARNService, 'find', function (k) {
+        if ('YARN' === k) return Em.Object.create({
+          'allQueueNames': ["root", "root/queue1", "root/queue1/queue2", "root/queue1/queue3"]
+        });
+      });
+    });
+    afterEach(function () {
+      controller = '';
+      App.YARNService.find.restore();
+    });
+
+
+    var testCases = [
+      {
+        msg: 'AMS Queue metric with regex as name and regex as path should be replaced with
actual metric name and path of all existing queues',
+        inputMetrics: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            level: 'COMPONENT',
+            name: 'yarn.QueueMetrics.Queue=(.+).AppsFailed',
+            point_in_time: false,
+            service_name: 'YARN',
+            temporal: true,
+            type: 'GANGLIA',
+            widget_id: 'metrics/yarn/Queue/$1.replaceAll("([.])","/")/AppsFailed'
+          }
+        ],
+        expectedResult: [
+          {
+            "component_name": "RESOURCEMANAGER",
+            "level": "COMPONENT",
+            "name": "yarn.QueueMetrics.Queue=root.AppsFailed",
+            "point_in_time": false,
+            "service_name": "YARN",
+            "temporal": true,
+            "type": "GANGLIA",
+            "widget_id": "metrics/yarn/Queue/root/AppsFailed"
+          },
+          {
+            "component_name": "RESOURCEMANAGER",
+            "level": "COMPONENT",
+            "name": "yarn.QueueMetrics.Queue=root.queue1.AppsFailed",
+            "point_in_time": false,
+            "service_name": "YARN",
+            "temporal": true,
+            "type": "GANGLIA",
+            "widget_id": "metrics/yarn/Queue/root/queue1/AppsFailed"
+          },
+          {
+            "component_name": "RESOURCEMANAGER",
+            "level": "COMPONENT",
+            "name": "yarn.QueueMetrics.Queue=root.queue1.queue2.AppsFailed",
+            "point_in_time": false,
+            "service_name": "YARN",
+            "temporal": true,
+            "type": "GANGLIA",
+            "widget_id": "metrics/yarn/Queue/root/queue1/queue2/AppsFailed"
+          },
+          {
+            "component_name": "RESOURCEMANAGER",
+            "level": "COMPONENT",
+            "name": "yarn.QueueMetrics.Queue=root.queue1.queue3.AppsFailed",
+            "point_in_time": false,
+            "service_name": "YARN",
+            "temporal": true,
+            "type": "GANGLIA",
+            "widget_id": "metrics/yarn/Queue/root/queue1/queue3/AppsFailed"
+          }
+        ]
+      },
+      {
+        msg: 'JMX Queue metric with regex as name and regex as path should be replaced with
actual metric name and path of all existing queues',
+        inputMetrics: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/$1.replaceAll(",q(\d+)=","/").substring(1)/AppsFailed'
+          }
+        ],
+        expectedResult: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/root/AppsFailed'
+          },
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/root/queue1/AppsFailed'
+          },
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1,q2=queue2.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/root/queue1/queue2/AppsFailed'
+          },
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1,q2=queue3.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/root/queue1/queue3/AppsFailed'
+          }
+        ]
+      },
+      {
+        msg: 'AMS Queue metric without regex in name and path should retain same name and
path',
+        inputMetrics: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            level: 'COMPONENT',
+            name: 'yarn.QueueMetrics.Queue.Clustermetrics.AppsFailed',
+            point_in_time: false,
+            service_name: 'YARN',
+            temporal: true,
+            type: 'GANGLIA',
+            widget_id: 'metrics/yarn/Queue/Clustermetrics/AppsFailed'
+          }
+        ],
+        expectedResult: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            level: 'COMPONENT',
+            name: 'yarn.QueueMetrics.Queue.Clustermetrics.AppsFailed',
+            point_in_time: false,
+            service_name: 'YARN',
+            temporal: true,
+            type: 'GANGLIA',
+            widget_id: 'metrics/yarn/Queue/Clustermetrics/AppsFailed'
+          }
+        ]
+      },
+      {
+        msg: 'JMX Queue metric without regex in name and path should retain same name and
path',
+        inputMetrics: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics.clusterMetric.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/clusterMetric/AppsFailed'
+          }
+        ],
+        expectedResult: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics.clusterMetric.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/clusterMetric/AppsFailed'
+          }
+        ]
+      }
+    ];
+    testCases.forEach(function (_testCase) {
+      it(_testCase.msg, function () {
+        var result = controller.substitueQueueMetrics(_testCase.inputMetrics);
+        expect(JSON.stringify(result)).to.equal(JSON.stringify(_testCase.expectedResult));
+      });
+    });
+  });
+});


Mime
View raw message