ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From avija...@apache.org
Subject [5/5] ambari git commit: AMBARI-18154 : Ambari Dashboard, Cluster load widget - Incorrect value in Nodes._avg metric. (avijayan)
Date Wed, 14 Sep 2016 22:03:51 GMT
AMBARI-18154 : Ambari Dashboard, Cluster load widget - Incorrect value in Nodes._avg metric.
(avijayan)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aa0528ec
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aa0528ec
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aa0528ec

Branch: refs/heads/trunk
Commit: aa0528ecab47850f44dd5575ab7861083e67c6ea
Parents: 4a09934
Author: Aravindan Vijayan <avijayan@hortonworks.com>
Authored: Tue Sep 13 20:17:40 2016 -0700
Committer: Aravindan Vijayan <avijayan@hortonworks.com>
Committed: Wed Sep 14 15:03:34 2016 -0700

----------------------------------------------------------------------
 .../TimelineMetricClusterAggregatorSecond.java  | 34 ++++++++++++++++++--
 ...melineMetricClusterAggregatorSecondTest.java |  6 ++--
 .../src/main/resources/ganglia_properties.json  |  2 +-
 3 files changed, 36 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/aa0528ec/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
index 6731eb3..5f40d21 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
@@ -34,11 +34,15 @@ import java.io.IOException;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_APP_ID;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.SERVER_SIDE_TIMESIFT_ADJUSTMENT;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_SQL;
@@ -130,6 +134,7 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
     throws SQLException, IOException {
     Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics =
       new HashMap<TimelineClusterMetric, MetricClusterAggregate>();
+    int numLiveHosts = 0;
 
     TimelineMetric metric = null;
     if (rs.next()) {
@@ -145,18 +150,25 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
           metric.addMetricValues(nextMetric.getMetricValues());
         } else {
           // Process the current metric
-          processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
+          int numHosts = processAggregateClusterMetrics(aggregateClusterMetrics, metric,
timeSlices);
+          numLiveHosts = Math.max(numHosts, numLiveHosts);
           metric = nextMetric;
         }
       }
     }
     // Process last metric
     if (metric != null) {
-      processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
+      int numHosts = processAggregateClusterMetrics(aggregateClusterMetrics, metric, timeSlices);
+      numLiveHosts = Math.max(numHosts, numLiveHosts);
     }
 
     // Add app level aggregates to save
     aggregateClusterMetrics.putAll(appAggregator.getAggregateClusterMetrics());
+
+    // Add liveHosts metric.
+    long timestamp = timeSlices.get(timeSlices.size() - 1)[1];
+    processLiveHostsMetric(aggregateClusterMetrics, numLiveHosts, timestamp);
+
     return aggregateClusterMetrics;
   }
 
@@ -165,10 +177,11 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
    * timeline.metrics.cluster.aggregator.minute.timeslice.interval
    * Normalize value by averaging them within the interval
    */
-  protected void processAggregateClusterMetrics(Map<TimelineClusterMetric, MetricClusterAggregate>
aggregateClusterMetrics,
+  protected int processAggregateClusterMetrics(Map<TimelineClusterMetric, MetricClusterAggregate>
aggregateClusterMetrics,
                                               TimelineMetric metric, List<Long[]> timeSlices)
{
     // Create time slices
     Map<TimelineClusterMetric, Double> clusterMetrics = sliceFromTimelineMetric(metric,
timeSlices);
+    int numHosts = 0;
 
     if (clusterMetrics != null && !clusterMetrics.isEmpty()) {
       for (Map.Entry<TimelineClusterMetric, Double> clusterMetricEntry :
@@ -188,10 +201,13 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
           aggregate.updateMax(avgValue);
           aggregate.updateMin(avgValue);
         }
+
+        numHosts = aggregate.getNumberOfHosts();
         // Update app level aggregates
         appAggregator.processTimelineClusterMetric(clusterMetric, metric.getHostName(), avgValue);
       }
     }
+    return numHosts;
   }
 
   protected Map<TimelineClusterMetric, Double> sliceFromTimelineMetric(
@@ -374,4 +390,16 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
     return -1l;
   }
 
+  private void processLiveHostsMetric(Map<TimelineClusterMetric, MetricClusterAggregate>
aggregateClusterMetrics,
+                                     int numLiveHosts, long timestamp) {
+
+    TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(
+      "live_hosts", HOST_APP_ID, null, timestamp, null);
+
+    MetricClusterAggregate metricClusterAggregate = new MetricClusterAggregate((double) numLiveHosts,
+      1, null, (double) numLiveHosts, (double) numLiveHosts);
+
+    aggregateClusterMetrics.put(timelineClusterMetric, metricClusterAggregate);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/aa0528ec/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
index 0f93bab..014772f 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java
@@ -259,8 +259,9 @@ public class TimelineMetricClusterAggregatorSecondTest {
     aggregateClusterMetrics.clear();
 
     timelineMetric.setType("COUNTER");
-    secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric,
timeslices);
+    int liveHosts = secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics,
timelineMetric, timeslices);
 
+    Assert.assertEquals(liveHosts, 1);
     Assert.assertEquals(aggregateClusterMetrics.size(), 4);
     timelineClusterMetric.setTimestamp(startTime + 30*seconds);
     Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));
@@ -294,8 +295,9 @@ public class TimelineMetricClusterAggregatorSecondTest {
     aggregateClusterMetrics.clear();
 
     timelineMetric.setType("COUNTER");
-    secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics, timelineMetric,
timeslices);
+    liveHosts = secondAggregator.processAggregateClusterMetrics(aggregateClusterMetrics,
timelineMetric, timeslices);
 
+    Assert.assertEquals(liveHosts, 1);
     Assert.assertEquals(aggregateClusterMetrics.size(), 4);
     timelineClusterMetric.setTimestamp(startTime + 60*seconds);
     Assert.assertTrue(aggregateClusterMetrics.containsKey(timelineClusterMetric));

http://git-wip-us.apache.org/repos/asf/ambari/blob/aa0528ec/ambari-server/src/main/resources/ganglia_properties.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/ganglia_properties.json b/ambari-server/src/main/resources/ganglia_properties.json
index 5632bc0..bd2a547 100644
--- a/ambari-server/src/main/resources/ganglia_properties.json
+++ b/ambari-server/src/main/resources/ganglia_properties.json
@@ -47,7 +47,7 @@
         "metric":"load_report.Nodes",
         "pointInTime":false,
         "temporal":true,
-        "amsId":"cpu_num"
+        "amsId":"live_hosts"
       },
       "metrics/load/Procs":{
         "metric":"load_report.Procs",


Mime
View raw message