ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-11260. Cluster aggregate metric graphs are showing averages, not sum. (swagle)
Date Thu, 21 May 2015 17:08:30 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 97eec61d1 -> 9371bb760


AMBARI-11260. Cluster aggregate metric graphs are showing averages, not sum. (swagle)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9371bb76
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9371bb76
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9371bb76

Branch: refs/heads/trunk
Commit: 9371bb7601c42b250ec258b85a95a5cc1f55ba61
Parents: 97eec61
Author: Siddharth Wagle <swagle@hortonworks.com>
Authored: Thu May 21 10:08:26 2015 -0700
Committer: Siddharth Wagle <swagle@hortonworks.com>
Committed: Thu May 21 10:08:26 2015 -0700

----------------------------------------------------------------------
 .../server/api/services/AmbariMetaInfo.java     |  92 ++++++++++++-----
 .../internal/AbstractPropertyProvider.java      |   4 +-
 .../metrics/MetricsReportPropertyProvider.java  |   2 +
 .../MetricsReportPropertyProviderProxy.java     |  15 +++
 .../timeline/AMSComponentPropertyProvider.java  |  58 +----------
 .../metrics/timeline/AMSPropertyProvider.java   |  34 ++-----
 .../timeline/AMSReportPropertyProvider.java     |  42 +++++++-
 .../controller/utilities/PropertyHelper.java    |  76 ++++++++++++++
 .../timeline/AMSPropertyProviderTest.java       |   2 +
 .../timeline/AMSReportPropertyProviderTest.java |  50 +++++++++-
 .../ams/aggregate_cluster_metrics.json          | 100 +++++++++++++++++++
 11 files changed, 361 insertions(+), 114 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index 0761614..f411aaa 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@ -18,24 +18,10 @@
 
 package org.apache.ambari.server.api.services;
 
-import java.io.File;
-import java.io.FileReader;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Scanner;
-import java.util.Set;
-
-import javax.xml.bind.JAXBException;
-
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.ParentObjectNotFoundException;
 import org.apache.ambari.server.StackAccessException;
@@ -71,16 +57,32 @@ import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
 import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
 import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
 import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptorFactory;
+import org.apache.ambari.server.state.stack.Metric;
 import org.apache.ambari.server.state.stack.MetricDefinition;
 import org.apache.ambari.server.state.stack.OsFamily;
 import org.apache.ambari.server.state.stack.UpgradePack;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
+import javax.xml.bind.JAXBException;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.Set;
+
+import static org.apache.ambari.server.controller.spi.Resource.InternalType.Component;
+import static org.apache.ambari.server.controller.utilities.PropertyHelper.AGGREGATE_FUNCTION_IDENTIFIERS;
 
 
 /**
@@ -867,7 +869,7 @@ public class AmbariMetaInfo {
       try {
         map = gson.fromJson(new FileReader(svc.getMetricsFile()), type);
 
-        svc.setMetrics(map);
+        svc.setMetrics(updateComponentMetricMapWithAggregateFunctionIds(map));
 
       } catch (Exception e) {
         LOG.error ("Could not read the metrics file", e);
@@ -879,6 +881,50 @@ public class AmbariMetaInfo {
   }
 
   /**
+   * Add aggregate function support for all stack defined metrics.
+   */
+  private Map<String, Map<String, List<MetricDefinition>>> updateComponentMetricMapWithAggregateFunctionIds(
+      Map<String, Map<String, List<MetricDefinition>>> metricMap) {
+
+    if (!metricMap.isEmpty()) {
+      // For every Component
+      for (Map<String, List<MetricDefinition>> componentMetricDef :  metricMap.values())
{
+        // For every Component / HostComponent category
+        for (Map.Entry<String, List<MetricDefinition>> metricDefEntry : componentMetricDef.entrySet())
{
+          // NOTE: Only Component aggregates supported for now.
+          if (metricDefEntry.getKey().equals(Component.name())) {
+            //For every metric definition
+            for (MetricDefinition metricDefinition : metricDefEntry.getValue()) {
+              Map<String, Metric> newMetrics = new HashMap<String, Metric>();
+              // Metrics System metrics only
+              if (metricDefinition.getType().equals("ganglia")) {
+                // For every function id
+                for (String identifierToAdd : AGGREGATE_FUNCTION_IDENTIFIERS) {
+                  for (Map.Entry<String, Metric> metricEntry : metricDefinition.getMetrics().entrySet())
{
+                    String newMetricKey = metricEntry.getKey() + identifierToAdd;
+                    Metric currentMetric = metricEntry.getValue();
+                    Metric newMetric = new Metric(
+                      currentMetric.getName() + identifierToAdd,
+                      currentMetric.isPointInTime(),
+                      currentMetric.isTemporal(),
+                      currentMetric.isAmsHostMetric(),
+                      currentMetric.getUnit()
+                    );
+                    newMetrics.put(newMetricKey, newMetric);
+                  }
+                }
+              }
+              metricDefinition.getMetrics().putAll(newMetrics);
+            }
+          }
+        }
+      }
+    }
+
+    return metricMap;
+  }
+
+  /**
    * Gets the metrics for a Role (component).
    * @return the list of defined metrics.
    */

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
index 98b42e0..96fa24b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
@@ -21,6 +21,8 @@ package org.apache.ambari.server.controller.internal;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.text.DecimalFormat;
+import java.util.Date;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -32,6 +34,7 @@ import java.util.regex.Pattern;
 import org.apache.ambari.server.controller.metrics.MetricReportingAdapter;
 import org.apache.ambari.server.controller.spi.PropertyProvider;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
 
 /**
@@ -384,5 +387,4 @@ public abstract class AbstractPropertyProvider extends BaseProvider implements
P
       return length > 0 ? dataPoints[length - 1][0] : 0;
     }
   }
-
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java
index 20e11ad..5399436 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java
@@ -35,6 +35,8 @@ public abstract class MetricsReportPropertyProvider extends AbstractPropertyProv
 
   protected final ComponentSSLConfiguration configuration;
 
+  protected static final MetricsPaddingMethod DEFAULT_PADDING_METHOD =
+    new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS);
 
   // ----- Constants --------------------------------------------------------
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java
index 1bcf01a..a92cb37 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java
@@ -80,6 +80,21 @@ public class MetricsReportPropertyProviderProxy extends AbstractPropertyProvider
       clusterNamePropertyId);
   }
 
+  /**
+   * Allow delegates to support special properties not stack defined.
+   */
+  @Override
+  public Set<String> checkPropertyIds(Set<String> propertyIds) {
+    MetricsService metricsService = metricsServiceProvider.getMetricsServiceType();
+    Set<String> checkedPropertyIds = super.checkPropertyIds(propertyIds);
+
+    if (metricsService != null && metricsService.equals(TIMELINE_METRICS)) {
+      return amsMetricsReportProvider.checkPropertyIds(checkedPropertyIds);
+    } else {
+      return checkedPropertyIds;
+    }
+  }
+
   @Override
   public Set<Resource> populateResources(Set<Resource> resources, Request request,
                                          Predicate predicate) throws SystemException {

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
index ded96cf..d5f415a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
@@ -20,17 +20,10 @@ package org.apache.ambari.server.controller.metrics.timeline;
 import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
 import org.apache.ambari.server.controller.internal.PropertyInfo;
 import org.apache.ambari.server.controller.metrics.MetricHostProvider;
-import org.apache.ambari.server.controller.spi.Predicate;
-import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.controller.utilities.PredicateHelper;
 import org.apache.ambari.server.controller.utilities.StreamProvider;
-import org.apache.commons.lang.StringUtils;
 
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
-import java.util.Set;
 
 public class AMSComponentPropertyProvider extends AMSPropertyProvider {
 
@@ -41,59 +34,10 @@ public class AMSComponentPropertyProvider extends AMSPropertyProvider
{
                                           String clusterNamePropertyId,
                                           String componentNamePropertyId) {
 
-    super(updateComponentMetricsWithAggregateFunctionSupport(componentPropertyInfoMap),
-      streamProvider, configuration, hostProvider,
+    super(componentPropertyInfoMap, streamProvider, configuration, hostProvider,
       clusterNamePropertyId, null, componentNamePropertyId);
   }
 
-  /**
-   * This method adds supported propertyInfo for component metrics with
-   * aggregate function ids. API calls with multiple aggregate functions
-   * applied to a single metric need this support.
-   *
-   * Currently this support is added only for Component metrics,
-   * this can be easily extended to all levels by moving this method to the
-   * base class: @AMSPropertyProvider.
-   */
-  private static Map<String, Map<String, PropertyInfo>> updateComponentMetricsWithAggregateFunctionSupport(
-        Map<String, Map<String, PropertyInfo>> componentMetrics) {
-
-    if (componentMetrics == null || componentMetrics.isEmpty()) {
-      return componentMetrics;
-    }
-
-    // For every component
-    for (Map<String, PropertyInfo> componentMetricInfo : componentMetrics.values())
{
-      Map<String, PropertyInfo> aggregateMetrics = new HashMap<String, PropertyInfo>();
-      // For every metric
-      for (Map.Entry<String, PropertyInfo> metricEntry : componentMetricInfo.entrySet())
{
-        // For each aggregate function id
-        for (String identifierToAdd : aggregateFunctionIdentifierMap.values()) {
-          String metricInfoKey = metricEntry.getKey() + identifierToAdd;
-          // This disallows metric key suffix of the form "._sum._sum" for
-          // the sake of avoiding duplicates
-          if (componentMetricInfo.containsKey(metricInfoKey)) {
-            continue;
-          }
-
-          PropertyInfo propertyInfo = metricEntry.getValue();
-          PropertyInfo metricInfoValue = new PropertyInfo(
-            propertyInfo.getPropertyId() + identifierToAdd,
-            propertyInfo.isTemporal(),
-            propertyInfo.isPointInTime());
-          metricInfoValue.setAmsHostMetric(propertyInfo.isAmsHostMetric());
-          metricInfoValue.setAmsId(propertyInfo.getAmsId());
-          metricInfoValue.setUnit(propertyInfo.getUnit());
-
-          aggregateMetrics.put(metricInfoKey, metricInfoValue);
-        }
-      }
-      componentMetricInfo.putAll(aggregateMetrics);
-    }
-
-    return componentMetrics;
-  }
-
   @Override
   protected String getHostName(Resource resource) {
     return null;

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
index d5af2b7..1011ded 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
@@ -39,13 +39,13 @@ import org.codehaus.jackson.map.AnnotationIntrospector;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.map.ObjectReader;
 import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.SocketTimeoutException;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -54,6 +54,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+
 import static org.apache.ambari.server.Role.HBASE_MASTER;
 import static org.apache.ambari.server.Role.HBASE_REGIONSERVER;
 import static org.apache.ambari.server.Role.METRICS_COLLECTOR;
@@ -67,14 +68,6 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider
{
   private final static ObjectReader timelineObjectReader;
   private static final String METRIC_REGEXP_PATTERN = "\\([^)]*\\)";
   private static final int COLLECTOR_DEFAULT_PORT = 6188;
-  protected static enum AGGREGATE_FUNCTION_IDENTIFIER {
-    SUM,
-    MIN,
-    MAX,
-    AVG
-  }
-  protected static final EnumMap<AGGREGATE_FUNCTION_IDENTIFIER, String> aggregateFunctionIdentifierMap
=
-    new EnumMap<AGGREGATE_FUNCTION_IDENTIFIER, String>(AGGREGATE_FUNCTION_IDENTIFIER.class);
 
   static {
     TIMELINE_APPID_MAP.put(HBASE_MASTER.name(), "HBASE");
@@ -87,11 +80,6 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider
{
     //noinspection deprecation
     mapper.getSerializationConfig().setSerializationInclusion(Inclusion.NON_NULL);
     timelineObjectReader = mapper.reader(TimelineMetrics.class);
-
-    aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.SUM, "._sum");
-    aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.MIN, "._min");
-    aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.MAX, "._max");
-    aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.AVG, "._avg");
   }
 
   public AMSPropertyProvider(Map<String, Map<String, PropertyInfo>> componentPropertyInfoMap,
@@ -121,28 +109,18 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider
{
    */
   @Override
   public Set<String> checkPropertyIds(Set<String> propertyIds) {
+    Set<String> supportedIds = new HashSet<String>();
     for (String propertyId : propertyIds) {
       if (propertyId.startsWith(ZERO_PADDING_PARAM)
-          || hasAggregateFunctionSuffix(propertyId)) {
-        propertyIds.remove(propertyId);
+          || PropertyHelper.hasAggregateFunctionSuffix(propertyId)) {
+        supportedIds.add(propertyId);
       }
     }
+    propertyIds.removeAll(supportedIds);
     return propertyIds;
   }
 
   /**
-   * Check if property ends with a trailing suffix
-   */
-  protected boolean hasAggregateFunctionSuffix(String propertyId) {
-    for (String aggregateFunctionId : aggregateFunctionIdentifierMap.values()) {
-      if (propertyId.endsWith(aggregateFunctionId)) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
    * The information required to make a single call to the Metrics service.
    */
   class MetricsRequest {

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
index febcf75..2dbff68 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.controller.metrics.timeline;
 import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
 import org.apache.ambari.server.controller.internal.PropertyInfo;
 import org.apache.ambari.server.controller.metrics.MetricHostProvider;
+import org.apache.ambari.server.controller.metrics.MetricsPaddingMethod;
 import org.apache.ambari.server.controller.metrics.MetricsPropertyProvider;
 import org.apache.ambari.server.controller.metrics.MetricsReportPropertyProvider;
 import org.apache.ambari.server.controller.spi.Predicate;
@@ -27,6 +28,7 @@ import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.TemporalInfo;
+import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.controller.utilities.StreamProvider;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
@@ -41,16 +43,21 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.SocketTimeoutException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.apache.ambari.server.controller.metrics.MetricsPaddingMethod.ZERO_PADDING_PARAM;
 import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService.TIMELINE_METRICS;
+import static org.apache.ambari.server.controller.utilities.PropertyHelper.updateMetricsWithAggregateFunctionSupport;
 
 public class AMSReportPropertyProvider extends MetricsReportPropertyProvider {
   private static ObjectMapper mapper;
   private final static ObjectReader timelineObjectReader;
+  private MetricsPaddingMethod metricsPaddingMethod;
 
   static {
     mapper = new ObjectMapper();
@@ -71,6 +78,22 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider
{
       hostProvider, clusterNamePropertyId);
   }
 
+  /**
+   * Support properties with aggregate functions and metrics padding method.
+   */
+  @Override
+  public Set<String> checkPropertyIds(Set<String> propertyIds) {
+    Set<String> supportedIds = new HashSet<String>();
+    for (String propertyId : propertyIds) {
+      if (propertyId.startsWith(ZERO_PADDING_PARAM)
+          || PropertyHelper.hasAggregateFunctionSuffix(propertyId)) {
+        supportedIds.add(propertyId);
+      }
+    }
+    propertyIds.removeAll(supportedIds);
+    return propertyIds;
+  }
+
   @Override
   public Set<Resource> populateResources(Set<Resource> resources,
                Request request, Predicate predicate) throws SystemException {
@@ -96,13 +119,27 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider
{
    * @throws SystemException if unable to populate the resource
    */
   private boolean populateResource(Resource resource, Request request, Predicate predicate)
-    throws SystemException {
+      throws SystemException {
 
     Set<String> propertyIds = getPropertyIds();
 
     if (propertyIds.isEmpty()) {
       return true;
     }
+
+    metricsPaddingMethod = DEFAULT_PADDING_METHOD;
+
+    Set<String> requestPropertyIds = request.getPropertyIds();
+    if (requestPropertyIds != null && !requestPropertyIds.isEmpty()) {
+      for (String propertyId : requestPropertyIds) {
+        if (propertyId.startsWith(ZERO_PADDING_PARAM)) {
+          String paddingStrategyStr = propertyId.substring(ZERO_PADDING_PARAM.length() +
1);
+          metricsPaddingMethod = new MetricsPaddingMethod(
+            MetricsPaddingMethod.PADDING_STRATEGY.valueOf(paddingStrategyStr));
+        }
+      }
+    }
+
     String clusterName = (String) resource.getPropertyValue(clusterNamePropertyId);
 
     // Check liveliness of host
@@ -165,6 +202,9 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider
{
 
         for (TimelineMetric metric : timelineMetrics.getMetrics()) {
           if (metric.getMetricName() != null && metric.getMetricValues() != null)
{
+            // Pad zeros or nulls if needed
+            metricsPaddingMethod.applyPaddingStrategy(metric, temporalInfo);
+
             String propertyId = propertyIdMap.get(metric.getMetricName());
             if (propertyId != null) {
               resource.setProperty(propertyId, getValue(metric, true));

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java
index cde8581..cefe953 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java
@@ -20,8 +20,10 @@ package org.apache.ambari.server.controller.utilities;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Matcher;
@@ -34,6 +36,7 @@ import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.spi.SortRequest;
 import org.apache.ambari.server.controller.spi.TemporalInfo;
+import org.apache.commons.lang.StringUtils;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.type.TypeReference;
 
@@ -49,6 +52,15 @@ public class PropertyHelper {
   private static final String KEY_PROPERTIES_FILE = "key_properties.json";
   private static final char EXTERNAL_PATH_SEP = '/';
 
+  /**
+   * Aggregate functions implicitly supported by the Metrics Service
+   */
+  public static final List<String> AGGREGATE_FUNCTION_IDENTIFIERS =
+    Arrays.asList("._sum", "._max", "._min", "._avg");
+
+  private static final List<Resource.InternalType> REPORT_METRIC_RESOURCES =
+    Arrays.asList(Resource.InternalType.Cluster, Resource.InternalType.Host);
+
   private static final Map<Resource.InternalType, Set<String>> PROPERTY_IDS =
readPropertyIds(PROPERTIES_FILE);
   private static final Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>>
JMX_PROPERTY_IDS = readPropertyProviderIds(JMX_PROPERTIES_FILE);
   private static final Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>>
GANGLIA_PROPERTY_IDS = readPropertyProviderIds(GANGLIA_PROPERTIES_FILE);
@@ -439,6 +451,9 @@ public class PropertyHelper {
           }
           componentMetrics.put(componentEntry.getKey(), metrics);
         }
+        if (REPORT_METRIC_RESOURCES.contains(resourceEntry.getKey())) {
+          updateMetricsWithAggregateFunctionSupport(componentMetrics);
+        }
         resourceMetrics.put(resourceEntry.getKey(), componentMetrics);
       }
       return resourceMetrics;
@@ -550,4 +565,65 @@ public class PropertyHelper {
       return unit;
     }
   }
+
+  /**
+   * This method adds supported propertyInfo for component metrics with
+   * aggregate function ids. API calls with multiple aggregate functions
+   * applied to a single metric need this support.
+   *
+   * Currently this support is added only for Component & Report metrics.
+   * This can be easily extended by making the call from the appropriate
+   * sub class of: @AMSPropertyProvider.
+   *
+   */
+  public static void updateMetricsWithAggregateFunctionSupport(Map<String, Map<String,
PropertyInfo>> metrics) {
+
+    if (metrics == null || metrics.isEmpty()) {
+      return;
+    }
+
+    // For every component
+    for (Map.Entry<String, Map<String, PropertyInfo>> metricInfoEntry : metrics.entrySet())
{
+      Map<String, PropertyInfo> metricInfo = metricInfoEntry.getValue();
+
+      Map<String, PropertyInfo> aggregateMetrics = new HashMap<String, PropertyInfo>();
+      // For every metric
+      for (Map.Entry<String, PropertyInfo> metricEntry : metricInfo.entrySet()) {
+        // For each aggregate function id
+        for (String identifierToAdd : AGGREGATE_FUNCTION_IDENTIFIERS) {
+          String metricInfoKey = metricEntry.getKey() + identifierToAdd;
+          // This disallows metric key suffix of the form "._sum._sum" for
+          // the sake of avoiding duplicates
+          if (metricInfo.containsKey(metricInfoKey)) {
+            continue;
+          }
+
+          PropertyInfo propertyInfo = metricEntry.getValue();
+          PropertyInfo metricInfoValue = new PropertyInfo(
+            propertyInfo.getPropertyId() + identifierToAdd,
+            propertyInfo.isTemporal(),
+            propertyInfo.isPointInTime());
+          metricInfoValue.setAmsHostMetric(propertyInfo.isAmsHostMetric());
+          metricInfoValue.setAmsId(!StringUtils.isEmpty(propertyInfo.getAmsId()) ?
+            propertyInfo.getAmsId() + identifierToAdd : null);
+          metricInfoValue.setUnit(propertyInfo.getUnit());
+
+          aggregateMetrics.put(metricInfoKey, metricInfoValue);
+        }
+      }
+      metricInfo.putAll(aggregateMetrics);
+    }
+  }
+
+  /**
+   * Check if property ends with a trailing suffix of function id.
+   */
+  public static boolean hasAggregateFunctionSuffix(String propertyId) {
+    for (String aggregateFunctionId : AGGREGATE_FUNCTION_IDENTIFIERS) {
+      if (propertyId.endsWith(aggregateFunctionId)) {
+        return true;
+      }
+    }
+    return false;
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
index 81c53cc..923f2f7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
@@ -47,6 +47,7 @@ import org.powermock.modules.junit4.PowerMockRunner;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -444,6 +445,7 @@ public class AMSPropertyProviderTest {
 
     Map<String, Map<String, PropertyInfo>> propertyIds =
       PropertyHelper.getMetricPropertyIds(Resource.Type.Component);
+    PropertyHelper.updateMetricsWithAggregateFunctionSupport(propertyIds);
 
     AMSComponentPropertyProvider propertyProvider = new AMSComponentPropertyProvider(
       propertyIds,

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java
index 8fba0e7..c0ce419 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java
@@ -42,6 +42,7 @@ public class AMSReportPropertyProviderTest {
   private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("Clusters",
"cluster_name");
   private static final String FILE_PATH_PREFIX = "ams" + File.separator;
   private static final String SINGLE_HOST_METRICS_FILE_PATH = FILE_PATH_PREFIX + "single_host_metric.json";
+  private static final String AGGREGATE_CLUSTER_METRICS_FILE_PATH = FILE_PATH_PREFIX + "aggregate_cluster_metrics.json";
 
   @Test
   public void testPopulateResources() throws Exception {
@@ -59,13 +60,13 @@ public class AMSReportPropertyProviderTest {
         sslConfiguration,
         metricHostProvider,
         CLUSTER_NAME_PROPERTY_ID
-      );
+    );
 
     String propertyId = PropertyHelper.getPropertyId("metrics/cpu", "User");
     Resource resource = new ResourceImpl(Resource.Type.Cluster);
     resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1");
     Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
-    temporalInfoMap.put(propertyId, new TemporalInfoImpl(1416528819369L, 1416528819569L,
1L));
+    temporalInfoMap.put(propertyId, new TemporalInfoImpl(1416445244701L, 1416445244901L,
1L));
     Request request = PropertyHelper.getReadRequest(
       Collections.singleton(propertyId), temporalInfoMap);
     Set<Resource> resources =
@@ -77,10 +78,51 @@ public class AMSReportPropertyProviderTest {
     URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 8188);
     uriBuilder.addParameter("metricNames", "cpu_user");
     uriBuilder.addParameter("appId", "HOST");
-    uriBuilder.addParameter("startTime", "1416528819369");
-    uriBuilder.addParameter("endTime", "1416528819569");
+    uriBuilder.addParameter("startTime", "1416445244701");
+    uriBuilder.addParameter("endTime", "1416445244901");
     Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec());
     Number[][] val = (Number[][]) res.getPropertyValue("metrics/cpu/User");
     Assert.assertEquals(111, val.length);
   }
+
+  @Test
+  public void testPopulateResourceWithAggregateFunction() throws Exception {
+    TestStreamProvider streamProvider = new TestStreamProvider(AGGREGATE_CLUSTER_METRICS_FILE_PATH);
+    TestMetricHostProvider metricHostProvider = new TestMetricHostProvider();
+    ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class);
+
+    Map<String, Map<String, PropertyInfo>> propertyIds =
+      PropertyHelper.getMetricPropertyIds(Resource.Type.Cluster);
+
+    AMSReportPropertyProvider propertyProvider =
+      new AMSReportPropertyProvider(
+        propertyIds,
+        streamProvider,
+        sslConfiguration,
+        metricHostProvider,
+        CLUSTER_NAME_PROPERTY_ID
+    );
+
+    String propertyId = PropertyHelper.getPropertyId("metrics/cpu", "User._sum");
+    Resource resource = new ResourceImpl(Resource.Type.Cluster);
+    resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1");
+    Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+    temporalInfoMap.put(propertyId, new TemporalInfoImpl(1432033256912L, 1432033257912L,
1L));
+    Request request = PropertyHelper.getReadRequest(
+      Collections.singleton(propertyId), temporalInfoMap);
+    Set<Resource> resources =
+      propertyProvider.populateResources(Collections.singleton(resource), request, null);
+    Assert.assertEquals(1, resources.size());
+    Resource res = resources.iterator().next();
+    Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next());
+    Assert.assertNotNull(properties);
+    URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 8188);
+    uriBuilder.addParameter("metricNames", "cpu_user._sum");
+    uriBuilder.addParameter("appId", "HOST");
+    uriBuilder.addParameter("startTime", "1432033256912");
+    uriBuilder.addParameter("endTime", "1432033257912");
+    Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec());
+    Number[][] val = (Number[][]) res.getPropertyValue("metrics/cpu/User._sum");
+    Assert.assertEquals(91, val.length);
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json b/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json
new file mode 100644
index 0000000..d3987cf
--- /dev/null
+++ b/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json
@@ -0,0 +1,100 @@
+{"metrics": [
+  {
+    "timestamp": 1432033257912,
+    "metricname": "cpu_user._sum",
+    "appid": "HOST",
+    "starttime": 1432033257912,
+    "metrics": {
+      "1432033257912": 10347.5,
+      "1432033287912": 11821.25,
+      "1432033317912": 10200.0,
+      "1432033347912": 12147.5,
+      "1432033377912": 10530.3125,
+      "1432033407912": 12243.75,
+      "1432033437912": 10220.0,
+      "1432033467912": 12512.5,
+      "1432033497912": 10302.5,
+      "1432033527912": 12165.625,
+      "1432033557912": 10200.0,
+      "1432033587912": 12478.75,
+      "1432033617912": 10299.6875,
+      "1432033647912": 12532.5,
+      "1432033677912": 10180.0,
+      "1432033707912": 12047.5,
+      "1432033737912": 10340.9375,
+      "1432033767912": 11628.75,
+      "1432033797912": 10140.0,
+      "1432033827912": 12151.25,
+      "1432033857912": 10375.3125,
+      "1432033887912": 11683.125,
+      "1432033917912": 10200.0,
+      "1432033947912": 12177.5,
+      "1432033977912": 10372.8125,
+      "1432034007912": 11806.25,
+      "1432034037912": 10180.0,
+      "1432034067912": 11233.75,
+      "1432034097912": 10389.0625,
+      "1432034127912": 11730.0,
+      "1432034157912": 10240.0,
+      "1432034187912": 11493.125,
+      "1432034217912": 10360.0,
+      "1432034247912": 11346.25,
+      "1432034277912": 10240.0,
+      "1432034307912": 11329.375,
+      "1432034337912": 10391.5625,
+      "1432034367912": 11148.125,
+      "1432034397912": 10240.0,
+      "1432034427912": 11193.125,
+      "1432034457912": 10399.6875,
+      "1432034487912": 11188.125,
+      "1432034517912": 10280.0,
+      "1432034547912": 11183.125,
+      "1432034577912": 10455.625,
+      "1432034607912": 12975.625,
+      "1432034637912": 10280.0,
+      "1432034667912": 13041.25,
+      "1432034697912": 10297.5,
+      "1432034727912": 13141.875,
+      "1432034757912": 10260.0,
+      "1432034787912": 12880.0,
+      "1432034817912": 10288.75,
+      "1432034847912": 13072.5,
+      "1432034877912": 10240.0,
+      "1432034907912": 13304.375,
+      "1432034937912": 10306.25,
+      "1432034967912": 13011.25,
+      "1432034997912": 10270.0,
+      "1432035027912": 12463.75,
+      "1432035057912": 10339.6875,
+      "1432035087912": 12656.875,
+      "1432035117912": 10270.0,
+      "1432035147912": 12428.125,
+      "1432035177912": 10347.8125,
+      "1432035207912": 12195.0,
+      "1432035237912": 10320.0,
+      "1432035267912": 11796.25,
+      "1432035297912": 10418.4375,
+      "1432035327912": 11716.25,
+      "1432035357912": 10390.0,
+      "1432035387912": 11400.0,
+      "1432035417912": 10463.125,
+      "1432035447912": 11177.5,
+      "1432035477912": 10390.0,
+      "1432035507912": 11073.125,
+      "1432035537912": 10460.9375,
+      "1432035567912": 10285.0,
+      "1432035597912": 10350.0,
+      "1432035627912": 10968.75,
+      "1432035657912": 10440.3125,
+      "1432035687912": 10471.25,
+      "1432035717912": 10380.0,
+      "1432035747912": 10379.375,
+      "1432035777912": 10483.4375,
+      "1432035807912": 10336.875,
+      "1432035837912": 10290.0,
+      "1432035867912": 10173.75,
+      "1432035897912": 10477.8125,
+      "1432035927912": 10346.875
+    }
+  }
+]}
\ No newline at end of file


Mime
View raw message