ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-14007. Fix precision issues in AMS. (Aravindan Vijayan via swagle)
Date Mon, 23 Nov 2015 05:28:35 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 8799a8b46 -> 718ca1441


AMBARI-14007. Fix precision issues in AMS. (Aravindan Vijayan via swagle)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/718ca144
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/718ca144
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/718ca144

Branch: refs/heads/branch-2.1
Commit: 718ca144160c69de921d1a577a597ec2999e8211
Parents: 8799a8b
Author: Siddharth Wagle <swagle@hortonworks.com>
Authored: Sun Nov 22 21:28:24 2015 -0800
Committer: Siddharth Wagle <swagle@hortonworks.com>
Committed: Sun Nov 22 21:28:24 2015 -0800

----------------------------------------------------------------------
 .../metrics2/sink/timeline/Precision.java       |  16 +++
 .../PrecisionLimitExceededException.java        |  36 ++++++
 .../metrics/timeline/PhoenixHBaseAccessor.java  |   8 +-
 .../timeline/query/PhoenixTransactSQL.java      |   5 +-
 .../webapp/TimelineWebServices.java             |   3 +
 .../metrics/MetricsPaddingMethod.java           |   5 +
 .../metrics/timeline/AMSPropertyProvider.java   |   5 +-
 .../timeline/AMSReportPropertyProvider.java     |   5 +-
 .../metrics/timeline/MetricsRequestHelper.java  |  91 +++++++++++---
 .../cache/TimelineMetricCacheEntryFactory.java  |   9 +-
 .../server/upgrade/UpgradeCatalog213.java       |   4 +-
 .../0.1.0/configuration/ams-site.xml            |   4 +-
 .../timeline/MetricsRequestHelperTest.java      | 118 +++++++++++++++++++
 .../timeline/cache/TimelineMetricCacheTest.java |  10 +-
 .../server/upgrade/UpgradeCatalog213Test.java   |   2 +-
 15 files changed, 280 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java
b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java
index 900e5dd..31044cc 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/Precision.java
@@ -60,4 +60,20 @@ public enum Precision {
       return Precision.SECONDS;
     }
   }
+
+  public static Precision getHigherPrecision(Precision precision) {
+
+    if (precision == null)
+      return null;
+
+    if (precision.equals(Precision.SECONDS)) {
+      return Precision.MINUTES;
+    } else if (precision.equals(Precision.MINUTES)) {
+      return Precision.HOURS;
+    } else if (precision.equals(Precision.HOURS)) {
+      return Precision.DAYS;
+    } else {
+      return null;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/PrecisionLimitExceededException.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/PrecisionLimitExceededException.java
b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/PrecisionLimitExceededException.java
new file mode 100644
index 0000000..962a071
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/PrecisionLimitExceededException.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics2.sink.timeline;
+
+public class PrecisionLimitExceededException extends IllegalArgumentException {
+
+  private static final long serialVersionUID = 1L;
+
+  public PrecisionLimitExceededException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public PrecisionLimitExceededException(String message) {
+    super(message);
+  }
+
+  public PrecisionLimitExceededException(Throwable cause) {
+    super(cause);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index b5998ee..4dc1a95 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -105,10 +105,12 @@ public class PhoenixHBaseAccessor {
   // cluster and host levels.
   static final long DEFAULT_OUT_OF_BAND_TIME_ALLOWANCE = 300000;
   /**
-   * 8 metrics * 60minutes * 24hours => Reasonable upper bound on the limit such that
our Precision calculation for a given time range makes sense.
+   * 22 metrics for 2hours in SECONDS (10 second data)
+   * => Reasonable upper bound on the limit such that our Precision calculation for a
given time range makes sense.
    */
-  private static final int METRICS_PER_MINUTE = 8;
-  public static int RESULTSET_LIMIT = (int)TimeUnit.HOURS.toMinutes(24) * METRICS_PER_MINUTE;
+  private static final int METRICS_PER_MINUTE = 22;
+  private static final int POINTS_PER_MINUTE = 6;
+  public static int RESULTSET_LIMIT = (int)TimeUnit.HOURS.toMinutes(2) * METRICS_PER_MINUTE
* POINTS_PER_MINUTE ;
 
   private static final TimelineMetricReadHelper TIMELINE_METRIC_READ_HELPER = new TimelineMetricReadHelper();
   private static ObjectMapper mapper = new ObjectMapper();

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
index 8962021..eaf2efe 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
 import org.apache.hadoop.metrics2.sink.timeline.Precision;
 
@@ -437,7 +438,7 @@ public class PhoenixTransactSQL {
         rowsPerMetric = TimeUnit.MILLISECONDS.toHours(range);
         break;
       case MINUTES:
-        rowsPerMetric = TimeUnit.MILLISECONDS.toMinutes(range);
+        rowsPerMetric = TimeUnit.MILLISECONDS.toMinutes(range)/2; //2 minute data in METRIC_AGGREGATE_MINUTE
table.
         break;
       default:
         rowsPerMetric = TimeUnit.MILLISECONDS.toSeconds(range)/10; //10 second data in METRIC_AGGREGATE
table
@@ -445,7 +446,7 @@ public class PhoenixTransactSQL {
 
     long totalRowsRequested = rowsPerMetric * condition.getMetricNames().size();
     if (totalRowsRequested > PhoenixHBaseAccessor.RESULTSET_LIMIT) {
-      throw new IllegalArgumentException("Requested precision (" + precision + ") for given
time range causes " +
+      throw new PrecisionLimitExceededException("Requested precision (" + precision + ")
for given time range causes " +
         "result set size of " + totalRowsRequested + ", which exceeds the limit - "
         + PhoenixHBaseAccessor.RESULTSET_LIMIT + ". Please request higher precision.");
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
index a13086a..51535b2 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
@@ -24,6 +24,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
@@ -378,6 +379,8 @@ public class TimelineWebServices {
     } catch (Precision.PrecisionFormatException pfe) {
       throw new BadRequestException("precision should be seconds, minutes " +
         "or hours");
+    } catch (PrecisionLimitExceededException iae) {
+      throw new PrecisionLimitExceededException(iae.getMessage());
     } catch (IllegalArgumentException iae) {
       throw new BadRequestException(iae.getMessage());
     } catch (SQLException sql) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
index 930cb91..00dbaa8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.controller.metrics;
 import org.apache.ambari.server.controller.spi.TemporalInfo;
 import org.apache.hadoop.metrics2.sink.timeline.Precision;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+
 import java.util.Iterator;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
@@ -55,6 +56,10 @@ public class MetricsPaddingMethod {
 
     TreeMap<Long, Double> values = metric.getMetricValues();
 
+    if (values==null || values.isEmpty()) {
+      return;
+    }
+
     long intervalStartTime = longToMillis(temporalInfo.getStartTime());
     long intervalEndTime = longToMillis(temporalInfo.getEndTime());
     long dataStartTime = longToMillis(values.firstKey());

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
index 730c6ec..b9f54db 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
@@ -172,7 +172,10 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider
{
         return metricCache.getAppTimelineMetricsFromCache(metricCacheKey);
       }
 
-      return requestHelper.fetchTimelineMetrics(metricCacheKey.getSpec());
+      Long startTime = (metricCacheKey.getTemporalInfo() != null) ? metricCacheKey.getTemporalInfo().getStartTimeMillis():null;
+      Long endTime = (metricCacheKey.getTemporalInfo() != null) ? metricCacheKey.getTemporalInfo().getEndTimeMillis():null;
+
+      return requestHelper.fetchTimelineMetrics(uriBuilder,startTime, endTime);
     }
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
index 6bdc605..e40a5a5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
@@ -33,7 +33,6 @@ import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.TemporalInfo;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
-import org.apache.ambari.server.controller.utilities.StreamProvider;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
 import org.apache.http.client.utils.URIBuilder;
@@ -209,7 +208,9 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider
{
         timelineMetrics = metricCache.getAppTimelineMetricsFromCache(metricCacheKey);
       } else {
         try {
-          timelineMetrics = requestHelper.fetchTimelineMetrics(uriBuilder.toString());
+          timelineMetrics = requestHelper.fetchTimelineMetrics(uriBuilder,
+            temporalInfo.getStartTimeMillis(),
+            temporalInfo.getEndTimeMillis());
         } catch (IOException e) {
           timelineMetrics = null;
         }

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java
index 0eff41c..94014f8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,12 +17,13 @@
  */
 package org.apache.ambari.server.controller.metrics.timeline;
 
-import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
 import org.apache.ambari.server.controller.internal.URLStreamProvider;
-import org.apache.ambari.server.controller.utilities.StreamProvider;
 import org.apache.commons.httpclient.HttpStatus;
+import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.http.NameValuePair;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.http.client.utils.URIBuilder;
 import org.codehaus.jackson.map.AnnotationIntrospector;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.map.ObjectReader;
@@ -38,8 +39,11 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.SocketTimeoutException;
+import java.net.URISyntaxException;
+import java.util.Collections;
 import java.util.Date;
-import java.util.Map;
+import java.util.Iterator;
+import java.util.List;
 
 /**
  * Helper class to call AMS backend that is utilized by @AMSPropertyProvider
@@ -64,22 +68,36 @@ public class MetricsRequestHelper {
     this.streamProvider = streamProvider;
   }
 
-  public TimelineMetrics fetchTimelineMetrics(String spec) throws IOException {
-    LOG.debug("Metrics request url = " + spec);
+  public TimelineMetrics fetchTimelineMetrics(URIBuilder uriBuilder, Long startTime, Long
endTime) throws IOException {
+    LOG.debug("Metrics request url = " + uriBuilder.toString());
     BufferedReader reader = null;
     TimelineMetrics timelineMetrics = null;
     try {
 
-      HttpURLConnection connection = streamProvider.processURL(spec, HttpMethod.GET, (String)null,
null);
-      if (connection.getResponseCode() == HttpStatus.SC_BAD_REQUEST) {
-        InputStream errorStream = connection.getErrorStream();
-        reader = new BufferedReader(new InputStreamReader(errorStream));
-        throw new IOException(reader.readLine());
+      HttpURLConnection connection = streamProvider.processURL(uriBuilder.toString(), HttpMethod.GET,
+        (String) null, Collections.<String, List<String>>emptyMap());
+
+      if (!checkConnectionForPrecisionException(connection)) {
+        //Try one more time with higher precision
+        String higherPrecision = getHigherPrecision(uriBuilder, startTime, endTime);
+        if (higherPrecision != null) {
+          LOG.debug("Requesting metrics with higher precision : " + higherPrecision);
+          uriBuilder.setParameter("precision", higherPrecision);
+          String newSpec = uriBuilder.toString();
+          connection = streamProvider.processURL(newSpec, HttpMethod.GET, (String) null,
+            Collections.<String, List<String>>emptyMap());
+          if (!checkConnectionForPrecisionException(connection)) {
+            throw new IOException("Encountered Precision exception : Higher precision request
also failed.");
+          }
+        } else {
+          throw new IOException("Encountered Precision exception : Unable to request higher
precision");
+        }
       }
 
       InputStream inputStream = connection.getInputStream();
       reader = new BufferedReader(new InputStreamReader(inputStream));
       timelineMetrics = timelineObjectReader.readValue(reader);
+
       if (LOG.isTraceEnabled()) {
         for (TimelineMetric metric : timelineMetrics.getMetrics()) {
           LOG.trace("metric: " + metric.getMetricName() +
@@ -103,7 +121,12 @@ public class MetricsRequestHelper {
         LOG.error(errorMsg);
         throw io;
       }
-
+    } catch (URISyntaxException e) {
+      String errorMsg = "Error getting timeline metrics : " + e.getMessage();
+      LOG.error(errorMsg);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(errorMsg, e);
+      }
     } finally {
       if (reader != null) {
         try {
@@ -111,9 +134,9 @@ public class MetricsRequestHelper {
         } catch (IOException e) {
           if (LOG.isWarnEnabled()) {
             if (LOG.isDebugEnabled()) {
-              LOG.warn("Unable to close http input stream : spec=" + spec, e);
+              LOG.warn("Unable to close http input stream : spec=" + uriBuilder.toString(),
e);
             } else {
-              LOG.warn("Unable to close http input stream : spec=" + spec);
+              LOG.warn("Unable to close http input stream : spec=" + uriBuilder.toString());
             }
           }
         }
@@ -121,4 +144,40 @@ public class MetricsRequestHelper {
     }
     return timelineMetrics;
   }
+
+  private boolean checkConnectionForPrecisionException(HttpURLConnection connection)
+    throws IOException, URISyntaxException {
+
+    if (connection != null && connection.getResponseCode() == HttpStatus.SC_BAD_REQUEST)
{
+      InputStream errorStream = connection.getErrorStream();
+      BufferedReader reader = new BufferedReader(new InputStreamReader(errorStream));
+      String errorMessage = reader.readLine();
+      if (errorMessage != null && errorMessage.contains("PrecisionLimitExceededException"))
{
+        LOG.debug("Encountered Precision exception while requesting metrics : " + errorMessage);
+        return false;
+      } else {
+        throw new IOException(errorMessage);
+      }
+    }
+    return true;
+  }
+
+  private String getHigherPrecision(URIBuilder uriBuilder, Long startTime, Long endTime)
throws URISyntaxException {
+
+    Precision currentPrecision = null;
+    List<NameValuePair> queryParams = uriBuilder.getQueryParams();
+    for (Iterator<NameValuePair> it = queryParams.iterator(); it.hasNext();) {
+      NameValuePair nvp = it.next();
+      if (nvp.getName().equals("precision")) {
+        currentPrecision = Precision.getPrecision(nvp.getValue());
+      }
+    }
+    if (currentPrecision == null && startTime != null && endTime != null)
{
+      currentPrecision = Precision.getPrecision(startTime, endTime);
+    }
+    Precision higherPrecision = Precision.getHigherPrecision(currentPrecision);
+
+    return (higherPrecision != null) ? higherPrecision.toString() : null;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java
index 9100afd..ed0f878 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheEntryFactory.java
@@ -51,8 +51,6 @@ public class TimelineMetricCacheEntryFactory implements UpdatingCacheEntryFactor
   private MetricsRequestHelper requestHelperForGets;
   private MetricsRequestHelper requestHelperForUpdates;
   private final Long BUFFER_TIME_DIFF_CATCHUP_INTERVAL;
-  public static final long HOUR = 3600000; // 1 hour
-  public static final long DAY = 86400000; // 1 day
 
   @Inject
   public TimelineMetricCacheEntryFactory(Configuration configuration) {
@@ -87,7 +85,10 @@ public class TimelineMetricCacheEntryFactory implements UpdatingCacheEntryFactor
 
     TimelineMetrics timelineMetrics = null;
     try {
-      timelineMetrics = requestHelperForGets.fetchTimelineMetrics(metricCacheKey.getSpec());
+      URIBuilder uriBuilder = new URIBuilder(metricCacheKey.getSpec());
+      timelineMetrics = requestHelperForGets.fetchTimelineMetrics(uriBuilder,
+        metricCacheKey.getTemporalInfo().getStartTimeMillis(),
+        metricCacheKey.getTemporalInfo().getEndTimeMillis());
     } catch (IOException io) {
       LOG.debug("Caught IOException on fetching metrics. " + io.getMessage());
     }
@@ -174,7 +175,7 @@ public class TimelineMetricCacheEntryFactory implements UpdatingCacheEntryFactor
       uriBuilder.setParameter("precision",requestedPrecision.toString());
 
       try {
-        TimelineMetrics newTimeSeries = requestHelperForUpdates.fetchTimelineMetrics(uriBuilder.toString());
+        TimelineMetrics newTimeSeries = requestHelperForUpdates.fetchTimelineMetrics(uriBuilder,
newStartTime, newEndTime);
 
         // Update existing time series with new values
         updateTimelineMetricsInCache(newTimeSeries, existingMetrics,

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
index cc1838b..3165722 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java
@@ -915,8 +915,8 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
           if (amsSite != null) {
             Map<String, String> newProperties = new HashMap<>();
 
-            //Changed AMS result set limit from 5760 to 11520.
-            newProperties.put("timeline.metrics.service.default.result.limit", String.valueOf(11520));
+            //Changed AMS result set limit from 5760 to 15840.
+            newProperties.put("timeline.metrics.service.default.result.limit", String.valueOf(15840));
 
             //Interval
             newProperties.put("timeline.metrics.cluster.aggregator.second.interval", String.valueOf(120));

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index be698fc..04b7806 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -357,10 +357,10 @@
   </property>
   <property>
     <name>timeline.metrics.service.default.result.limit</name>
-    <value>11520</value>
+    <value>15840</value>
     <description>
       Max result limit on number of rows returned. Calculated as follows:
-      8 aggregate metrics/min * 60 * 24 : Retrieve MINUTE data for 24 hours.
+      22 aggregate metrics/min * 2 * 60 * 6 : Retrieve 10 SECOND data for 2 hours.
     </description>
     <display-name>Metrics service default result limit</display-name>
     <value-attributes>

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java
new file mode 100644
index 0000000..52792b1
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelperTest.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.metrics.timeline;
+
+import org.apache.ambari.server.controller.internal.URLStreamProvider;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.http.client.utils.URIBuilder;
+import org.codehaus.jackson.map.AnnotationIntrospector;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectWriter;
+import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+import org.easymock.EasyMock;
+import org.easymock.EasyMockSupport;
+import org.junit.Test;
+
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.isNull;
+import static org.easymock.EasyMock.replay;
+
+public class MetricsRequestHelperTest {
+
+  @Test
+  public void testFetchTimelineMetrics() throws Exception {
+
+    EasyMockSupport easyMockSupport = new EasyMockSupport();
+    final long now = System.currentTimeMillis();
+    TimelineMetrics metrics = new TimelineMetrics();
+    TimelineMetric timelineMetric = new TimelineMetric();
+    timelineMetric.setMetricName("cpu_user");
+    timelineMetric.setAppId("app1");
+    TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+    metricValues.put(now + 100, 1.0);
+    metricValues.put(now + 200, 2.0);
+    metricValues.put(now + 300, 3.0);
+    timelineMetric.setMetricValues(metricValues);
+    metrics.getMetrics().add(timelineMetric);
+
+    ObjectMapper mapper = new ObjectMapper();
+    AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
+    mapper.setAnnotationIntrospector(introspector);
+    ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
+    String metricsResponse = writer.writeValueAsString(metrics);
+
+    InputStream inputStream = IOUtils.toInputStream(metricsResponse);
+    HttpURLConnection httpURLConnectionMock = createMock(HttpURLConnection.class);
+    expect(httpURLConnectionMock.getInputStream()).andReturn(inputStream).once();
+    expect(httpURLConnectionMock.getResponseCode()).andReturn(HttpStatus.SC_OK).once();
+
+    URLStreamProvider urlStreamProviderMock = createMock(URLStreamProvider.class);
+    expect(urlStreamProviderMock.processURL(EasyMock.isA(String.class), EasyMock.isA(String.class),
+      isNull(String.class), EasyMock.isA(Map.class))).andReturn(httpURLConnectionMock).once();
+
+    replay(httpURLConnectionMock, urlStreamProviderMock);
+
+    //Case 1 : No error.
+    String randomSpec = "http://localhost:6188/ws/v1/timeline/metrics?metricNames=cpu_wio&hostname=host1&appId=HOST"
+
+      "&startTime=1447912834&endTime=1447990034&precision=SECONDS";
+    MetricsRequestHelper metricsRequestHelper = new MetricsRequestHelper(urlStreamProviderMock);
+    metricsRequestHelper.fetchTimelineMetrics(new URIBuilder(randomSpec), now, now + 300);
+
+    easyMockSupport.verifyAll();
+
+    //Case 2 : Precision Error returned first time.
+    String metricsPrecisionErrorResponse = "{\"exception\": \"PrecisionLimitExceededException\",\n"
+
+      "\"message\": \"Requested precision (SECONDS) for given time range causes result set
size of 169840, " +
+      "which exceeds the limit - 15840. Please request higher precision.\",\n" +
+      "\"javaClassName\": \"org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException\"\n"
+
+      "}";
+
+    InputStream errorStream = IOUtils.toInputStream(metricsPrecisionErrorResponse);
+    inputStream = IOUtils.toInputStream(metricsResponse); //Reloading stream.
+
+    httpURLConnectionMock = createMock(HttpURLConnection.class);
+    expect(httpURLConnectionMock.getErrorStream()).andReturn(errorStream).once();
+    expect(httpURLConnectionMock.getInputStream()).andReturn(inputStream).once();
+    expect(httpURLConnectionMock.getResponseCode())
+      .andReturn(HttpStatus.SC_BAD_REQUEST).once()
+      .andReturn(HttpStatus.SC_OK).once();
+
+    urlStreamProviderMock = createMock(URLStreamProvider.class);
+    expect(urlStreamProviderMock.processURL(EasyMock.isA(String.class), EasyMock.isA(String.class),
+      isNull(String.class), EasyMock.isA(Map.class))).andReturn(httpURLConnectionMock).times(2);
+
+    replay(httpURLConnectionMock, urlStreamProviderMock);
+
+    metricsRequestHelper = new MetricsRequestHelper(urlStreamProviderMock);
+    metricsRequestHelper.fetchTimelineMetrics(new URIBuilder(randomSpec), now, now + 300);
+
+    easyMockSupport.verifyAll();
+
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
index 53661fa..3b3c131 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
@@ -17,10 +17,6 @@
  */
 package org.apache.ambari.server.controller.metrics.timeline.cache;
 
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
 import junit.framework.Assert;
 import net.sf.ehcache.Cache;
 import net.sf.ehcache.CacheManager;
@@ -34,8 +30,6 @@ import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.internal.TemporalInfoImpl;
 import org.apache.ambari.server.controller.metrics.timeline.MetricsRequestHelper;
 import org.apache.ambari.server.controller.spi.TemporalInfo;
-import org.apache.ambari.server.state.stack.OsFamily;
-import org.apache.hadoop.metrics2.sink.timeline.Precision;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
 import org.apache.http.client.utils.URIBuilder;
@@ -55,8 +49,8 @@ import java.util.Map;
 import java.util.TreeMap;
 
 import static org.apache.ambari.server.controller.metrics.timeline.cache.TimelineMetricCacheProvider.TIMELINE_METRIC_CACHE_INSTANCE_NAME;
+import static org.easymock.EasyMock.anyLong;
 import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.capture;
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.createMockBuilder;
 import static org.easymock.EasyMock.createNiceMock;
@@ -478,7 +472,7 @@ public class TimelineMetricCacheTest {
     newKey.setSpec("");
 
     MetricsRequestHelper metricsRequestHelperForGets = createMock(MetricsRequestHelper.class);
-    expect(metricsRequestHelperForGets.fetchTimelineMetrics(EasyMock.isA(String.class)))
+    expect(metricsRequestHelperForGets.fetchTimelineMetrics(EasyMock.isA(URIBuilder.class),
anyLong(), anyLong()))
       .andReturn(metrics).andReturn(newMetrics);
     replay(metricsRequestHelperForGets);
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/718ca144/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
index fa3e365..8fc81da 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java
@@ -761,7 +761,7 @@ public class UpgradeCatalog213Test {
     };
     Map<String, String> newPropertiesAmsSite = new HashMap<String, String>()
{
       {
-        put("timeline.metrics.service.default.result.limit", String.valueOf(11520));
+        put("timeline.metrics.service.default.result.limit", String.valueOf(15840));
         put("timeline.metrics.cluster.aggregator.second.interval", String.valueOf(120));
         put("timeline.metrics.cluster.aggregator.minute.interval", String.valueOf(300));
         put("timeline.metrics.host.aggregator.minute.interval", String.valueOf(300));


Mime
View raw message