ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-5707. Aggregator exceptions due to misconfiguration.
Date Fri, 21 Nov 2014 18:16:12 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-metrics-dev dd066b945 -> 063335855


AMBARI-5707. Aggregator exceptions due to misconfiguration.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/06333585
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/06333585
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/06333585

Branch: refs/heads/branch-metrics-dev
Commit: 0633358555468d2c8c5d6d8d30902a82a5036247
Parents: dd066b9
Author: Siddharth Wagle <swagle@hortonworks.com>
Authored: Fri Nov 21 10:15:58 2014 -0800
Committer: Siddharth Wagle <swagle@hortonworks.com>
Committed: Fri Nov 21 10:15:58 2014 -0800

----------------------------------------------------------------------
 .../timeline/AbstractTimelineAggregator.java     | 19 ++++++++++---------
 .../timeline/TimelineMetricAggregatorHourly.java | 11 ++++++-----
 .../timeline/TimelineMetricAggregatorMinute.java | 10 ++++++----
 .../TimelineMetricClusterAggregator.java         | 19 +++++++++++--------
 .../TimelineMetricClusterAggregatorHourly.java   | 19 ++++++++++---------
 .../timeline/TimelineMetricConfiguration.java    |  3 ++-
 6 files changed, 45 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java
b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java
index 6bd6507..43ec648 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java
@@ -33,6 +33,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Date;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
   .timeline.TimelineMetricConfiguration.AGGREGATOR_CHECKPOINT_DELAY;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
@@ -42,7 +43,7 @@ public abstract class AbstractTimelineAggregator implements Runnable {
   protected final PhoenixHBaseAccessor hBaseAccessor;
   private final Log LOG;
   private static final ObjectMapper mapper;
-  protected final long checkpointDelay;
+  protected final long checkpointDelayMillis;
   protected final Integer resultsetFetchSize;
   protected Configuration metricsConf;
 
@@ -54,8 +55,8 @@ public abstract class AbstractTimelineAggregator implements Runnable {
                                     Configuration metricsConf) {
     this.hBaseAccessor = hBaseAccessor;
     this.metricsConf = metricsConf;
-    this.checkpointDelay = metricsConf.getInt(AGGREGATOR_CHECKPOINT_DELAY,
-      120000);
+    this.checkpointDelayMillis = SECONDS.toMillis(
+      metricsConf.getInt(AGGREGATOR_CHECKPOINT_DELAY, 120));
     this.resultsetFetchSize = metricsConf.getInt(RESULTSET_FETCH_SIZE, 2000);
     this.LOG = LogFactory.getLog(this.getClass());
   }
@@ -63,7 +64,7 @@ public abstract class AbstractTimelineAggregator implements Runnable {
   @Override
   public void run() {
     LOG.info("Started Timeline aggregator thread @ " + new Date());
-    Long SLEEP_INTERVAL = getSleepInterval();
+    Long SLEEP_INTERVAL = getSleepIntervalMillis();
 
     while (true) {
       long currentTime = System.currentTimeMillis();
@@ -80,7 +81,7 @@ public abstract class AbstractTimelineAggregator implements Runnable {
           // Assuming first run, save checkpoint and sleep.
           // Set checkpoint to 2 minutes in the past to allow the
           // agents/collectors to catch up
-          saveCheckPoint(currentTime - checkpointDelay);
+          saveCheckPoint(currentTime - checkpointDelayMillis);
         }
       } catch (IOException io) {
         LOG.warn("Unable to write last checkpoint time. Resuming sleep.", io);
@@ -131,7 +132,7 @@ public abstract class AbstractTimelineAggregator implements Runnable {
   private boolean isLastCheckPointTooOld(long checkpoint) {
     return checkpoint != -1 &&
       ((System.currentTimeMillis() - checkpoint) >
-        getCheckpointCutOffInterval());
+        getCheckpointCutOffIntervalMillis());
   }
 
   private long readCheckPoint() {
@@ -164,12 +165,12 @@ public abstract class AbstractTimelineAggregator implements Runnable
{
   // TODO: Abstract out doWork implementation for cluster and host levels
   protected abstract boolean doWork(long startTime, long endTime);
 
-  protected abstract Long getSleepInterval();
+  protected abstract Long getSleepIntervalMillis();
 
   protected abstract Integer getCheckpointCutOffMultiplier();
 
-  protected Long getCheckpointCutOffInterval() {
-    return getCheckpointCutOffMultiplier() * getSleepInterval();
+  protected Long getCheckpointCutOffIntervalMillis() {
+    return getCheckpointCutOffMultiplier() * getSleepIntervalMillis();
   }
 
   protected abstract boolean isDisabled();

http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java
b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java
index 6a84024..16f5ab9 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java
@@ -34,6 +34,7 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
   .timeline.PhoenixTransactSQL.*;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
@@ -55,7 +56,7 @@ public class TimelineMetricAggregatorHourly extends AbstractTimelineAggregator
{
   private static final String MINUTE_AGGREGATE_HOURLY_CHECKPOINT_FILE =
     "timeline-metrics-host-aggregator-hourly-checkpoint";
   private final String checkpointLocation;
-  private final Long sleepInterval;
+  private final Long sleepIntervalMillis;
   private final Integer checkpointCutOffMultiplier;
 
   public TimelineMetricAggregatorHourly(PhoenixHBaseAccessor hBaseAccessor,
@@ -69,8 +70,8 @@ public class TimelineMetricAggregatorHourly extends AbstractTimelineAggregator
{
     checkpointLocation = FilenameUtils.concat(checkpointDir,
       MINUTE_AGGREGATE_HOURLY_CHECKPOINT_FILE);
 
-    sleepInterval = metricsConf.getLong(HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL,
-      3600000l);
+    sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+      (HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l));
     checkpointCutOffMultiplier =
       metricsConf.getInt(HOST_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
   }
@@ -181,8 +182,8 @@ public class TimelineMetricAggregatorHourly extends AbstractTimelineAggregator
{
   }
 
   @Override
-  protected Long getSleepInterval() {
-    return sleepInterval;
+  protected Long getSleepIntervalMillis() {
+    return sleepIntervalMillis;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java
b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java
index 274d0b5..ac9d12e 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java
@@ -32,6 +32,7 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.Condition;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
@@ -48,7 +49,7 @@ public class TimelineMetricAggregatorMinute extends AbstractTimelineAggregator
{
   private static final String MINUTE_AGGREGATE_CHECKPOINT_FILE =
     "timeline-metrics-host-aggregator-checkpoint";
   private final String checkpointLocation;
-  private final Long sleepInterval;
+  private final Long sleepIntervalMillis;
   private final Integer checkpointCutOffMultiplier;
 
   public TimelineMetricAggregatorMinute(PhoenixHBaseAccessor hBaseAccessor,
@@ -61,7 +62,8 @@ public class TimelineMetricAggregatorMinute extends AbstractTimelineAggregator
{
     checkpointLocation = FilenameUtils.concat(checkpointDir,
       MINUTE_AGGREGATE_CHECKPOINT_FILE);
 
-    sleepInterval = metricsConf.getLong(HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300000l);
 // 5 mins
+    sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+      (HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l));  // 5 mins
     checkpointCutOffMultiplier =
       metricsConf.getInt(HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 3);
   }
@@ -163,8 +165,8 @@ public class TimelineMetricAggregatorMinute extends AbstractTimelineAggregator
{
   }
 
   @Override
-  protected Long getSleepInterval() {
-    return sleepInterval;
+  protected Long getSleepIntervalMillis() {
+    return sleepIntervalMillis;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java
b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java
index 080703e..c52451e 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java
@@ -34,6 +34,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.Condition;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.GET_METRIC_SQL;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
@@ -54,8 +55,8 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator
   private static final String CLUSTER_AGGREGATOR_CHECKPOINT_FILE =
     "timeline-metrics-cluster-aggregator-checkpoint";
   private final String checkpointLocation;
-  private final Long sleepInterval;
-  public final int timeSliceInterval;
+  private final Long sleepIntervalMillis;
+  public final int timeSliceIntervalMillis;
   private final Integer checkpointCutOffMultiplier;
 
   public TimelineMetricClusterAggregator(PhoenixHBaseAccessor hBaseAccessor,
@@ -68,8 +69,10 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator
     checkpointLocation = FilenameUtils.concat(checkpointDir,
       CLUSTER_AGGREGATOR_CHECKPOINT_FILE);
 
-    sleepInterval = metricsConf.getLong(CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 120000l);
-    timeSliceInterval = metricsConf.getInt(CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 15000);
+    sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+      (CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 120l));
+    timeSliceIntervalMillis = (int)SECONDS.toMillis(metricsConf.getInt
+      (CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 15));
     checkpointCutOffMultiplier =
       metricsConf.getInt(CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
   }
@@ -108,8 +111,8 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator
       // Create time slices
       long sliceStartTime = startTime;
       while (sliceStartTime < endTime) {
-        timeSlices.add(new Long[] { sliceStartTime, sliceStartTime + timeSliceInterval});
-        sliceStartTime += timeSliceInterval;
+        timeSlices.add(new Long[] { sliceStartTime, sliceStartTime + timeSliceIntervalMillis});
+        sliceStartTime += timeSliceIntervalMillis;
       }
 
       while (rs.next()) {
@@ -172,8 +175,8 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator
   }
 
   @Override
-  protected Long getSleepInterval() {
-    return sleepInterval;
+  protected Long getSleepIntervalMillis() {
+    return sleepIntervalMillis;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java
b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java
index c380adb..e886b71 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java
@@ -32,6 +32,7 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
   .timeline.PhoenixHBaseAccessor.getMetricClusterAggregateFromResultSet;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
@@ -60,9 +61,9 @@ public class TimelineMetricClusterAggregatorHourly extends
   private static final String CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE =
     "timeline-metrics-cluster-aggregator-hourly-checkpoint";
   private final String checkpointLocation;
-  private final long sleepInterval;
+  private final long sleepIntervalMillis;
   private final Integer checkpointCutOffMultiplier;
-  private long checkpointCutOffInterval;
+  private long checkpointCutOffIntervalMillis;
 
   public TimelineMetricClusterAggregatorHourly(
     PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf) {
@@ -74,9 +75,9 @@ public class TimelineMetricClusterAggregatorHourly extends
     checkpointLocation = FilenameUtils.concat(checkpointDir,
       CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE);
 
-    sleepInterval = metricsConf.getLong
-      (CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600000l);
-    checkpointCutOffInterval = 7200000l;
+    sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong
+      (CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l));
+    checkpointCutOffIntervalMillis = 7200000l;
     checkpointCutOffMultiplier = metricsConf.getInt
       (CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
   }
@@ -202,8 +203,8 @@ public class TimelineMetricClusterAggregatorHourly extends
   }
 
   @Override
-  protected Long getSleepInterval() {
-    return sleepInterval;
+  protected Long getSleepIntervalMillis() {
+    return sleepIntervalMillis;
   }
 
   @Override
@@ -212,8 +213,8 @@ public class TimelineMetricClusterAggregatorHourly extends
   }
 
   @Override
-  protected Long getCheckpointCutOffInterval() {
-    return checkpointCutOffInterval;
+  protected Long getCheckpointCutOffIntervalMillis() {
+    return checkpointCutOffIntervalMillis;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index dd952b4..6b19847 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -21,7 +21,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * Configuration class that reads properties from ams-site.xml
+ * Configuration class that reads properties from ams-site.xml. All values
+ * for time or intervals are given in seconds.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving


Mime
View raw message