ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject [03/34] ambari git commit: AMBARI-21329 : No data on templated Grafana dashboards on HDF cluster. (avijayan)
Date Mon, 26 Jun 2017 13:57:37 GMT
AMBARI-21329 : No data on templated Grafana dashboards on HDF cluster. (avijayan)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8b90145c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8b90145c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8b90145c

Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 8b90145caadf73c550c2d209eec4044ba924de9a
Parents: 8f80fe7
Author: Aravindan Vijayan <avijayan@hortonworks.com>
Authored: Fri Jun 23 14:27:08 2017 -0700
Committer: Aravindan Vijayan <avijayan@hortonworks.com>
Committed: Fri Jun 23 14:27:08 2017 -0700

----------------------------------------------------------------------
 .../timeline/AbstractTimelineMetricsSink.java   | 25 ++++--
 .../cache/HandleConnectExceptionTest.java       | 86 +++++++++++++++++++-
 .../timeline/HBaseTimelineMetricStore.java      |  6 +-
 .../timeline/TimelineMetricConfiguration.java   | 10 +++
 .../system/impl/AmbariMetricSinkImpl.java       |  3 +
 .../system/impl/DatabaseMetricsSource.java      |  4 +-
 .../metrics/system/impl/JvmMetricsSource.java   | 12 ++-
 .../metrics/system/impl/MetricsServiceImpl.java |  5 +-
 8 files changed, 133 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index 7a84627..337f640 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -290,22 +290,29 @@ public abstract class AbstractTimelineMetricsSink {
 
   protected boolean emitMetrics(TimelineMetrics metrics) {
     String connectUrl;
+    boolean validCollectorHost = true;
+
     if (isHostInMemoryAggregationEnabled()) {
       connectUrl = constructTimelineMetricUri("http", "localhost", String.valueOf(getHostInMemoryAggregationPort()));
     } else {
       String collectorHost  = getCurrentCollectorHost();
+      if (collectorHost == null) {
+        validCollectorHost = false;
+      }
       connectUrl = getCollectorUri(collectorHost);
     }
 
-    String jsonData = null;
-    LOG.debug("EmitMetrics connectUrl = "  + connectUrl);
-    try {
-      jsonData = mapper.writeValueAsString(metrics);
-    } catch (IOException e) {
-      LOG.error("Unable to parse metrics", e);
-    }
-    if (jsonData != null) {
-      return emitMetricsJson(connectUrl, jsonData);
+    if (validCollectorHost) {
+      String jsonData = null;
+      LOG.debug("EmitMetrics connectUrl = "  + connectUrl);
+      try {
+        jsonData = mapper.writeValueAsString(metrics);
+      } catch (IOException e) {
+        LOG.error("Unable to parse metrics", e);
+      }
+      if (jsonData != null) {
+        return emitMetricsJson(connectUrl, jsonData);
+      }
     }
     return false;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
index 4eb75eb..3be2162 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
@@ -45,7 +45,7 @@ import static org.powermock.api.easymock.PowerMock.replayAll;
 public class HandleConnectExceptionTest {
   private static final String COLLECTOR_URL = "collector";
   private TestTimelineMetricsSink sink;
-  
+
   @Before
   public void init(){
     sink = new TestTimelineMetricsSink();
@@ -88,6 +88,17 @@ public class HandleConnectExceptionTest {
     }
   }
 
+  @Test
+  public void testEmitMetricsWithNullHost() {
+    TestTimelineMetricsSinkWithNullHost sinkWithNullHost = new TestTimelineMetricsSinkWithNullHost();
+
+    boolean success = sinkWithNullHost.emitMetrics(new TimelineMetrics());
+    Assert.assertFalse(success);
+
+    success = sinkWithNullHost.emitMetrics(new TimelineMetrics());
+    Assert.assertTrue(success);
+  }
+
   private class TestTimelineMetricsSink extends AbstractTimelineMetricsSink{
     @Override
     protected String getCollectorUri(String host) {
@@ -146,4 +157,77 @@ public class HandleConnectExceptionTest {
     }
 
   }
+
+  private class TestTimelineMetricsSinkWithNullHost extends AbstractTimelineMetricsSink {
+
+    int ctr = 0;
+
+    @Override
+    protected String getCollectorUri(String host) {
+      return COLLECTOR_URL;
+    }
+
+    @Override
+    protected String getCollectorProtocol() {
+      return "http";
+    }
+
+    @Override
+    protected String getCollectorPort() {
+      return "2181";
+    }
+
+    @Override
+    protected int getTimeoutSeconds() {
+      return 10;
+    }
+
+    @Override
+    protected String getZookeeperQuorum() {
+      return "localhost:2181";
+    }
+
+    @Override
+    protected Collection<String> getConfiguredCollectorHosts() {
+      return Arrays.asList("localhost");
+    }
+
+    @Override
+    protected String getHostname() {
+      return "h1";
+    }
+
+    @Override
+    protected boolean isHostInMemoryAggregationEnabled() {
+      return false;
+    }
+
+    @Override
+    protected int getHostInMemoryAggregationPort() {
+      return 0;
+    }
+
+    @Override
+    public boolean emitMetrics(TimelineMetrics metrics) {
+      super.init();
+      return super.emitMetrics(metrics);
+    }
+
+    @Override
+    protected synchronized String findPreferredCollectHost() {
+      if (ctr == 0) {
+        ctr++;
+        return null;
+      } else {
+        return "localhost";
+      }
+    }
+
+    @Override
+    protected boolean emitMetricsJson(String connectUrl, String jsonData) {
+      return true;
+    }
+
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index f984253..12c27a4 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -415,7 +415,11 @@ public class HBaseTimelineMetricStore extends AbstractService implements
Timelin
           throws SQLException, IOException {
 
     Map<String, Set<String>> hostedApps = metricMetadataManager.getHostedAppsCache();
-    Map<String, Set<String>> instanceHosts = metricMetadataManager.getHostedInstanceCache();
+    Map<String, Set<String>> instanceHosts = new HashMap<>();
+    if (configuration.getTimelineMetricsMultipleClusterSupport()) {
+      instanceHosts = metricMetadataManager.getHostedInstanceCache();
+    }
+
     Map<String, Map<String, Set<String>>> instanceAppHosts = new HashMap<>();
 
     if (MapUtils.isEmpty(instanceHosts)) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 023465b..006a403 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -290,6 +290,9 @@ public class TimelineMetricConfiguration {
   public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES
=
     "timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles";
 
+  public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
+    "timeline.metrics.support.multiple.clusters";
+
   public static final String HOST_APP_ID = "HOST";
 
   public static final String DEFAULT_INSTANCE_PORT = "12001";
@@ -440,6 +443,13 @@ public class TimelineMetricConfiguration {
     return 3;
   }
 
+  public boolean getTimelineMetricsMultipleClusterSupport() {
+    if (metricsConf != null) {
+      return Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS,
"false"));
+    }
+    return false;
+  }
+
   public String getTimelineServiceRpcAddress() {
     String defaultRpcAddress = "0.0.0.0:60200";
     if (metricsConf != null) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
index 7dec18f..5c5801d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
@@ -206,6 +206,7 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink
implements
     timelineMetricsCache = new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval);
 
     if (CollectionUtils.isNotEmpty(collectorHosts)) {
+      LOG.info("Metric Sink initialized with collectorHosts : " + collectorHosts.toString());
       isInitialized = true;
     }
   }
@@ -234,6 +235,8 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink
implements
         timelineMetrics.setMetrics(metricList);
         emitMetrics(timelineMetrics);
       }
+    } else {
+      LOG.debug("Metric Sink not yet initialized. Discarding metrics.");
     }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
index 423b19f..a93b96c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/DatabaseMetricsSource.java
@@ -59,7 +59,7 @@ public class DatabaseMetricsSource extends AbstractMetricsSource {
     super.init(metricsConfig, sink);
     configuration = metricsConfig;
     initializeFilterSets();
-    LOG.info("DatabaseMetricsSource initialized.");
+    LOG.info("Initialized Ambari DB Metrics Source...");
   }
 
   /**
@@ -83,11 +83,11 @@ public class DatabaseMetricsSource extends AbstractMetricsSource {
 
   @Override
   public void start() {
-    LOG.info("Starting Database Metrics source...");
     ThreadFactory threadFactory = new ThreadFactoryBuilder()
       .setNameFormat("DatabaseMetricsSource-%d")
       .build();
     executor = Executors.newSingleThreadExecutor(threadFactory);
+    LOG.info("Started Ambari DB Metrics source...");
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
index 348cc4f..ad05aa2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
@@ -60,20 +60,24 @@ public class JvmMetricsSource extends AbstractMetricsSource {
     registerAll(JVM_PREFIX + ".threads", new ThreadStatesGaugeSet(), registry);
     registry.register(JVM_PREFIX + ".file.open.descriptor.ratio", new FileDescriptorRatioGauge());
     interval = Integer.parseInt(configuration.getProperty("interval", "10"));
-    LOG.info("JVM Metrics source initialized.");
+    LOG.info("Initialized JVM Metrics source...");
   }
 
   @Override
   public void start() {
-    LOG.info("Starting JVM Metrics source...");
     try {
       executor.scheduleWithFixedDelay(new Runnable() {
         @Override
         public void run() {
-          sink.publish(getMetrics());
-          LOG.debug("********* Published JVM metrics to sink **********");
+          try {
+            LOG.debug("Publishing JVM metrics to sink");
+            sink.publish(getMetrics());
+          } catch (Exception e) {
+            LOG.debug("Error in publishing JVM metrics to sink.");
+          }
         }
       }, interval, interval, TimeUnit.SECONDS);
+      LOG.info("Started JVM Metrics source...");
     } catch (Exception e) {
       LOG.info("Throwing exception when starting metric source", e);
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8b90145c/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
index 0172b1d..ab8d779 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
@@ -61,9 +61,12 @@ public class MetricsServiceImpl implements MetricsService {
         Executors.newScheduledThreadPool(1).scheduleWithFixedDelay(new Runnable() {
           @Override
           public void run() {
-            LOG.info("Checking for metrics sink initialization");
             if (!sink.isInitialized()) {
+              LOG.info("Attempting to initialize metrics sink");
               initializeMetricsSink();
+              if (sink.isInitialized()) {
+                LOG.info("Metric sink initialization successful");
+              }
             }
           }
         }, 5, 5, TimeUnit.MINUTES);


Mime
View raw message