ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-16123 Reduce error logs on the Sink path if AMS is down (dsen)
Date Thu, 28 Apr 2016 17:57:11 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 2ff332504 -> 51480ecac


AMBARI-16123 Reduce error logs on the Sink path if AMS is down (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/51480eca
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/51480eca
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/51480eca

Branch: refs/heads/trunk
Commit: 51480ecaccae514035d3fae2b0a4757f51152567
Parents: 2ff3325
Author: Dmytro Sen <dsen@apache.org>
Authored: Thu Apr 28 20:33:22 2016 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Thu Apr 28 20:56:48 2016 +0300

----------------------------------------------------------------------
 .../timeline/AbstractTimelineMetricsSink.java   | 25 ++++++++++++++++----
 .../cache/HandleConnectExceptionTest.java       | 23 +++++++++++++-----
 2 files changed, 37 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/51480eca/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index 5a532c5..217f265 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -38,6 +38,7 @@ import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.KeyStore;
+import java.util.concurrent.atomic.AtomicInteger;
 
 public abstract class AbstractTimelineMetricsSink {
   public static final String TAGS_FOR_PREFIX_PROPERTY_PREFIX = "tagsForPrefix.";
@@ -57,6 +58,9 @@ public abstract class AbstractTimelineMetricsSink {
   public static final String SSL_KEYSTORE_TYPE_PROPERTY = "truststore.type";
   public static final String SSL_KEYSTORE_PASSWORD_PROPERTY = "truststore.password";
 
+  protected static final AtomicInteger failedCollectorConnectionsCounter = new AtomicInteger(0);
+  public static int NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS = 100;
+
   private SSLSocketFactory sslSocketFactory;
 
   protected final Log LOG;
@@ -109,9 +113,12 @@ public abstract class AbstractTimelineMetricsSink {
         }
       }
       cleanupInputStream(connection.getInputStream());
+      //reset failedCollectorConnectionsCounter to "0"
+      failedCollectorConnectionsCounter.set(0);
     } catch (IOException ioe) {
       StringBuilder errorMessage =
-          new StringBuilder("Unable to connect to collector, " + connectUrl + "\n");
+          new StringBuilder("Unable to connect to collector, " + connectUrl + "\n"
+                  + "This exceptions will be ignored for next " + NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS
+ " times\n");
       try {
         if ((connection != null)) {
           errorMessage.append(cleanupInputStream(connection.getErrorStream()));
@@ -119,12 +126,20 @@ public abstract class AbstractTimelineMetricsSink {
       } catch (IOException e) {
         //NOP
       }
-      if (LOG.isDebugEnabled()) {
-        LOG.debug(errorMessage, ioe);
+
+      if (failedCollectorConnectionsCounter.getAndIncrement() == 0) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(errorMessage, ioe);
+        } else {
+          LOG.info(errorMessage);
+        }
+        throw new UnableToConnectException(ioe).setConnectUrl(connectUrl);
       } else {
-        LOG.info(errorMessage);
+        failedCollectorConnectionsCounter.compareAndSet(NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS,
0);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(String.format("Ignoring %s AMS connection exceptions", NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS));
+        }
       }
-      throw new UnableToConnectException(ioe).setConnectUrl(connectUrl);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/51480eca/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
index 4c1a2cb..ad7b1ac 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
@@ -23,6 +23,7 @@ import java.net.HttpURLConnection;
 import java.net.URL;
 
 import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
 import org.apache.hadoop.metrics2.sink.timeline.UnableToConnectException;
 import org.junit.Assert;
@@ -50,12 +51,12 @@ public class HandleConnectExceptionTest {
     OutputStream os = createNiceMock(OutputStream.class);
     HttpURLConnection connection = createNiceMock(HttpURLConnection.class);
     URL url = createNiceMock(URL.class);
-
+    AbstractTimelineMetricsSink.NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS = 2;
     try {
-      expectNew(URL.class, "collector").andReturn(url);
-      expect(url.openConnection()).andReturn(connection).once();
-      expect(connection.getOutputStream()).andReturn(os).once();
-      expect(connection.getResponseCode()).andThrow(new IOException());
+      expectNew(URL.class, "collector").andReturn(url).anyTimes();
+      expect(url.openConnection()).andReturn(connection).anyTimes();
+      expect(connection.getOutputStream()).andReturn(os).anyTimes();
+      expect(connection.getResponseCode()).andThrow(new IOException()).anyTimes();
 
       replayAll();
     } catch (Exception e) {
@@ -65,8 +66,18 @@ public class HandleConnectExceptionTest {
 
   @Test
   public void handleTest(){
-    try{
+    emitMetricsWithExpectedException(new TimelineMetrics());
+    try {
       sink.emitMetrics(new TimelineMetrics());
+    } catch (Exception e) {
+      Assert.fail("There should be no exception");
+    }
+    emitMetricsWithExpectedException(new TimelineMetrics());
+  }
+
+  private void emitMetricsWithExpectedException(TimelineMetrics timelineMetrics) {
+    try{
+      sink.emitMetrics(timelineMetrics);
       Assert.fail();
     }catch(UnableToConnectException e){
       Assert.assertEquals(COLLECTOR_URL, e.getConnectUrl());


Mime
View raw message