aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject aurora git commit: Expose stats on ZooKeeper connection state
Date Tue, 10 Jan 2017 22:36:02 GMT
Repository: aurora
Updated Branches:
  refs/heads/master eb3660d9e -> 3bdf0274c


Expose stats on ZooKeeper connection state

* zk_connection_state_STATE shows 1 if STATE is current connection state, otherwise 0.
* zk_connection_state_STATE_counter represents occurence times of the STATE since scheduler
state

Bugs closed: AURORA-1838

Reviewed at https://reviews.apache.org/r/54624/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/3bdf0274
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/3bdf0274
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/3bdf0274

Branch: refs/heads/master
Commit: 3bdf0274c18b77df354d3e0bf138e25072ed4706
Parents: eb3660d
Author: Jing Chen <milantracy@gmail.com>
Authored: Tue Jan 10 23:35:21 2017 +0100
Committer: Stephan Erb <serb@apache.org>
Committed: Tue Jan 10 23:35:21 2017 +0100

----------------------------------------------------------------------
 .../CuratorServiceDiscoveryModule.java          | 69 +++++++++++++++++++-
 .../discovery/AbstractDiscoveryModuleTest.java  |  4 ++
 2 files changed, 72 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/3bdf0274/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java
b/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java
index 999a542..ea167a8 100644
--- a/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java
@@ -15,11 +15,13 @@ package org.apache.aurora.scheduler.discovery;
 
 import java.net.InetSocketAddress;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Predicate;
 
 import javax.inject.Singleton;
 
 import com.google.common.base.Joiner;
+import com.google.common.base.Supplier;
 import com.google.common.collect.FluentIterable;
 import com.google.inject.Exposed;
 import com.google.inject.PrivateModule;
@@ -32,6 +34,7 @@ import org.apache.aurora.common.io.Codec;
 import org.apache.aurora.common.net.InetSocketAddressHelper;
 import org.apache.aurora.common.quantity.Amount;
 import org.apache.aurora.common.quantity.Time;
+import org.apache.aurora.common.stats.StatsProvider;
 import org.apache.aurora.common.thrift.ServiceInstance;
 import org.apache.aurora.common.zookeeper.Credentials;
 import org.apache.aurora.common.zookeeper.ServerSet;
@@ -41,7 +44,10 @@ import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.framework.api.ACLProvider;
+import org.apache.curator.framework.listen.Listenable;
 import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.state.ConnectionState;
+import org.apache.curator.framework.state.ConnectionStateListener;
 import org.apache.curator.retry.BoundedExponentialBackoffRetry;
 import org.apache.curator.utils.PathUtils;
 import org.apache.zookeeper.data.ACL;
@@ -58,6 +64,8 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
   private final String discoveryPath;
   private final ZooKeeperConfig zooKeeperConfig;
 
+  private ConnectionState currentState;
+
   CuratorServiceDiscoveryModule(String discoveryPath, ZooKeeperConfig zooKeeperConfig) {
     this.discoveryPath = PathUtils.validatePath(discoveryPath);
     this.zooKeeperConfig = requireNonNull(zooKeeperConfig);
@@ -76,7 +84,8 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
   CuratorFramework provideCuratorFramework(
       ShutdownRegistry shutdownRegistry,
       @ServiceDiscoveryBindings.ZooKeeper Iterable<InetSocketAddress> zooKeeperCluster,
-      ACLProvider aclProvider) {
+      ACLProvider aclProvider,
+      StatsProvider statsProvider) {
 
     String connectString =
         FluentIterable.from(zooKeeperCluster)
@@ -87,6 +96,31 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
       connectString = connectString + zooKeeperConfig.getChrootPath().get();
     }
 
+    // export current connection state
+    for (ConnectionState connectionState : ConnectionState.values()) {
+      statsProvider.makeGauge(
+          zkConnectionGaugeName(connectionState),
+          new Supplier<Integer>() {
+            @Override
+            public Integer get() {
+              return connectionState.equals(currentState) ? 1 : 0;
+            }
+          }
+      );
+    }
+
+    // connection state counter
+    AtomicLong zkConnectionConnectedCounter =
+        statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.CONNECTED));
+    AtomicLong zkConnectionReadonlyCounter =
+        statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.READ_ONLY));
+    AtomicLong zkConnectionSuspendedCounter =
+        statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.SUSPENDED));
+    AtomicLong zkConnectionReconnectedCounter =
+        statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.RECONNECTED));
+    AtomicLong zkConnectionLostCounter =
+        statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.LOST));
+
     // This emulates the default BackoffHelper configuration used by the legacy commons/zookeeper
     // stack. BackoffHelper is unbounded, this dies after around 5 minutes using the 10 retries.
     // NB: BoundedExponentialBackoffRetry caps max retries at 29 if you send it a larger
value.
@@ -110,6 +144,31 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
     }
 
     CuratorFramework curatorFramework = builder.build();
+    Listenable<ConnectionStateListener> connectionStateListener = curatorFramework
+        .getConnectionStateListenable();
+    connectionStateListener.addListener((CuratorFramework client, ConnectionState newState)
-> {
+      currentState = newState;
+      switch (newState) {
+        case CONNECTED:
+          zkConnectionConnectedCounter.getAndIncrement();
+          break;
+        case READ_ONLY:
+          zkConnectionReadonlyCounter.getAndIncrement();
+          break;
+        case SUSPENDED:
+          zkConnectionSuspendedCounter.getAndIncrement();
+          break;
+        case RECONNECTED:
+          zkConnectionReconnectedCounter.getAndIncrement();
+          break;
+        case LOST:
+          zkConnectionLostCounter.getAndIncrement();
+          break;
+        default:
+          currentState = null;
+          break;
+      }
+    });
 
     // TODO(John Sirois): It would be nice to use a Service to control the lifecycle here,
but other
     // services (org.apache.aurora.scheduler.http.JettyServerModule.RedirectMonitor) rely
on this
@@ -182,4 +241,12 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
   SingletonService provideSingletonService(CuratorFramework client, Codec<ServiceInstance>
codec) {
     return new CuratorSingletonService(client, discoveryPath, MEMBER_TOKEN, codec);
   }
+
+  private String zkConnectionStateCounterName(ConnectionState state) {
+    return zkConnectionGaugeName(state) + "_counter";
+  }
+
+  private String zkConnectionGaugeName(ConnectionState state) {
+    return "zk_connection_state_" + state;
+  }
 }

http://git-wip-us.apache.org/repos/asf/aurora/blob/3bdf0274/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java
b/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java
index d90192b..0f2121e 100644
--- a/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java
@@ -25,11 +25,13 @@ import com.google.inject.Module;
 
 import org.apache.aurora.common.quantity.Amount;
 import org.apache.aurora.common.quantity.Time;
+import org.apache.aurora.common.stats.StatsProvider;
 import org.apache.aurora.common.testing.TearDownTestCase;
 import org.apache.aurora.common.zookeeper.Credentials;
 import org.apache.aurora.common.zookeeper.SingletonService;
 import org.apache.aurora.common.zookeeper.ZooKeeperUtils;
 import org.apache.aurora.scheduler.app.ServiceGroupMonitor;
+import org.apache.aurora.scheduler.testing.FakeStatsProvider;
 import org.junit.Test;
 
 import static org.junit.Assert.assertNotNull;
@@ -58,6 +60,8 @@ abstract class AbstractDiscoveryModuleTest extends TearDownTestCase {
                 bind(ServiceDiscoveryBindings.ZOO_KEEPER_ACL_KEY)
                     .toInstance(ZooKeeperUtils.OPEN_ACL_UNSAFE);
 
+                bind(StatsProvider.class).toInstance(new FakeStatsProvider());
+
                 bindExtraRequirements(binder());
               }
             },


Mime
View raw message