distributedlog-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject [11/20] incubator-distributedlog git commit: DL-108: Log rate limiting more clearly
Date Wed, 28 Dec 2016 01:05:19 GMT
DL-108: Log rate limiting more clearly


Project: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/commit/ac0b7ec8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/tree/ac0b7ec8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/diff/ac0b7ec8

Branch: refs/heads/master
Commit: ac0b7ec84fbd0d59e79181953295a8ec94f01530
Parents: c6ea175
Author: Yiming Zang <yzang@twitter.com>
Authored: Fri Sep 16 17:35:51 2016 -0700
Committer: Sijie Guo <sijieg@twitter.com>
Committed: Tue Dec 27 16:49:28 2016 -0800

----------------------------------------------------------------------
 .../twitter/distributedlog/BKDistributedLogNamespace.java |  4 ++--
 .../com/twitter/distributedlog/BKLogSegmentWriter.java    |  3 ++-
 .../java/com/twitter/distributedlog/WriteLimiter.java     |  5 +++++
 .../twitter/distributedlog/util/LimitedPermitManager.java |  5 +++++
 .../com/twitter/distributedlog/util/PermitLimiter.java    | 10 ++++++++++
 .../com/twitter/distributedlog/util/PermitManager.java    | 10 ++++++++++
 .../twitter/distributedlog/util/SimplePermitLimiter.java  |  5 +++++
 .../service/stream/limiter/ServiceRequestLimiter.java     |  4 ++--
 .../service/stream/limiter/StreamRequestLimiter.java      |  4 ++--
 9 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ac0b7ec8/distributedlog-core/src/main/java/com/twitter/distributedlog/BKDistributedLogNamespace.java
----------------------------------------------------------------------
diff --git a/distributedlog-core/src/main/java/com/twitter/distributedlog/BKDistributedLogNamespace.java
b/distributedlog-core/src/main/java/com/twitter/distributedlog/BKDistributedLogNamespace.java
index 0df2f1c..f8d347a 100644
--- a/distributedlog-core/src/main/java/com/twitter/distributedlog/BKDistributedLogNamespace.java
+++ b/distributedlog-core/src/main/java/com/twitter/distributedlog/BKDistributedLogNamespace.java
@@ -1080,8 +1080,8 @@ public class BKDistributedLogNamespace implements DistributedLogNamespace
{
         }
 
         // Unregister gauge to avoid GC spiral
-        ((LimitedPermitManager)this.logSegmentRollingPermitManager).unregisterGauge();
-        ((SimplePermitLimiter)this.writeLimiter).unregisterGauge();
+        this.logSegmentRollingPermitManager.close();
+        this.writeLimiter.close();
 
         // Shutdown log segment metadata stores
         Utils.close(writerSegmentMetadataStore);

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ac0b7ec8/distributedlog-core/src/main/java/com/twitter/distributedlog/BKLogSegmentWriter.java
----------------------------------------------------------------------
diff --git a/distributedlog-core/src/main/java/com/twitter/distributedlog/BKLogSegmentWriter.java
b/distributedlog-core/src/main/java/com/twitter/distributedlog/BKLogSegmentWriter.java
index 8276125..8029f89 100644
--- a/distributedlog-core/src/main/java/com/twitter/distributedlog/BKLogSegmentWriter.java
+++ b/distributedlog-core/src/main/java/com/twitter/distributedlog/BKLogSegmentWriter.java
@@ -534,8 +534,9 @@ class BKLogSegmentWriter implements LogSegmentWriter, AddCallback, Runnable,
Siz
     private void closeInternal(final boolean abort,
                                final AtomicReference<Throwable> throwExc,
                                final Promise<Void> closePromise) {
-        // remove stats
+        // clean stats resources
         this.transmitOutstandingLogger.unregisterGauge("requests", transmitOutstandingGauge);
+        this.writeLimiter.close();
 
         // Cancel the periodic keep alive schedule first
         if (null != periodicKeepAliveSchedule) {

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ac0b7ec8/distributedlog-core/src/main/java/com/twitter/distributedlog/WriteLimiter.java
----------------------------------------------------------------------
diff --git a/distributedlog-core/src/main/java/com/twitter/distributedlog/WriteLimiter.java
b/distributedlog-core/src/main/java/com/twitter/distributedlog/WriteLimiter.java
index 9b5cdd0..0b24c1a 100644
--- a/distributedlog-core/src/main/java/com/twitter/distributedlog/WriteLimiter.java
+++ b/distributedlog-core/src/main/java/com/twitter/distributedlog/WriteLimiter.java
@@ -54,4 +54,9 @@ public class WriteLimiter {
         streamLimiter.release(permits);
         globalLimiter.release(permits);
     }
+
+    public void close() {
+        streamLimiter.close();
+        globalLimiter.close();
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ac0b7ec8/distributedlog-core/src/main/java/com/twitter/distributedlog/util/LimitedPermitManager.java
----------------------------------------------------------------------
diff --git a/distributedlog-core/src/main/java/com/twitter/distributedlog/util/LimitedPermitManager.java
b/distributedlog-core/src/main/java/com/twitter/distributedlog/util/LimitedPermitManager.java
index 4b917b2..dc25023 100644
--- a/distributedlog-core/src/main/java/com/twitter/distributedlog/util/LimitedPermitManager.java
+++ b/distributedlog-core/src/main/java/com/twitter/distributedlog/util/LimitedPermitManager.java
@@ -146,6 +146,11 @@ public class LimitedPermitManager implements PermitManager, Runnable,
Watcher {
     }
 
     @Override
+    public void close() {
+        unregisterGauge();
+    }
+
+    @Override
     synchronized public boolean allowObtainPermits() {
         forceSetAllowPermits(true);
         return true;

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ac0b7ec8/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitLimiter.java
----------------------------------------------------------------------
diff --git a/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitLimiter.java
b/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitLimiter.java
index 61366ac..41c28a3 100644
--- a/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitLimiter.java
+++ b/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitLimiter.java
@@ -31,6 +31,11 @@ public interface PermitLimiter {
         @Override
         public void release(int permits) {
         }
+
+        @Override
+        public void close() {
+
+        }
     };
 
     /**
@@ -44,4 +49,9 @@ public interface PermitLimiter {
      * Release a permit.
      */
     void release(int permits);
+
+    /**
+     * Close the resources created by the limiter
+     */
+    void close();
 }

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ac0b7ec8/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitManager.java
----------------------------------------------------------------------
diff --git a/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitManager.java
b/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitManager.java
index f93c7bf..6a6d574 100644
--- a/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitManager.java
+++ b/distributedlog-core/src/main/java/com/twitter/distributedlog/util/PermitManager.java
@@ -50,6 +50,11 @@ public interface PermitManager {
             return false;
         }
 
+        @Override
+        public void close() {
+            // nop
+        }
+
     };
 
     /**
@@ -80,4 +85,9 @@ public interface PermitManager {
      *          permit context to disallow
      */
     boolean disallowObtainPermits(Permit permit);
+
+    /**
+     * Release the resources
+     */
+    void close();
 }

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ac0b7ec8/distributedlog-core/src/main/java/com/twitter/distributedlog/util/SimplePermitLimiter.java
----------------------------------------------------------------------
diff --git a/distributedlog-core/src/main/java/com/twitter/distributedlog/util/SimplePermitLimiter.java
b/distributedlog-core/src/main/java/com/twitter/distributedlog/util/SimplePermitLimiter.java
index 2482ece..4086a1e 100644
--- a/distributedlog-core/src/main/java/com/twitter/distributedlog/util/SimplePermitLimiter.java
+++ b/distributedlog-core/src/main/java/com/twitter/distributedlog/util/SimplePermitLimiter.java
@@ -99,6 +99,11 @@ public class SimplePermitLimiter implements PermitLimiter {
         permits.addAndGet(-permitsToRelease);
     }
 
+    @Override
+    public void close() {
+        unregisterGauge();
+    }
+
     @VisibleForTesting
     public int getPermits() {
         return permits.get();

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ac0b7ec8/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/ServiceRequestLimiter.java
----------------------------------------------------------------------
diff --git a/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/ServiceRequestLimiter.java
b/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/ServiceRequestLimiter.java
index 615ff21..69a8470 100644
--- a/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/ServiceRequestLimiter.java
+++ b/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/ServiceRequestLimiter.java
@@ -66,7 +66,7 @@ public class ServiceRequestLimiter extends DynamicRequestLimiter<StreamOp>
{
             .overlimit(new OverlimitFunction<StreamOp>() {
                 @Override
                 public void apply(StreamOp request) throws OverCapacityException {
-                    throw new OverCapacityException("RPS limit exceeded for the service instance");
+                    throw new OverCapacityException("Being rate limited: RPS limit exceeded
for the service instance");
                 }
             });
 
@@ -80,7 +80,7 @@ public class ServiceRequestLimiter extends DynamicRequestLimiter<StreamOp>
{
             .overlimit(new OverlimitFunction<StreamOp>() {
                 @Override
                 public void apply(StreamOp request) throws OverCapacityException {
-                    throw new OverCapacityException("BPS limit exceeded for the service instance");
+                    throw new OverCapacityException("Being rate limited: BPS limit exceeded
for the service instance");
                 }
             });
 

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ac0b7ec8/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/StreamRequestLimiter.java
----------------------------------------------------------------------
diff --git a/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/StreamRequestLimiter.java
b/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/StreamRequestLimiter.java
index b0e5ba4..b4836d1 100644
--- a/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/StreamRequestLimiter.java
+++ b/distributedlog-service/src/main/java/com/twitter/distributedlog/service/stream/limiter/StreamRequestLimiter.java
@@ -52,7 +52,7 @@ public class StreamRequestLimiter extends DynamicRequestLimiter<StreamOp>
{
             .overlimit(new OverlimitFunction<StreamOp>() {
                 @Override
                 public void apply(StreamOp op) throws OverCapacityException {
-                    throw new OverCapacityException("RPS limit exceeded for stream " + streamName);
+                    throw new OverCapacityException("Being rate limited: RPS limit exceeded
for stream " + streamName);
                 }
             });
         RequestLimiterBuilder rpsSoftLimiterBuilder = RequestLimiterBuilder.newRpsLimiterBuilder()
@@ -66,7 +66,7 @@ public class StreamRequestLimiter extends DynamicRequestLimiter<StreamOp>
{
             .overlimit(new OverlimitFunction<StreamOp>() {
                 @Override
                 public void apply(StreamOp op) throws OverCapacityException {
-                    throw new OverCapacityException("BPS limit exceeded for stream " + streamName);
+                    throw new OverCapacityException("Being rate limited: BPS limit exceeded
for stream " + streamName);
                 }
             });
         RequestLimiterBuilder bpsSoftLimiterBuilder = RequestLimiterBuilder.newBpsLimiterBuilder()


Mime
View raw message