From commits-return-71588-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Sat Apr 21 02:00:17 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 1E461180645 for ; Sat, 21 Apr 2018 02:00:16 +0200 (CEST) Received: (qmail 10414 invoked by uid 500); 21 Apr 2018 00:00:16 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 10405 invoked by uid 99); 21 Apr 2018 00:00:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 21 Apr 2018 00:00:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BAB7DF31FB; Sat, 21 Apr 2018 00:00:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: huaxiangsun@apache.org To: commits@hbase.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-19924 hbase rpc throttling does not work for multi() with request count rater. Date: Sat, 21 Apr 2018 00:00:15 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/branch-2 05f8e9419 -> 298ce9624 HBASE-19924 hbase rpc throttling does not work for multi() with request count rater. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/298ce962 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/298ce962 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/298ce962 Branch: refs/heads/branch-2 Commit: 298ce962460b575544f922795579488a3451c904 Parents: 05f8e94 Author: Huaxiang Sun Authored: Fri Apr 20 16:54:03 2018 -0700 Committer: Huaxiang Sun Committed: Fri Apr 20 16:55:23 2018 -0700 ---------------------------------------------------------------------- .../hbase/quotas/DefaultOperationQuota.java | 4 +-- .../hadoop/hbase/quotas/NoopQuotaLimiter.java | 6 ++-- .../hadoop/hbase/quotas/QuotaLimiter.java | 12 ++++--- .../hadoop/hbase/quotas/TimeBasedLimiter.java | 37 ++++++++++--------- .../hadoop/hbase/quotas/TestQuotaState.java | 38 +++++++++++++++++--- 5 files changed, 66 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/298ce962/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java index 80b39a8..1265a42 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java @@ -69,13 +69,13 @@ public class DefaultOperationQuota implements OperationQuota { for (final QuotaLimiter limiter: limiters) { if (limiter.isBypass()) continue; - limiter.checkQuota(writeConsumed, readConsumed); + limiter.checkQuota(numWrites, writeConsumed, numReads + numScans, readConsumed); readAvailable = Math.min(readAvailable, limiter.getReadAvailable()); writeAvailable = Math.min(writeAvailable, limiter.getWriteAvailable()); } for (final QuotaLimiter limiter: limiters) { - limiter.grabQuota(writeConsumed, readConsumed); + limiter.grabQuota(numWrites, writeConsumed, numReads + numScans, readConsumed); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/298ce962/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/NoopQuotaLimiter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/NoopQuotaLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/NoopQuotaLimiter.java index acfdc52..3cca955 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/NoopQuotaLimiter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/NoopQuotaLimiter.java @@ -35,13 +35,13 @@ class NoopQuotaLimiter implements QuotaLimiter { } @Override - public void checkQuota(long estimateWriteSize, long estimateReadSize) - throws RpcThrottlingException { + public void checkQuota(long writeReqs, long estimateWriteSize, long readReqs, + long estimateReadSize) throws RpcThrottlingException { // no-op } @Override - public void grabQuota(long writeSize, long readSize) { + public void grabQuota(long writeReqs, long writeSize, long readReqs, long readSize) { // no-op } http://git-wip-us.apache.org/repos/asf/hbase/blob/298ce962/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiter.java index 1144aec..7cb29b3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiter.java @@ -31,22 +31,26 @@ public interface QuotaLimiter { /** * Checks if it is possible to execute the specified operation. * + * @param writeReqs the write requests that will be checked against the available quota * @param estimateWriteSize the write size that will be checked against the available quota + * @param readReqs the read requests that will be checked against the available quota * @param estimateReadSize the read size that will be checked against the available quota - * @throws RpcThrottlingException thrown if not enough avialable resources to perform operation. + * @throws RpcThrottlingException thrown if not enough available resources to perform operation. */ - void checkQuota(long estimateWriteSize, long estimateReadSize) - throws RpcThrottlingException; + void checkQuota(long writeReqs, long estimateWriteSize, long readReqs, long estimateReadSize) + throws RpcThrottlingException; /** * Removes the specified write and read amount from the quota. * At this point the write and read amount will be an estimate, * that will be later adjusted with a consumeWrite()/consumeRead() call. * + * @param writeReqs the write requests that will be removed from the current quota * @param writeSize the write size that will be removed from the current quota + * @param readReqs the read requests that will be removed from the current quota * @param readSize the read size that will be removed from the current quota */ - void grabQuota(long writeSize, long readSize); + void grabQuota(long writeReqs, long writeSize, long readReqs, long readSize); /** * Removes or add back some write amount to the quota. http://git-wip-us.apache.org/repos/asf/hbase/blob/298ce962/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java index 12bee80..02dffcf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java @@ -110,47 +110,50 @@ public class TimeBasedLimiter implements QuotaLimiter { } @Override - public void checkQuota(long writeSize, long readSize) throws RpcThrottlingException { - if (!reqsLimiter.canExecute()) { + public void checkQuota(long writeReqs, long estimateWriteSize, long readReqs, + long estimateReadSize) throws RpcThrottlingException { + if (!reqsLimiter.canExecute(writeReqs + readReqs)) { RpcThrottlingException.throwNumRequestsExceeded(reqsLimiter.waitInterval()); } - if (!reqSizeLimiter.canExecute(writeSize + readSize)) { - RpcThrottlingException.throwRequestSizeExceeded(reqSizeLimiter - .waitInterval(writeSize + readSize)); + if (!reqSizeLimiter.canExecute(estimateWriteSize + estimateReadSize)) { + RpcThrottlingException.throwRequestSizeExceeded( + reqSizeLimiter.waitInterval(estimateWriteSize + estimateReadSize)); } - if (writeSize > 0) { - if (!writeReqsLimiter.canExecute()) { + if (estimateWriteSize > 0) { + if (!writeReqsLimiter.canExecute(writeReqs)) { RpcThrottlingException.throwNumWriteRequestsExceeded(writeReqsLimiter.waitInterval()); } - if (!writeSizeLimiter.canExecute(writeSize)) { - RpcThrottlingException.throwWriteSizeExceeded(writeSizeLimiter.waitInterval(writeSize)); + if (!writeSizeLimiter.canExecute(estimateWriteSize)) { + RpcThrottlingException.throwWriteSizeExceeded( + writeSizeLimiter.waitInterval(estimateWriteSize)); } } - if (readSize > 0) { - if (!readReqsLimiter.canExecute()) { + if (estimateReadSize > 0) { + if (!readReqsLimiter.canExecute(readReqs)) { RpcThrottlingException.throwNumReadRequestsExceeded(readReqsLimiter.waitInterval()); } - if (!readSizeLimiter.canExecute(readSize)) { - RpcThrottlingException.throwReadSizeExceeded(readSizeLimiter.waitInterval(readSize)); + if (!readSizeLimiter.canExecute(estimateReadSize)) { + RpcThrottlingException.throwReadSizeExceeded( + readSizeLimiter.waitInterval(estimateReadSize)); } } } @Override - public void grabQuota(long writeSize, long readSize) { + public void grabQuota(long writeReqs, long writeSize, long readReqs, long readSize) { assert writeSize != 0 || readSize != 0; - reqsLimiter.consume(1); + reqsLimiter.consume(writeReqs + readReqs); reqSizeLimiter.consume(writeSize + readSize); if (writeSize > 0) { - writeReqsLimiter.consume(1); + writeReqsLimiter.consume(writeReqs); writeSizeLimiter.consume(writeSize); } if (readSize > 0) { - readReqsLimiter.consume(1); + readReqsLimiter.consume(readReqs); readSizeLimiter.consume(readSize); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/298ce962/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java index 8a77e0e..0cbc445 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java @@ -203,6 +203,34 @@ public class TestQuotaState { assertNoopLimiter(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); } + @Test(timeout = 60000) + public void testTableThrottleWithBatch() { + final TableName TABLE_A = TableName.valueOf("TableA"); + final int TABLE_A_THROTTLE_1 = 3; + final long LAST_UPDATE_1 = 10; + + UserQuotaState quotaInfo = new UserQuotaState(); + assertEquals(0, quotaInfo.getLastUpdate()); + assertTrue(quotaInfo.isBypass()); + + // Add A table limiters + UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE_1); + otherQuotaState.setQuotas(TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); + assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); + assertFalse(otherQuotaState.isBypass()); + + quotaInfo.update(otherQuotaState); + assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); + assertFalse(quotaInfo.isBypass()); + QuotaLimiter limiter = quotaInfo.getTableLimiter(TABLE_A); + try { + limiter.checkQuota(TABLE_A_THROTTLE_1 + 1, TABLE_A_THROTTLE_1 + 1, 0, 0); + fail("Should have thrown RpcThrottlingException"); + } catch (RpcThrottlingException e) { + // expected + } + } + private Quotas buildReqNumThrottle(final long limit) { return Quotas.newBuilder() .setThrottle(Throttle.newBuilder() @@ -214,8 +242,8 @@ public class TestQuotaState { private void assertThrottleException(final QuotaLimiter limiter, final int availReqs) { assertNoThrottleException(limiter, availReqs); try { - limiter.checkQuota(1, 1); - fail("Should have thrown ThrottlingException"); + limiter.checkQuota(1, 1, 0, 0); + fail("Should have thrown RpcThrottlingException"); } catch (RpcThrottlingException e) { // expected } @@ -224,11 +252,11 @@ public class TestQuotaState { private void assertNoThrottleException(final QuotaLimiter limiter, final int availReqs) { for (int i = 0; i < availReqs; ++i) { try { - limiter.checkQuota(1, 1); + limiter.checkQuota(1, 1, 0, 0); } catch (RpcThrottlingException e) { - fail("Unexpected ThrottlingException after " + i + " requests. limit=" + availReqs); + fail("Unexpected RpcThrottlingException after " + i + " requests. limit=" + availReqs); } - limiter.grabQuota(1, 1); + limiter.grabQuota(1, 1, 0, 0); } }