Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6D95A1865B for ; Mon, 6 Jul 2015 20:27:48 +0000 (UTC) Received: (qmail 70374 invoked by uid 500); 6 Jul 2015 20:27:48 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 70328 invoked by uid 500); 6 Jul 2015 20:27:48 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 70319 invoked by uid 99); 6 Jul 2015 20:27:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Jul 2015 20:27:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 31E9BE0419; Mon, 6 Jul 2015 20:27:48 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: busbey@apache.org To: commits@hbase.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-14017 Procedure v2 - MasterProcedureQueue fix concurrency issue on table queue deletion Date: Mon, 6 Jul 2015 20:27:48 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/master 608c3aa15 -> 1713f1fca HBASE-14017 Procedure v2 - MasterProcedureQueue fix concurrency issue on table queue deletion Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1713f1fc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1713f1fc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1713f1fc Branch: refs/heads/master Commit: 1713f1fcaf9d721a97bc564faaf070f2e6b0b1d1 Parents: 608c3aa Author: Matteo Bertozzi Authored: Mon Jul 6 11:36:20 2015 -0700 Committer: Sean Busbey Committed: Mon Jul 6 15:05:16 2015 -0500 ---------------------------------------------------------------------- .../master/procedure/MasterProcedureQueue.java | 14 ++++-- .../procedure/TestMasterProcedureQueue.java | 51 ++++++++++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/1713f1fc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureQueue.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureQueue.java index af9eecf..c4c7747 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureQueue.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureQueue.java @@ -67,7 +67,7 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { * server that was carrying meta should rise to the top of the queue (this is how it used to * work when we had handlers and ServerShutdownHandler ran). TODO: special handling of servers * that were carrying system tables on crash; do I need to have these servers have priority? - * + * *

Apart from the special-casing of meta and system tables, fairq is what we want */ private final ProcedureFairRunQueues serverFairQ; @@ -366,7 +366,7 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { if (queue != null) { lock.lock(); try { - if (queue.isEmpty() && !queue.isLocked()) { + if (queue.isEmpty() && queue.acquireDeleteLock()) { tableFairQ.remove(table); // Remove the table lock @@ -390,7 +390,7 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { void addFront(Procedure proc); void addBack(Procedure proc); Long poll(); - boolean isLocked(); + boolean acquireDeleteLock(); } /** @@ -443,6 +443,10 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { } @Override + public synchronized boolean acquireDeleteLock() { + return tryExclusiveLock(); + } + public synchronized boolean isLocked() { return isExclusiveLock() || sharedLock > 0; } @@ -477,7 +481,7 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { public synchronized void releaseExclusiveLock() { exclusiveLock = false; } - + @Override public String toString() { return this.runnables.toString(); @@ -571,4 +575,4 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { } } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hbase/blob/1713f1fc/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureQueue.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureQueue.java index 3fa4f0e..7e6e356 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureQueue.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureQueue.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; @@ -66,6 +67,56 @@ public class TestMasterProcedureQueue { assertEquals(0, queue.size()); } + @Test + public void testConcurrentCreateDelete() throws Exception { + final MasterProcedureQueue procQueue = queue; + final TableName table = TableName.valueOf("testtb"); + final AtomicBoolean running = new AtomicBoolean(true); + final AtomicBoolean failure = new AtomicBoolean(false); + Thread createThread = new Thread() { + @Override + public void run() { + try { + while (running.get() && !failure.get()) { + if (procQueue.tryAcquireTableExclusiveLock(table, "create")) { + procQueue.releaseTableExclusiveLock(table); + } + } + } catch (Throwable e) { + LOG.error("create failed", e); + failure.set(true); + } + } + }; + + Thread deleteThread = new Thread() { + @Override + public void run() { + try { + while (running.get() && !failure.get()) { + if (procQueue.tryAcquireTableExclusiveLock(table, "delete")) { + procQueue.releaseTableExclusiveLock(table); + } + procQueue.markTableAsDeleted(table); + } + } catch (Throwable e) { + LOG.error("delete failed", e); + failure.set(true); + } + } + }; + + createThread.start(); + deleteThread.start(); + for (int i = 0; i < 100 && running.get() && !failure.get(); ++i) { + Thread.sleep(100); + } + running.set(false); + createThread.join(); + deleteThread.join(); + assertEquals(false, failure.get()); + } + /** * Verify simple create/insert/fetch/delete of the table queue. */