Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 940E117BD0 for ; Fri, 25 Sep 2015 18:09:53 +0000 (UTC) Received: (qmail 68387 invoked by uid 500); 25 Sep 2015 18:09:53 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 68346 invoked by uid 500); 25 Sep 2015 18:09:53 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 68337 invoked by uid 99); 25 Sep 2015 18:09:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 25 Sep 2015 18:09:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DA884E0332; Fri, 25 Sep 2015 18:09:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: enis@apache.org To: commits@hbase.apache.org Message-Id: <0f900edf70a543fea62dda5adac8e395@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-14474 DeadLock in RpcClientImpl.Connection.close() Date: Fri, 25 Sep 2015 18:09:52 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/branch-1.2 db72f0a37 -> ff4d93e59 HBASE-14474 DeadLock in RpcClientImpl.Connection.close() Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff4d93e5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff4d93e5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff4d93e5 Branch: refs/heads/branch-1.2 Commit: ff4d93e59d7393918eeedf12108fb4c371bf0402 Parents: db72f0a Author: Enis Soztutar Authored: Fri Sep 25 11:04:53 2015 -0700 Committer: Enis Soztutar Committed: Fri Sep 25 11:09:14 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hbase/ipc/RpcClientImpl.java | 25 +++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/ff4d93e5/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java index b09674c..cb18952 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java @@ -914,14 +914,20 @@ public class RpcClientImpl extends AbstractRpcClient { IPCUtil.write(this.out, header, call.param, cellBlock); } catch (IOException e) { // We set the value inside the synchronized block, this way the next in line - // won't even try to write - markClosed(e); - close(); + // won't even try to write. Otherwise we might miss a call in the calls map? + shouldCloseConnection.set(true); writeException = e; interrupt(); } } + // call close outside of the synchronized (outLock) to prevent deadlock - HBASE-14474 + if (writeException != null) { + if (markClosed(writeException)) { + close(); + } + } + // We added a call, and may be started the connection close. In both cases, we // need to notify the reader. synchronized (this) { @@ -1022,10 +1028,11 @@ public class RpcClientImpl extends AbstractRpcClient { e.getStackTrace(), doNotRetry); } - protected synchronized void markClosed(IOException e) { + protected synchronized boolean markClosed(IOException e) { if (e == null) throw new NullPointerException(); - if (shouldCloseConnection.compareAndSet(false, true)) { + boolean ret = shouldCloseConnection.compareAndSet(false, true); + if (ret) { if (LOG.isTraceEnabled()) { LOG.trace(getName() + ": marking at should close, reason: " + e.getMessage()); } @@ -1034,6 +1041,7 @@ public class RpcClientImpl extends AbstractRpcClient { } notifyAll(); } + return ret; } @@ -1142,14 +1150,15 @@ public class RpcClientImpl extends AbstractRpcClient { } if (connsToClose != null) { for (Connection conn : connsToClose) { - conn.markClosed(new InterruptedIOException("RpcClient is closing")); - conn.close(); + if (conn.markClosed(new InterruptedIOException("RpcClient is closing"))) { + conn.close(); + } } } // wait until all connections are closed while (!connections.isEmpty()) { try { - Thread.sleep(100); + Thread.sleep(10); } catch (InterruptedException e) { LOG.info("Interrupted while stopping the client. We still have " + connections.size() + " connections.");