Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BA84C1932F for ; Wed, 30 Mar 2016 19:17:28 +0000 (UTC) Received: (qmail 11972 invoked by uid 500); 30 Mar 2016 19:17:28 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 11933 invoked by uid 500); 30 Mar 2016 19:17:28 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 11922 invoked by uid 99); 30 Mar 2016 19:17:28 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Mar 2016 19:17:28 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4F6E8DFC8E; Wed, 30 Mar 2016 19:17:28 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ekoifman@apache.org To: commits@hive.apache.org Message-Id: <609df294d0de4d40bfe5c5bc8712c3cf@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-10249 ACID: show locks should show who the lock is waiting for (Eugene Koifman, reviewed by Wei Zheng) Date: Wed, 30 Mar 2016 19:17:28 +0000 (UTC) Repository: hive Updated Branches: refs/heads/master 51efcb80e -> 4e9f95a1b HIVE-10249 ACID: show locks should show who the lock is waiting for (Eugene Koifman, reviewed by Wei Zheng) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4e9f95a1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4e9f95a1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4e9f95a1 Branch: refs/heads/master Commit: 4e9f95a1bad89ac4ea0cefc65eeba7a1e56a948d Parents: 51efcb8 Author: Eugene Koifman Authored: Wed Mar 30 12:17:06 2016 -0700 Committer: Eugene Koifman Committed: Wed Mar 30 12:17:06 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hive/metastore/txn/TxnDbUtil.java | 6 ++- .../hadoop/hive/metastore/txn/TxnHandler.java | 46 ++++++++++++++++---- .../hive/metastore/txn/TestTxnHandler.java | 2 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 16 ++++++- .../hive/ql/lockmgr/TestDbTxnManager2.java | 28 ++++++++++++ .../clientpositive/dbtxnmgr_showlocks.q.out | 6 +-- 6 files changed, 89 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index df480ea..c82d23a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -103,7 +103,11 @@ public final class TxnDbUtil { " HL_ACQUIRED_AT bigint," + " HL_USER varchar(128) NOT NULL," + " HL_HOST varchar(128) NOT NULL," + - " PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID))"); + " HL_HEARTBEAT_COUNT integer," + + " HL_AGENT_INFO varchar(128)," + + " HL_BLOCKEDBY_EXT_ID bigint," + + " HL_BLOCKEDBY_INT_ID bigint," + + " PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID))"); stmt.execute("CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID)"); stmt.execute("CREATE TABLE NEXT_LOCK_ID (" + " NL_NEXT bigint NOT NULL)"); http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 21faff4..be3c6de 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -847,8 +847,8 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { */ private static class LockInfoExt extends LockInfo { private final ShowLocksResponseElement e; - LockInfoExt(ShowLocksResponseElement e, long intLockId) { - super(e, intLockId); + LockInfoExt(ShowLocksResponseElement e) { + super(e); this.e = e; } } @@ -864,7 +864,8 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { stmt = dbConn.createStatement(); String s = "select hl_lock_ext_id, hl_txnid, hl_db, hl_table, hl_partition, hl_lock_state, " + - "hl_lock_type, hl_last_heartbeat, hl_acquired_at, hl_user, hl_host, hl_lock_int_id from HIVE_LOCKS"; + "hl_lock_type, hl_last_heartbeat, hl_acquired_at, hl_user, hl_host, hl_lock_int_id," + + "hl_blockedby_ext_id, hl_blockedby_int_id from HIVE_LOCKS"; LOG.debug("Doing to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); while (rs.next()) { @@ -892,7 +893,16 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { if (!rs.wasNull()) e.setAcquiredat(acquiredAt); e.setUser(rs.getString(10)); e.setHostname(rs.getString(11)); - sortedList.add(new LockInfoExt(e, rs.getLong(12))); + e.setLockIdInternal(rs.getLong(12)); + long id = rs.getLong(13); + if(!rs.wasNull()) { + e.setBlockedByExtId(id); + } + id = rs.getLong(14); + if(!rs.wasNull()) { + e.setBlockedByIntId(id); + } + sortedList.add(new LockInfoExt(e)); } LOG.debug("Going to rollback"); dbConn.rollback(); @@ -1142,6 +1152,10 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { private static void shouldNeverHappen(long txnid) { throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid)); } + private static void shouldNeverHappen(long txnid, long extLockId, long intLockId) { + throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid) + " " + + JavaUtils.lockIdToString(extLockId) + " " + intLockId); + } public void addDynamicPartitions(AddDynamicPartitions rqst) throws NoSuchTxnException, TxnAbortedException, MetaException { Connection dbConn = null; @@ -1673,15 +1687,15 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { } txnId = rs.getLong("hl_txnid");//returns 0 if value is NULL } - LockInfo(ShowLocksResponseElement e, long intLockId) { + LockInfo(ShowLocksResponseElement e) { extLockId = e.getLockid(); - this.intLockId = intLockId; + intLockId = e.getLockIdInternal(); + txnId = e.getTxnid(); db = e.getDbname(); table = e.getTablename(); partition = e.getPartname(); state = e.getState(); type = e.getType(); - txnId = e.getTxnid(); } public boolean equals(Object other) { @@ -1980,9 +1994,10 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { LOG.debug("Going to execute query <" + query.toString() + ">"); Statement stmt = null; + ResultSet rs = null; try { stmt = dbConn.createStatement(); - ResultSet rs = stmt.executeQuery(query.toString()); + rs = stmt.executeQuery(query.toString()); SortedSet lockSet = new TreeSet(new LockInfoComparator()); while (rs.next()) { lockSet.add(new LockInfo(rs)); @@ -2053,7 +2068,20 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { switch (lockAction) { case WAIT: if(!ignoreConflict(info, locks[i])) { + /*we acquire all locks for a given query atomically; if 1 blocks, all go into (remain) in + * Waiting state. wait() will undo any 'acquire()' which may have happened as part of + * this (metastore db) transaction and then we record which lock blocked the lock + * we were testing ('info').*/ wait(dbConn, save); + String sqlText = "update HIVE_LOCKS" + + " set HL_BLOCKEDBY_EXT_ID=" + locks[i].extLockId + + ", HL_BLOCKEDBY_INT_ID=" + locks[i].intLockId + + " where HL_LOCK_EXT_ID=" + info.extLockId + " and HL_LOCK_INT_ID=" + info.intLockId; + LOG.debug("Executing sql: " + sqlText); + int updCnt = stmt.executeUpdate(sqlText); + if(updCnt != 1) { + shouldNeverHappen(info.txnId, info.extLockId, info.intLockId); + } LOG.debug("Going to commit"); dbConn.commit(); response.setState(LockState.WAITING); @@ -2082,7 +2110,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { dbConn.commit(); response.setState(LockState.ACQUIRED); } finally { - closeStmt(stmt); + close(rs, stmt, null); } return response; } http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java index 26a660a..37eacde 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java @@ -1153,7 +1153,7 @@ public class TestTxnHandler { } @Test - @Ignore + @Ignore("Wedges Derby") public void deadlockDetected() throws Exception { LOG.debug("Starting deadlock test"); if (txnHandler instanceof TxnHandler) { http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 56eecf6..b26f09d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -2541,6 +2541,8 @@ public class DDLTask extends Task implements Serializable { os.write(separator); os.writeBytes("State"); os.write(separator); + os.writeBytes("Blocked By"); + os.write(separator); os.writeBytes("Type"); os.write(separator); os.writeBytes("Transaction ID"); @@ -2557,7 +2559,12 @@ public class DDLTask extends Task implements Serializable { List locks = rsp.getLocks(); if (locks != null) { for (ShowLocksResponseElement lock : locks) { - os.writeBytes(Long.toString(lock.getLockid())); + if(lock.isSetLockIdInternal()) { + os.writeBytes(Long.toString(lock.getLockid()) + "." + Long.toString(lock.getLockIdInternal())); + } + else { + os.writeBytes(Long.toString(lock.getLockid())); + } os.write(separator); os.writeBytes(lock.getDbname()); os.write(separator); @@ -2567,6 +2574,13 @@ public class DDLTask extends Task implements Serializable { os.write(separator); os.writeBytes(lock.getState().toString()); os.write(separator); + if(lock.isSetBlockedByExtId()) {//both "blockedby" are either there or not + os.writeBytes(Long.toString(lock.getBlockedByExtId()) + "." + Long.toString(lock.getBlockedByIntId())); + } + else { + os.writeBytes(" ");//12 chars - try to keep cols aligned + } + os.write(separator); os.writeBytes(lock.getType().toString()); os.write(separator); os.writeBytes((lock.getTxnid() == 0) ? "NULL" : Long.toString(lock.getTxnid())); http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index d1b370e..836b507 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -240,6 +240,34 @@ public class TestDbTxnManager2 { otherTxnMgr.closeTxnManager(); } + /** + * check that locks in Waiting state show what they are waiting on + * This test is somewhat abusive in that it make DbLockManager retain locks for 2 + * different queries (which are not part of the same transaction) which can never + * happen in real use cases... but it makes testing convenient. + * @throws Exception + */ + @Test + public void testLockBlockedBy() throws Exception { + CommandProcessorResponse cpr = driver.run("create table TAB_BLOCKED (a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); + checkCmdOnDriver(cpr); + cpr = driver.compileAndRespond("select * from TAB_BLOCKED"); + checkCmdOnDriver(cpr); + txnMgr.acquireLocks(driver.getPlan(), ctx, "I AM SAM"); + List locks = getLocks(txnMgr); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks.get(0)); + cpr = driver.compileAndRespond("drop table TAB_BLOCKED"); + checkCmdOnDriver(cpr); + ((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(), ctx, "SAM I AM", false);//make non-blocking + locks = getLocks(txnMgr); + Assert.assertEquals("Unexpected lock count", 2, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks.get(0)); + checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "TAB_BLOCKED", null, locks.get(1)); + Assert.assertEquals("BlockedByExtId doesn't match", locks.get(0).getLockid(), locks.get(1).getBlockedByExtId()); + Assert.assertEquals("BlockedByIntId doesn't match", locks.get(0).getLockIdInternal(), locks.get(1).getBlockedByIntId()); + } + @Test public void testDummyTxnManagerOnAcidTable() throws Exception { // Create an ACID table with DbTxnManager http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out index d9d2ed6..46d8ea1 100644 --- a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out +++ b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out @@ -2,17 +2,17 @@ PREHOOK: query: show locks PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks POSTHOOK: type: SHOWLOCKS -Lock ID Database Table Partition State Type Transaction ID Last Hearbeat Acquired At User Hostname +Lock ID Database Table Partition State Blocked By Type Transaction ID Last Hearbeat Acquired At User PREHOOK: query: show locks extended PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks extended POSTHOOK: type: SHOWLOCKS -Lock ID Database Table Partition State Type Transaction ID Last Hearbeat Acquired At User Hostname +Lock ID Database Table Partition State Blocked By Type Transaction ID Last Hearbeat Acquired At User PREHOOK: query: show locks default PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks default POSTHOOK: type: SHOWLOCKS -Lock ID Database Table Partition State Type Transaction ID Last Hearbeat Acquired At User Hostname +Lock ID Database Table Partition State Blocked By Type Transaction ID Last Hearbeat Acquired At User PREHOOK: query: show transactions PREHOOK: type: SHOW TRANSACTIONS POSTHOOK: query: show transactions