Return-Path: Delivered-To: apmail-hive-commits-archive@www.apache.org Received: (qmail 30488 invoked from network); 6 Nov 2010 01:29:32 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 6 Nov 2010 01:29:32 -0000 Received: (qmail 87850 invoked by uid 500); 6 Nov 2010 01:30:03 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 87826 invoked by uid 500); 6 Nov 2010 01:30:03 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 87816 invoked by uid 99); 6 Nov 2010 01:30:03 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 06 Nov 2010 01:30:03 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 06 Nov 2010 01:29:59 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id BBD3F238890D; Sat, 6 Nov 2010 01:28:43 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1031914 - in /hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ ql/src/java/org/apache/... Date: Sat, 06 Nov 2010 01:28:43 -0000 To: commits@hive.apache.org From: heyongqiang@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20101106012843.BBD3F238890D@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: heyongqiang Date: Sat Nov 6 01:28:42 2010 New Revision: 1031914 URL: http://svn.apache.org/viewvc?rev=1031914&view=rev Log: HIVE-1765 Add queryid while locking.(namit via He Yongqiang) Modified: hive/trunk/CHANGES.txt hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LockTableDesc.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowLocksDesc.java hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java hive/trunk/ql/src/test/queries/clientpositive/lock1.q hive/trunk/ql/src/test/queries/clientpositive/lock2.q hive/trunk/ql/src/test/results/clientpositive/lock1.q.out hive/trunk/ql/src/test/results/clientpositive/lock2.q.out Modified: hive/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hive/trunk/CHANGES.txt?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/CHANGES.txt (original) +++ hive/trunk/CHANGES.txt Sat Nov 6 01:28:42 2010 @@ -226,6 +226,9 @@ Trunk - Unreleased HIVE-1768 Update transident_lastDdlTime only if not specified (Paul Yang via namit) + HIVE-1765 Add queryid while locking + (namit via He Yongqiang) + OPTIMIZATIONS BUG FIXES Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Sat Nov 6 01:28:42 2010 @@ -434,12 +434,12 @@ public class Driver implements CommandPr List locks = new LinkedList(); if (t != null) { - locks.add(new LockObject(new HiveLockObject(t), mode)); + locks.add(new LockObject(new HiveLockObject(t, plan.getQueryId()), mode)); return locks; } if (p != null) { - locks.add(new LockObject(new HiveLockObject(p), mode)); + locks.add(new LockObject(new HiveLockObject(p, plan.getQueryId()), mode)); // All the parents are locked in shared mode mode = HiveLockMode.SHARED; @@ -450,11 +450,13 @@ public class Driver implements CommandPr for (int idx = 0; idx < partns.length -1; idx++) { String partn = partns[idx]; partialName += partialName + partn; - locks.add(new LockObject(new HiveLockObject(new DummyPartition(p.getTable().getDbName() + "@" + p.getTable().getTableName() + "@" + partialName)), mode)); + locks.add(new LockObject(new HiveLockObject( + new DummyPartition(p.getTable().getDbName() + "@" + p.getTable().getTableName() + "@" + partialName), + plan.getQueryId()), mode)); partialName += "/"; } - locks.add(new LockObject(new HiveLockObject(p.getTable()), mode)); + locks.add(new LockObject(new HiveLockObject(p.getTable(), plan.getQueryId()), mode)); } return locks; } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Sat Nov 6 01:28:42 2010 @@ -1233,6 +1233,7 @@ public class DDLTask extends Task partSpec = lockTbl.getPartSpec(); if (partSpec == null) { - HiveLock lck = lockMgr.lock(new HiveLockObject(tbl), mode, true); + HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockTbl.getQueryId()), mode, true); if (lck == null) { return 1; } @@ -1330,7 +1335,7 @@ public class DDLTask extends Task partSpec = null; + boolean isExtended = false; - if (ast.getChildCount() == 1) { + if (ast.getChildCount() >= 1) { // table for which show locks is being executed ASTNode tableTypeExpr = (ASTNode) ast.getChild(0); tableName = getFullyQualifiedName((ASTNode)tableTypeExpr.getChild(0)); @@ -932,10 +933,14 @@ public class DDLSemanticAnalyzer extends ASTNode partspec = (ASTNode) tableTypeExpr.getChild(1); partSpec = getPartSpec(partspec); } + + if (ast.getChildCount() >= 2) { + isExtended = (ast.getChild(1).getType() == HiveParser.KW_EXTENDED); + } } ShowLocksDesc showLocksDesc = new ShowLocksDesc(ctx.getResFile(), tableName, - partSpec); + partSpec, isExtended); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), showLocksDesc), conf)); setFetchTask(createFetchTask(showLocksDesc.getSchema())); @@ -966,7 +971,8 @@ public class DDLSemanticAnalyzer extends partSpec = partSpecs.get(0); } - LockTableDesc lockTblDesc = new LockTableDesc(tableName, mode, partSpec); + LockTableDesc lockTblDesc = new LockTableDesc(tableName, mode, partSpec, + HiveConf.getVar(conf, ConfVars.HIVEQUERYID)); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), lockTblDesc), conf)); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g Sat Nov 6 01:28:42 2010 @@ -660,7 +660,7 @@ showStatement | KW_SHOW KW_TABLE KW_EXTENDED ((KW_FROM|KW_IN) db_name=Identifier)? KW_LIKE showStmtIdentifier partitionSpec? -> ^(TOK_SHOW_TABLESTATUS showStmtIdentifier $db_name? partitionSpec?) | KW_SHOW KW_LOCKS -> ^(TOK_SHOWLOCKS) - | KW_SHOW KW_LOCKS (parttype=partTypeExpr) -> ^(TOK_SHOWLOCKS $parttype) + | KW_SHOW KW_LOCKS (parttype=partTypeExpr) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWLOCKS $parttype $isExtended?) ; lockStatement Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LockTableDesc.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LockTableDesc.java?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LockTableDesc.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LockTableDesc.java Sat Nov 6 01:28:42 2010 @@ -34,14 +34,16 @@ public class LockTableDesc extends DDLDe private String tableName; private String mode; private Map partSpec; + private String queryId; public LockTableDesc() { } - public LockTableDesc(String tableName, String mode, Map partSpec) { + public LockTableDesc(String tableName, String mode, Map partSpec, String queryId) { this.tableName = tableName; this.mode = mode; this.partSpec = partSpec; + this.queryId = queryId; } public String getTableName() { @@ -67,4 +69,12 @@ public class LockTableDesc extends DDLDe public void setPartSpec(Map partSpec) { this.partSpec = partSpec; } + + public String getQueryId() { + return queryId; + } + + public void setQueryId(String queryId) { + this.queryId = queryId; + } } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowLocksDesc.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowLocksDesc.java?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowLocksDesc.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowLocksDesc.java Sat Nov 6 01:28:42 2010 @@ -33,6 +33,7 @@ public class ShowLocksDesc extends DDLDe String resFile; String tableName; HashMap partSpec; + boolean isExt; /** * table name for the result of show locks. @@ -58,10 +59,11 @@ public class ShowLocksDesc extends DDLDe * @param resFile */ public ShowLocksDesc(Path resFile, String tableName, - HashMap partSpec) { - this.resFile = resFile.toString(); - this.partSpec = partSpec; + HashMap partSpec, boolean isExt) { + this.resFile = resFile.toString(); + this.partSpec = partSpec; this.tableName = tableName; + this.isExt = isExt; } /** @@ -111,4 +113,19 @@ public class ShowLocksDesc extends DDLDe public void setResFile(String resFile) { this.resFile = resFile; } + + /** + * @return the isExt + */ + public boolean isExt() { + return isExt; + } + + /** + * @param isExt + * the isExt to set + */ + public void setExt(boolean isExt) { + this.isExt = isExt; + } } Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Sat Nov 6 01:28:42 2010 @@ -897,6 +897,7 @@ public class QTestUtil { "-I", "at java", "-I", "at junit", "-I", "Caused by:", + "-I", "QUERYID_LOCK:", "-I", "[.][.][.] [0-9]* more", (new File(logDir, tname + ".out")).getPath(), outFileName }; Modified: hive/trunk/ql/src/test/queries/clientpositive/lock1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/lock1.q?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/lock1.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/lock1.q Sat Nov 6 01:28:42 2010 @@ -8,6 +8,8 @@ SHOW LOCKS tstsrc; LOCK TABLE tstsrc shared; SHOW LOCKS; SHOW LOCKS tstsrc; +SHOW LOCKS tstsrc extended; + UNLOCK TABLE tstsrc; SHOW LOCKS; SHOW LOCKS tstsrc; Modified: hive/trunk/ql/src/test/queries/clientpositive/lock2.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/lock2.q?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/lock2.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/lock2.q Sat Nov 6 01:28:42 2010 @@ -14,6 +14,7 @@ LOCK TABLE tstsrcpart PARTITION(ds='2008 SHOW LOCKS; SHOW LOCKS tstsrcpart; SHOW LOCKS tstsrcpart PARTITION(ds='2008-04-08', hr='11'); +SHOW LOCKS tstsrcpart PARTITION(ds='2008-04-08', hr='11') extended; UNLOCK TABLE tstsrc; SHOW LOCKS; Modified: hive/trunk/ql/src/test/results/clientpositive/lock1.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/lock1.q.out?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/lock1.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/lock1.q.out Sat Nov 6 01:28:42 2010 @@ -49,6 +49,14 @@ POSTHOOK: type: SHOWLOCKS POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] default@tstsrc SHARED +PREHOOK: query: SHOW LOCKS tstsrc extended +PREHOOK: type: SHOWLOCKS +POSTHOOK: query: SHOW LOCKS tstsrc extended +POSTHOOK: type: SHOWLOCKS +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +default@tstsrc SHARED +QUERYID_LOCK:njain_20101104183131_7ee0480d-77f6-4bc2-8cec-26b808b026df PREHOOK: query: UNLOCK TABLE tstsrc PREHOOK: type: UNLOCKTABLE POSTHOOK: query: UNLOCK TABLE tstsrc Modified: hive/trunk/ql/src/test/results/clientpositive/lock2.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/lock2.q.out?rev=1031914&r1=1031913&r2=1031914&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/lock2.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/lock2.q.out Sat Nov 6 01:28:42 2010 @@ -97,6 +97,16 @@ POSTHOOK: Lineage: tstsrc.value SIMPLE [ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] default@tstsrcpart@ds=2008-04-08/hr=11 EXCLUSIVE +PREHOOK: query: SHOW LOCKS tstsrcpart PARTITION(ds='2008-04-08', hr='11') extended +PREHOOK: type: SHOWLOCKS +POSTHOOK: query: SHOW LOCKS tstsrcpart PARTITION(ds='2008-04-08', hr='11') extended +POSTHOOK: type: SHOWLOCKS +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +default@tstsrcpart@ds=2008-04-08/hr=11 EXCLUSIVE +QUERYID_LOCK:njain_20101105120303_b0e12da7-8ccf-4c7a-822f-166d35c6b6d4 PREHOOK: query: UNLOCK TABLE tstsrc PREHOOK: type: UNLOCKTABLE POSTHOOK: query: UNLOCK TABLE tstsrc