Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 23B2D116E5 for ; Sat, 28 Jun 2014 00:24:40 +0000 (UTC) Received: (qmail 57410 invoked by uid 500); 28 Jun 2014 00:24:40 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 57382 invoked by uid 500); 28 Jun 2014 00:24:40 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 57371 invoked by uid 99); 28 Jun 2014 00:24:39 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 28 Jun 2014 00:24:39 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 28 Jun 2014 00:24:28 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 70C132388994; Sat, 28 Jun 2014 00:24:02 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1606275 [2/3] - in /hive/branches/cbo: ./ common/src/java/org/apache/hadoop/hive/conf/ conf/ contrib/src/test/queries/clientnegative/ contrib/src/test/queries/clientpositive/ data/conf/ hbase-handler/src/java/org/apache/hadoop/hive/hbase/ ... Date: Sat, 28 Jun 2014 00:23:57 -0000 To: commits@hive.apache.org From: gunther@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140628002402.70C132388994@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java (original) +++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java Sat Jun 28 00:23:54 2014 @@ -120,8 +120,9 @@ public class TxnHandler { // database we'll look at the current transaction number first. If it // subsequently shows up in the open list that's ok. Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + Statement stmt = null; try { - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); String s = "select ntxn_next - 1 from NEXT_TXN_ID"; LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); @@ -157,7 +158,6 @@ public class TxnHandler { } txnInfo.add(new TxnInfo(rs.getLong(1), state, rs.getString(3), rs.getString(4))); } - stmt.close(); LOG.debug("Going to rollback"); dbConn.rollback(); return new GetOpenTxnsInfoResponse(hwm, txnInfo); @@ -170,6 +170,7 @@ public class TxnHandler { throw new MetaException("Unable to select from transaction database, " + StringUtils.stringifyException(e)); } finally { + closeStmt(stmt); closeDbConn(dbConn); } } @@ -180,9 +181,10 @@ public class TxnHandler { // database we'll look at the current transaction number first. If it // subsequently shows up in the open list that's ok. Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + Statement stmt = null; try { timeOutTxns(dbConn); - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); String s = "select ntxn_next - 1 from NEXT_TXN_ID"; LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); @@ -203,7 +205,6 @@ public class TxnHandler { while (rs.next()) { openList.add(rs.getLong(1)); } - stmt.close(); LOG.debug("Going to rollback"); dbConn.rollback(); return new GetOpenTxnsResponse(hwm, openList); @@ -216,6 +217,7 @@ public class TxnHandler { throw new MetaException("Unable to select from transaction database, " + StringUtils.stringifyException(e)); } finally { + closeStmt(stmt); closeDbConn(dbConn); } } @@ -235,13 +237,14 @@ public class TxnHandler { int numTxns = rqst.getNum_txns(); try { Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Statement stmt = null; try { // Make sure the user has not requested an insane amount of txns. int maxTxns = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_TXN_MAX_OPEN_BATCH); if (numTxns > maxTxns) numTxns = maxTxns; - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); String s = "select ntxn_next from NEXT_TXN_ID"; LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); @@ -279,6 +282,7 @@ public class TxnHandler { throw new MetaException("Unable to select from transaction database " + StringUtils.stringifyException(e)); } finally { + closeStmt(stmt); closeDbConn(dbConn); } } catch (DeadlockException e) { @@ -327,8 +331,9 @@ public class TxnHandler { long txnid = rqst.getTxnid(); try { Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Statement stmt = null; try { - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); // Before we do the commit heartbeat the txn. This is slightly odd in that we're going to // commit it, but it does two things. One, it makes sure the transaction is still valid. // Two, it avoids the race condition where we time out between now and when we actually @@ -369,6 +374,7 @@ public class TxnHandler { throw new MetaException("Unable to update transaction database " + StringUtils.stringifyException(e)); } finally { + closeStmt(stmt); closeDbConn(dbConn); } } catch (DeadlockException e) { @@ -468,6 +474,7 @@ public class TxnHandler { throws NoSuchLockException, TxnOpenException, MetaException { try { Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Statement stmt = null; try { // Odd as it seems, we need to heartbeat first because this touches the // lock table and assures that our locks our still valid. If they are @@ -487,7 +494,7 @@ public class TxnHandler { LOG.error(msg); throw new TxnOpenException(msg); } - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); String s = "delete from HIVE_LOCKS where hl_lock_ext_id = " + extLockId; LOG.debug("Going to execute update <" + s + ">"); int rc = stmt.executeUpdate(s); @@ -508,6 +515,7 @@ public class TxnHandler { throw new MetaException("Unable to update transaction database " + StringUtils.stringifyException(e)); } finally { + closeStmt(stmt); closeDbConn(dbConn); } } catch (DeadlockException e) { @@ -521,8 +529,9 @@ public class TxnHandler { Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); ShowLocksResponse rsp = new ShowLocksResponse(); List elems = new ArrayList(); + Statement stmt = null; try { - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); String s = "select hl_lock_ext_id, hl_txnid, hl_db, hl_table, hl_partition, hl_lock_state, " + "hl_lock_type, hl_last_heartbeat, hl_acquired_at, hl_user, hl_host from HIVE_LOCKS"; @@ -561,6 +570,7 @@ public class TxnHandler { throw new MetaException("Unable to select from transaction database " + StringUtils.stringifyException(e)); } finally { + closeStmt(stmt); closeDbConn(dbConn); } rsp.setLocks(elems); @@ -634,8 +644,9 @@ public class TxnHandler { // Put a compaction request in the queue. try { Connection dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE); + Statement stmt = null; try { - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); // Get the id for the next entry in the queue String s = "select ncq_next from NEXT_COMPACTION_QUEUE_ID"; @@ -705,6 +716,7 @@ public class TxnHandler { throw new MetaException("Unable to select from transaction database " + StringUtils.stringifyException(e)); } finally { + closeStmt(stmt); closeDbConn(dbConn); } } catch (DeadlockException e) { @@ -717,8 +729,9 @@ public class TxnHandler { public ShowCompactResponse showCompact(ShowCompactRequest rqst) throws MetaException { ShowCompactResponse response = new ShowCompactResponse(); Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + Statement stmt = null; try { - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); String s = "select cq_database, cq_table, cq_partition, cq_state, cq_type, cq_worker_id, " + "cq_start, cq_run_as from COMPACTION_QUEUE"; LOG.debug("Going to execute query <" + s + ">"); @@ -755,6 +768,7 @@ public class TxnHandler { throw new MetaException("Unable to select from transaction database " + StringUtils.stringifyException(e)); } finally { + closeStmt(stmt); closeDbConn(dbConn); } return response; @@ -765,8 +779,9 @@ public class TxnHandler { */ int numLocksInLockTable() throws SQLException, MetaException { Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + Statement stmt = null; try { - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); String s = "select count(*) from HIVE_LOCKS"; LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); @@ -777,6 +792,7 @@ public class TxnHandler { return rc; } finally { closeDbConn(dbConn); + closeStmt(stmt); } } @@ -819,6 +835,18 @@ public class TxnHandler { LOG.warn("Failed to close db connection " + e.getMessage()); } } + + /** + * Close statement instance. + * @param stmt statement instance. + */ + protected void closeStmt(Statement stmt) { + try { + if (stmt != null) stmt.close(); + } catch (SQLException e) { + LOG.warn("Failed to close statement " + e.getMessage()); + } + } /** * Determine if an exception was a deadlock. Unfortunately there is no standard way to do @@ -850,10 +878,10 @@ public class TxnHandler { * @throws org.apache.hadoop.hive.metastore.api.MetaException if the time cannot be determined */ protected long getDbTime(Connection conn) throws MetaException { + Statement stmt = null; try { - Statement stmt = conn.createStatement(); + stmt = conn.createStatement(); String s; - ResultSet rs; DatabaseProduct prod = determineDatabaseProduct(conn); switch (prod) { case DERBY: @@ -876,13 +904,15 @@ public class TxnHandler { throw new MetaException(msg); } LOG.debug("Going to execute query <" + s + ">"); - rs = stmt.executeQuery(s); + ResultSet rs = stmt.executeQuery(s); if (!rs.next()) throw new MetaException("No results from date query"); return rs.getTimestamp(1).getTime(); } catch (SQLException e) { String msg = "Unable to determine current time: " + e.getMessage(); LOG.error(msg); throw new MetaException(msg); + } finally { + closeStmt(stmt); } } @@ -1042,33 +1072,39 @@ public class TxnHandler { * @throws SQLException */ private int abortTxns(Connection dbConn, List txnids) throws SQLException { - Statement stmt = dbConn.createStatement(); - - // delete from HIVE_LOCKS first, we always access HIVE_LOCKS before TXNS - StringBuilder buf = new StringBuilder("delete from HIVE_LOCKS where hl_txnid in ("); - boolean first = true; - for (Long id : txnids) { - if (first) first = false; - else buf.append(','); - buf.append(id); - } - buf.append(')'); - LOG.debug("Going to execute update <" + buf.toString() + ">"); - stmt.executeUpdate(buf.toString()); - - buf = new StringBuilder("update TXNS set txn_state = '" + TXN_ABORTED + "' where txn_id in ("); - first = true; - for (Long id : txnids) { - if (first) first = false; - else buf.append(','); - buf.append(id); + Statement stmt = null; + int updateCnt = 0; + try { + stmt = dbConn.createStatement(); + + // delete from HIVE_LOCKS first, we always access HIVE_LOCKS before TXNS + StringBuilder buf = new StringBuilder("delete from HIVE_LOCKS where hl_txnid in ("); + boolean first = true; + for (Long id : txnids) { + if (first) first = false; + else buf.append(','); + buf.append(id); + } + buf.append(')'); + LOG.debug("Going to execute update <" + buf.toString() + ">"); + stmt.executeUpdate(buf.toString()); + + buf = new StringBuilder("update TXNS set txn_state = '" + TXN_ABORTED + "' where txn_id in ("); + first = true; + for (Long id : txnids) { + if (first) first = false; + else buf.append(','); + buf.append(id); + } + buf.append(')'); + LOG.debug("Going to execute update <" + buf.toString() + ">"); + updateCnt = stmt.executeUpdate(buf.toString()); + + LOG.debug("Going to commit"); + dbConn.commit(); + } finally { + closeStmt(stmt); } - buf.append(')'); - LOG.debug("Going to execute update <" + buf.toString() + ">"); - int updateCnt = stmt.executeUpdate(buf.toString()); - - LOG.debug("Going to commit"); - dbConn.commit(); return updateCnt; } @@ -1102,9 +1138,9 @@ public class TxnHandler { synchronized (lockLock) { // Clean up timed out locks before we attempt to acquire any. timeOutLocks(dbConn); - + Statement stmt = null; try { - Statement stmt = dbConn.createStatement(); + stmt = dbConn.createStatement(); // Get the next lock id. String s = "select nl_next from NEXT_LOCK_ID"; @@ -1183,6 +1219,8 @@ public class TxnHandler { } catch (NoSuchLockException e) { // This should never happen, as we just added the lock id throw new MetaException("Couldn't find a lock we just created!"); + } finally { + closeStmt(stmt); } } } @@ -1197,7 +1235,6 @@ public class TxnHandler { LOG.debug("Setting savepoint"); Savepoint save = dbConn.setSavepoint(); - Statement stmt = dbConn.createStatement(); StringBuilder query = new StringBuilder("select hl_lock_ext_id, " + "hl_lock_int_id, hl_db, hl_table, hl_partition, hl_lock_state, " + "hl_lock_type from HIVE_LOCKS where hl_db in ("); @@ -1267,98 +1304,104 @@ public class TxnHandler { } LOG.debug("Going to execute query <" + query.toString() + ">"); - ResultSet rs = stmt.executeQuery(query.toString()); - SortedSet lockSet = new TreeSet(new LockInfoComparator()); - while (rs.next()) { - lockSet.add(new LockInfo(rs)); - } - // Turn the tree set into an array so we can move back and forth easily - // in it. - LockInfo[] locks = (LockInfo[])lockSet.toArray(new LockInfo[1]); - - for (LockInfo info : locksBeingChecked) { - // Find the lock record we're checking - int index = -1; - for (int i = 0; i < locks.length; i++) { - if (locks[i].equals(info)) { - index = i; - break; - } + Statement stmt = null; + try { + stmt = dbConn.createStatement(); + ResultSet rs = stmt.executeQuery(query.toString()); + SortedSet lockSet = new TreeSet(new LockInfoComparator()); + while (rs.next()) { + lockSet.add(new LockInfo(rs)); } + // Turn the tree set into an array so we can move back and forth easily + // in it. + LockInfo[] locks = (LockInfo[])lockSet.toArray(new LockInfo[1]); - // If we didn't find the lock, then it must not be in the table - if (index == -1) { - LOG.debug("Going to rollback"); - dbConn.rollback(); - throw new MetaException("How did we get here, we heartbeated our lock before we started!"); - } + for (LockInfo info : locksBeingChecked) { + // Find the lock record we're checking + int index = -1; + for (int i = 0; i < locks.length; i++) { + if (locks[i].equals(info)) { + index = i; + break; + } + } + // If we didn't find the lock, then it must not be in the table + if (index == -1) { + LOG.debug("Going to rollback"); + dbConn.rollback(); + throw new MetaException("How did we get here, we heartbeated our lock before we started!"); + } - // If we've found it and it's already been marked acquired, - // then just look at the other locks. - if (locks[index].state == LockState.ACQUIRED) { - continue; - } - // Look at everything in front of this lock to see if it should block - // it or not. - boolean acquired = false; - for (int i = index - 1; i >= 0; i--) { - // Check if we're operating on the same database, if not, move on - if (!locks[index].db.equals(locks[i].db)) { + // If we've found it and it's already been marked acquired, + // then just look at the other locks. + if (locks[index].state == LockState.ACQUIRED) { continue; } - // If table is null on either of these, then they are claiming to - // lock the whole database and we need to check it. Otherwise, - // check if they are operating on the same table, if not, move on. - if (locks[index].table != null && locks[i].table != null - && !locks[index].table.equals(locks[i].table)) { - continue; - } + // Look at everything in front of this lock to see if it should block + // it or not. + boolean acquired = false; + for (int i = index - 1; i >= 0; i--) { + // Check if we're operating on the same database, if not, move on + if (!locks[index].db.equals(locks[i].db)) { + continue; + } - // If partition is null on either of these, then they are claiming to - // lock the whole table and we need to check it. Otherwise, - // check if they are operating on the same partition, if not, move on. - if (locks[index].partition != null && locks[i].partition != null - && !locks[index].partition.equals(locks[i].partition)) { - continue; - } + // If table is null on either of these, then they are claiming to + // lock the whole database and we need to check it. Otherwise, + // check if they are operating on the same table, if not, move on. + if (locks[index].table != null && locks[i].table != null + && !locks[index].table.equals(locks[i].table)) { + continue; + } - // We've found something that matches what we're trying to lock, - // so figure out if we can lock it too. - switch (jumpTable.get(locks[index].type).get(locks[i].type).get - (locks[i].state)) { - case ACQUIRE: - acquire(dbConn, stmt, extLockId, info.intLockId); - acquired = true; - break; - case WAIT: - wait(dbConn, save); - if (alwaysCommit) { - // In the case where lockNoWait has been called we don't want to commit because - // it's going to roll everything back. In every other case we want to commit here. - LOG.debug("Going to commit"); - dbConn.commit(); - } - response.setState(LockState.WAITING); - return response; - case KEEP_LOOKING: + // If partition is null on either of these, then they are claiming to + // lock the whole table and we need to check it. Otherwise, + // check if they are operating on the same partition, if not, move on. + if (locks[index].partition != null && locks[i].partition != null + && !locks[index].partition.equals(locks[i].partition)) { continue; + } + + // We've found something that matches what we're trying to lock, + // so figure out if we can lock it too. + switch (jumpTable.get(locks[index].type).get(locks[i].type).get + (locks[i].state)) { + case ACQUIRE: + acquire(dbConn, stmt, extLockId, info.intLockId); + acquired = true; + break; + case WAIT: + wait(dbConn, save); + if (alwaysCommit) { + // In the case where lockNoWait has been called we don't want to commit because + // it's going to roll everything back. In every other case we want to commit here. + LOG.debug("Going to commit"); + dbConn.commit(); + } + response.setState(LockState.WAITING); + return response; + case KEEP_LOOKING: + continue; + } + if (acquired) break; // We've acquired this lock component, + // so get out of the loop and look at the next component. } - if (acquired) break; // We've acquired this lock component, - // so get out of the loop and look at the next component. + + // If we've arrived here and we have not already acquired, it means there's nothing in the + // way of the lock, so acquire the lock. + if (!acquired) acquire(dbConn, stmt, extLockId, info.intLockId); } - // If we've arrived here and we have not already acquired, it means there's nothing in the - // way of the lock, so acquire the lock. - if (!acquired) acquire(dbConn, stmt, extLockId, info.intLockId); + // We acquired all of the locks, so commit and return acquired. + LOG.debug("Going to commit"); + dbConn.commit(); + response.setState(LockState.ACQUIRED); + } finally { + closeStmt(stmt); } - - // We acquired all of the locks, so commit and return acquired. - LOG.debug("Going to commit"); - dbConn.commit(); - response.setState(LockState.ACQUIRED); return response; } @@ -1397,20 +1440,25 @@ public class TxnHandler { throws NoSuchLockException, SQLException, MetaException { // If the lock id is 0, then there are no locks in this heartbeat if (extLockId == 0) return; - Statement stmt = dbConn.createStatement(); - long now = getDbTime(dbConn); + Statement stmt = null; + try { + stmt = dbConn.createStatement(); + long now = getDbTime(dbConn); - String s = "update HIVE_LOCKS set hl_last_heartbeat = " + - now + " where hl_lock_ext_id = " + extLockId; - LOG.debug("Going to execute update <" + s + ">"); - int rc = stmt.executeUpdate(s); - if (rc < 1) { - LOG.debug("Going to rollback"); - dbConn.rollback(); - throw new NoSuchLockException("No such lock: " + extLockId); + String s = "update HIVE_LOCKS set hl_last_heartbeat = " + + now + " where hl_lock_ext_id = " + extLockId; + LOG.debug("Going to execute update <" + s + ">"); + int rc = stmt.executeUpdate(s); + if (rc < 1) { + LOG.debug("Going to rollback"); + dbConn.rollback(); + throw new NoSuchLockException("No such lock: " + extLockId); + } + LOG.debug("Going to commit"); + dbConn.commit(); + } finally { + closeStmt(stmt); } - LOG.debug("Going to commit"); - dbConn.commit(); } // Heartbeats on the txn table. This commits, so do not enter it with any state @@ -1418,68 +1466,83 @@ public class TxnHandler { throws NoSuchTxnException, TxnAbortedException, SQLException, MetaException { // If the txnid is 0, then there are no transactions in this heartbeat if (txnid == 0) return; - Statement stmt = dbConn.createStatement(); - long now = getDbTime(dbConn); - // We need to check whether this transaction is valid and open - String s = "select txn_state from TXNS where txn_id = " + txnid; - LOG.debug("Going to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - if (!rs.next()) { - LOG.debug("Going to rollback"); - dbConn.rollback(); - throw new NoSuchTxnException("No such transaction: " + txnid); - } - if (rs.getString(1).charAt(0) == TXN_ABORTED) { - LOG.debug("Going to rollback"); - dbConn.rollback(); - throw new TxnAbortedException("Transaction " + txnid + - " already aborted"); + Statement stmt = null; + try { + stmt = dbConn.createStatement(); + long now = getDbTime(dbConn); + // We need to check whether this transaction is valid and open + String s = "select txn_state from TXNS where txn_id = " + txnid; + LOG.debug("Going to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + if (!rs.next()) { + LOG.debug("Going to rollback"); + dbConn.rollback(); + throw new NoSuchTxnException("No such transaction: " + txnid); + } + if (rs.getString(1).charAt(0) == TXN_ABORTED) { + LOG.debug("Going to rollback"); + dbConn.rollback(); + throw new TxnAbortedException("Transaction " + txnid + + " already aborted"); + } + s = "update TXNS set txn_last_heartbeat = " + now + + " where txn_id = " + txnid; + LOG.debug("Going to execute update <" + s + ">"); + stmt.executeUpdate(s); + LOG.debug("Going to commit"); + dbConn.commit(); + } finally { + closeStmt(stmt); } - s = "update TXNS set txn_last_heartbeat = " + now + - " where txn_id = " + txnid; - LOG.debug("Going to execute update <" + s + ">"); - stmt.executeUpdate(s); - LOG.debug("Going to commit"); - dbConn.commit(); } // NEVER call this function without first calling heartbeat(long, long) private long getTxnIdFromLockId(Connection dbConn, long extLockId) throws NoSuchLockException, MetaException, SQLException { - Statement stmt = dbConn.createStatement(); - String s = "select hl_txnid from HIVE_LOCKS where hl_lock_ext_id = " + - extLockId; - LOG.debug("Going to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - if (!rs.next()) { - throw new MetaException("This should never happen! We already " + - "checked the lock existed but now we can't find it!"); - } - long txnid = rs.getLong(1); - LOG.debug("Return txnid " + (rs.wasNull() ? -1 : txnid)); - return (rs.wasNull() ? -1 : txnid); + Statement stmt = null; + try { + stmt = dbConn.createStatement(); + String s = "select hl_txnid from HIVE_LOCKS where hl_lock_ext_id = " + + extLockId; + LOG.debug("Going to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + if (!rs.next()) { + throw new MetaException("This should never happen! We already " + + "checked the lock existed but now we can't find it!"); + } + long txnid = rs.getLong(1); + LOG.debug("Return txnid " + (rs.wasNull() ? -1 : txnid)); + return (rs.wasNull() ? -1 : txnid); + } finally { + closeStmt(stmt); + } } // NEVER call this function without first calling heartbeat(long, long) private List getLockInfoFromLockId(Connection dbConn, long extLockId) throws NoSuchLockException, MetaException, SQLException { - Statement stmt = dbConn.createStatement(); - String s = "select hl_lock_ext_id, hl_lock_int_id, hl_db, hl_table, " + - "hl_partition, hl_lock_state, hl_lock_type from HIVE_LOCKS where " + - "hl_lock_ext_id = " + extLockId; - LOG.debug("Going to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - boolean sawAtLeastOne = false; - List ourLockInfo = new ArrayList(); - while (rs.next()) { - ourLockInfo.add(new LockInfo(rs)); - sawAtLeastOne = true; - } - if (!sawAtLeastOne) { - throw new MetaException("This should never happen! We already " + - "checked the lock existed but now we can't find it!"); + Statement stmt = null; + try { + stmt = dbConn.createStatement(); + String s = "select hl_lock_ext_id, hl_lock_int_id, hl_db, hl_table, " + + "hl_partition, hl_lock_state, hl_lock_type from HIVE_LOCKS where " + + "hl_lock_ext_id = " + extLockId; + LOG.debug("Going to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + boolean sawAtLeastOne = false; + List ourLockInfo = new ArrayList(); + while (rs.next()) { + ourLockInfo.add(new LockInfo(rs)); + sawAtLeastOne = true; + } + if (!sawAtLeastOne) { + throw new MetaException("This should never happen! We already " + + "checked the lock existed but now we can't find it!"); + } + return ourLockInfo; + } finally { + closeStmt(stmt); } - return ourLockInfo; } // Clean time out locks from the database. This does a commit, @@ -1487,14 +1550,19 @@ public class TxnHandler { // open transactions. private void timeOutLocks(Connection dbConn) throws SQLException, MetaException { long now = getDbTime(dbConn); - Statement stmt = dbConn.createStatement(); - // Remove any timed out locks from the table. - String s = "delete from HIVE_LOCKS where hl_last_heartbeat < " + - (now - timeout); - LOG.debug("Going to execute update <" + s + ">"); - stmt.executeUpdate(s); - LOG.debug("Going to commit"); - dbConn.commit(); + Statement stmt = null; + try { + stmt = dbConn.createStatement(); + // Remove any timed out locks from the table. + String s = "delete from HIVE_LOCKS where hl_last_heartbeat < " + + (now - timeout); + LOG.debug("Going to execute update <" + s + ">"); + stmt.executeUpdate(s); + LOG.debug("Going to commit"); + dbConn.commit(); + } finally { + closeStmt(stmt); + } } // Abort timed out transactions. This calls abortTxn(), which does a commit, @@ -1502,19 +1570,24 @@ public class TxnHandler { // open transactions on the underlying database. private void timeOutTxns(Connection dbConn) throws SQLException, MetaException { long now = getDbTime(dbConn); - Statement stmt = dbConn.createStatement(); - // Abort any timed out locks from the table. - String s = "select txn_id from TXNS where txn_state = '" + TXN_OPEN + - "' and txn_last_heartbeat < " + (now - timeout); - LOG.debug("Going to execute query <" + s + ">"); - ResultSet rs = stmt.executeQuery(s); - List deadTxns = new ArrayList(); - // Limit the number of timed out transactions we do in one pass to keep from generating a - // huge delete statement - for (int i = 0; i < 20 && rs.next(); i++) deadTxns.add(rs.getLong(1)); - // We don't care whether all of the transactions get deleted or not, - // if some didn't it most likely means someone else deleted them in the interum - if (deadTxns.size() > 0) abortTxns(dbConn, deadTxns); + Statement stmt = null; + try { + stmt = dbConn.createStatement(); + // Abort any timed out locks from the table. + String s = "select txn_id from TXNS where txn_state = '" + TXN_OPEN + + "' and txn_last_heartbeat < " + (now - timeout); + LOG.debug("Going to execute query <" + s + ">"); + ResultSet rs = stmt.executeQuery(s); + List deadTxns = new ArrayList(); + // Limit the number of timed out transactions we do in one pass to keep from generating a + // huge delete statement + for (int i = 0; i < 20 && rs.next(); i++) deadTxns.add(rs.getLong(1)); + // We don't care whether all of the transactions get deleted or not, + // if some didn't it most likely means someone else deleted them in the interum + if (deadTxns.size() > 0) abortTxns(dbConn, deadTxns); + } finally { + closeStmt(stmt); + } } private static synchronized void setupJdbcConnectionPool(HiveConf conf) throws SQLException { Modified: hive/branches/cbo/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/cbo/pom.xml?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/pom.xml (original) +++ hive/branches/cbo/pom.xml Sat Jun 28 00:23:54 2014 @@ -740,7 +740,7 @@ true false false - -Xmx1024m -XX:MaxPermSize=256m + -Xmx2048m -XX:MaxPermSize=512m ${test.tmp.dir}/conf ${basedir}/${hive.path.to.root}/conf Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Sat Jun 28 00:23:54 2014 @@ -1076,16 +1076,14 @@ public class DDLTask extends Task newParams = alterDbDesc.getDatabaseProperties(); - if (database != null) { - Map params = database.getParameters(); - // if both old and new params are not null, merge them - if (params != null && newParams != null) { - params.putAll(newParams); - database.setParameters(params); - } else { // if one of them is null, replace the old params with the new - // one - database.setParameters(newParams); - } + Map params = database.getParameters(); + // if both old and new params are not null, merge them + if (params != null && newParams != null) { + params.putAll(newParams); + database.setParameters(params); + } else { + // if one of them is null, replace the old params with the new one + database.setParameters(newParams); } break; @@ -2618,7 +2616,7 @@ public class DDLTask extends Task funcs = null; if (showFuncs.getPattern() != null) { LOG.info("pattern: " + showFuncs.getPattern()); @@ -3216,16 +3214,20 @@ public class DDLTask extends Task params = null; - if(descDatabase.isExt()) { - params = database.getParameters(); - } - PrincipalType ownerType = database.getOwnerType(); - formatter.showDatabaseDescription(outStream, database.getName(), - database.getDescription(), database.getLocationUri(), - database.getOwnerName(), (null == ownerType) ? null : ownerType.name(), params); } + Map params = null; + if (descDatabase.isExt()) { + params = database.getParameters(); + } + String location = database.getLocationUri(); + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEST)) { + location = "location/in/test"; + } + PrincipalType ownerType = database.getOwnerType(); + formatter.showDatabaseDescription(outStream, database.getName(), + database.getDescription(), location, + database.getOwnerName(), (null == ownerType) ? null : ownerType.name(), params); + outStream.close(); outStream = null; } catch (IOException e) { @@ -4018,7 +4020,7 @@ public class DDLTask extends Task 0) { - // distKeyLength doesn't include tag, but includes buckNum in cachedKeys[0] - hashCode = hash.hash(firstKey.getBytes(), distKeyLength, 0); - } else if(bucketEval != null && bucketEval.length > 0) { - hashCode = computeHashCode(row, buckNum); + + // distKeyLength doesn't include tag, but includes buckNum in cachedKeys[0] + if (autoParallel && partitionEval.length > 0) { + hashCode = computeMurmurHash(firstKey); } else { - hashCode = computeHashCode(row); + hashCode = computeHashCode(row, buckNum); } firstKey.setHashCode(hashCode); @@ -384,7 +382,11 @@ public class ReduceSinkOperator extends union.setTag((byte) index); } - private int computeHashCode(Object row) throws HiveException { + protected final int computeMurmurHash(HiveKey firstKey) { + return hash.hash(firstKey.getBytes(), firstKey.getDistKeyLength(), 0); + } + + private int computeHashCode(Object row, int buckNum) throws HiveException { // Evaluate the HashCode int keyHashCode = 0; if (partitionEval.length == 0) { @@ -403,13 +405,7 @@ public class ReduceSinkOperator extends + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]); } } - return keyHashCode; - } - - private int computeHashCode(Object row, int buckNum) throws HiveException { - int keyHashCode = computeHashCode(row); - keyHashCode = keyHashCode * 31 + buckNum; - return keyHashCode; + return buckNum < 0 ? keyHashCode : keyHashCode * 31 + buckNum; } // Serialize the keys and append the tag Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Sat Jun 28 00:23:54 2014 @@ -3220,8 +3220,8 @@ public final class Utilities { HiveConf.setVar(conf, var, mWork.getInputformat()); } if (mWork.getIndexIntermediateFile() != null) { - conf.set("hive.index.compact.file", mWork.getIndexIntermediateFile()); - conf.set("hive.index.blockfilter.file", mWork.getIndexIntermediateFile()); + conf.set(ConfVars.HIVE_INDEX_COMPACT_FILE.varname, mWork.getIndexIntermediateFile()); + conf.set(ConfVars.HIVE_INDEX_BLOCKFILTER_FILE.varname, mWork.getIndexIntermediateFile()); } // Intentionally overwrites anything the user may have put here Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java Sat Jun 28 00:23:54 2014 @@ -551,8 +551,8 @@ public class ExecDriver extends Task= 0 && memUsage > 0) { reducerHash.initialize(limit, memUsage, conf.isMapGroupBy(), this); } + + autoParallel = conf.isAutoParallel(); + } catch(Exception e) { throw new HiveException(e); } @@ -265,8 +268,8 @@ public class VectorReduceSinkOperator ex populatedCachedDistributionKeys(vrg, rowIndex, 0); // replace bucketing columns with hashcode % numBuckets - int buckNum = 0; - if (bucketEval != null && bucketEval.length != 0) { + int buckNum = -1; + if (bucketEval != null) { buckNum = computeBucketNumber(vrg, rowIndex, conf.getNumBuckets()); cachedKeys[0][buckColIdxInKey] = new IntWritable(buckNum); } @@ -280,12 +283,11 @@ public class VectorReduceSinkOperator ex final int hashCode; - if(autoParallel && partitionEval.length > 0) { - hashCode = hash.hash(firstKey.getBytes(), firstKey.getDistKeyLength(), 0); - } else if(bucketEval != null && bucketEval.length > 0) { - hashCode = computeHashCode(vrg, rowIndex, buckNum); + // distKeyLength doesn't include tag, but includes buckNum in cachedKeys[0] + if (autoParallel && partitionEval.length > 0) { + hashCode = computeMurmurHash(firstKey); } else { - hashCode = computeHashCode(vrg, rowIndex); + hashCode = computeHashCode(vrg, rowIndex, buckNum); } firstKey.setHashCode(hashCode); @@ -417,7 +419,7 @@ public class VectorReduceSinkOperator ex return (BytesWritable)valueSerializer.serialize(cachedValues, valueObjectInspector); } - private int computeHashCode(VectorizedRowBatch vrg, int rowIndex) throws HiveException { + private int computeHashCode(VectorizedRowBatch vrg, int rowIndex, int buckNum) throws HiveException { // Evaluate the HashCode int keyHashCode = 0; if (partitionEval.length == 0) { @@ -440,13 +442,7 @@ public class VectorReduceSinkOperator ex partitionWriters[p].getObjectInspector()); } } - return keyHashCode; - } - - private int computeHashCode(VectorizedRowBatch vrg, int rowIndex, int buckNum) throws HiveException { - int keyHashCode = computeHashCode(vrg, rowIndex); - keyHashCode = keyHashCode * 31 + buckNum; - return keyHashCode; + return buckNum < 0 ? keyHashCode : keyHashCode * 31 + buckNum; } private int computeBucketNumber(VectorizedRowBatch vrg, int rowIndex, int numBuckets) throws HiveException { Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java Sat Jun 28 00:23:54 2014 @@ -340,15 +340,8 @@ public class RCFile { private static final Log LOG = LogFactory.getLog(RCFile.class); - public static final String RECORD_INTERVAL_CONF_STR = "hive.io.rcfile.record.interval"; - public static final String COLUMN_NUMBER_METADATA_STR = "hive.io.rcfile.column.number"; - public static final String COLUMN_NUMBER_CONF_STR = "hive.io.rcfile.column.number.conf"; - - public static final String TOLERATE_CORRUPTIONS_CONF_STR = - "hive.io.rcfile.tolerate.corruptions"; - // HACK: We actually need BlockMissingException, but that is not available // in all hadoop versions. public static final String BLOCK_MISSING_MESSAGE = @@ -985,8 +978,8 @@ public class RCFile { public Writer(FileSystem fs, Configuration conf, Path name, int bufferSize, short replication, long blockSize, Progressable progress, Metadata metadata, CompressionCodec codec) throws IOException { - RECORD_INTERVAL = conf.getInt(RECORD_INTERVAL_CONF_STR, RECORD_INTERVAL); - columnNumber = conf.getInt(COLUMN_NUMBER_CONF_STR, 0); + RECORD_INTERVAL = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_RCFILE_RECORD_INTERVAL); + columnNumber = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_RCFILE_COLUMN_NUMBER_CONF); if (metadata == null) { metadata = new Metadata(); @@ -1346,8 +1339,8 @@ public class RCFile { /** Create a new RCFile reader. */ public Reader(FileSystem fs, Path file, int bufferSize, Configuration conf, long start, long length) throws IOException { - tolerateCorruptions = conf.getBoolean( - TOLERATE_CORRUPTIONS_CONF_STR, false); + tolerateCorruptions = HiveConf.getBoolVar( + conf, HiveConf.ConfVars.HIVE_RCFILE_TOLERATE_CORRUPTIONS); conf.setInt("io.file.buffer.size", bufferSize); this.file = file; in = openFile(fs, file, bufferSize, length); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java Sat Jun 28 00:23:54 2014 @@ -25,6 +25,7 @@ import org.apache.commons.lang.StringUti import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable; import org.apache.hadoop.io.Writable; @@ -57,7 +58,7 @@ public class RCFileOutputFormat extends */ public static void setColumnNumber(Configuration conf, int columnNum) { assert columnNum > 0; - conf.setInt(RCFile.COLUMN_NUMBER_CONF_STR, columnNum); + conf.setInt(HiveConf.ConfVars.HIVE_RCFILE_COLUMN_NUMBER_CONF.varname, columnNum); } /** @@ -67,7 +68,7 @@ public class RCFileOutputFormat extends * @return number of columns for RCFile's writer */ public static int getColumnNumber(Configuration conf) { - return conf.getInt(RCFile.COLUMN_NUMBER_CONF_STR, 0); + return HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_RCFILE_COLUMN_NUMBER_CONF); } /** {@inheritDoc} */ Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java Sat Jun 28 00:23:54 2014 @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.RCFile; +import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; @@ -150,7 +151,7 @@ public class RCFileMergeMapper extends M if (outWriter == null) { codec = key.codec; columnNumber = key.keyBuffer.getColumnNumber(); - jc.setInt(RCFile.COLUMN_NUMBER_CONF_STR, columnNumber); + RCFileOutputFormat.setColumnNumber(jc, columnNumber); outWriter = new RCFile.Writer(fs, jc, outPath, null, codec); } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java Sat Jun 28 00:23:54 2014 @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.RCFile; +import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileKeyBufferWrapper; import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileValueBufferWrapper; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -124,7 +125,7 @@ public class ColumnTruncateMapper extend if (outWriter == null) { codec = key.getCodec(); columnNumber = key.getKeyBuffer().getColumnNumber(); - jc.setInt(RCFile.COLUMN_NUMBER_CONF_STR, columnNumber); + RCFileOutputFormat.setColumnNumber(jc, columnNumber); outWriter = new RCFile.Writer(fs, jc, outPath, null, codec); } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Sat Jun 28 00:23:54 2014 @@ -104,7 +104,6 @@ import org.apache.hadoop.hive.serde2.Des import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.shims.HadoopShims; -import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatus; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.util.StringUtils; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/DescDatabaseDesc.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/DescDatabaseDesc.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/DescDatabaseDesc.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/DescDatabaseDesc.java Sat Jun 28 00:23:54 2014 @@ -38,7 +38,7 @@ public class DescDatabaseDesc extends DD /** * thrift ddl for the result of describe database. */ - private static final String schema = "db_name,comment,location,parameters#string:string:string:string"; + private static final String schema = "db_name,comment,location,owner_name,owner_type,parameters#string:string:string:string:string:string"; public DescDatabaseDesc() { } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java Sat Jun 28 00:23:54 2014 @@ -116,6 +116,7 @@ public class SetProcessor implements Com if (varvalue.contains("\n")){ ss.err.println("Warning: Value had a \\n character in it."); } + varname = varname.trim(); if (varname.startsWith(SetProcessor.ENV_PREFIX)){ ss.err.println("env:* variables can not be set."); return 1; @@ -145,8 +146,8 @@ public class SetProcessor implements Com if (!confVars.isType(value)) { StringBuilder message = new StringBuilder(); message.append("'SET ").append(varname).append('=').append(varvalue); - message.append("' FAILED because ").append(key).append(" expects an "); - message.append(confVars.typeString()).append(" value."); + message.append("' FAILED because ").append(key).append(" expects "); + message.append(confVars.typeString()).append(" type value."); throw new IllegalArgumentException(message.toString()); } String fail = confVars.validate(value); @@ -156,6 +157,8 @@ public class SetProcessor implements Com message.append("' FAILED in validation : ").append(fail).append('.'); throw new IllegalArgumentException(message.toString()); } + } else if (key.startsWith("hive.")) { + throw new IllegalArgumentException("hive configuration " + key + " does not exists."); } } conf.verifyAndSet(key, value); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java Sat Jun 28 00:23:54 2014 @@ -130,7 +130,6 @@ public class GenericUDAFCount implements assert parameters.length == 0; ((CountAgg) agg).value++; } else { - assert parameters.length > 0; boolean countThisRow = true; for (Object nextParam : parameters) { if (nextParam == null) { Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (original) +++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java Sat Jun 28 00:23:54 2014 @@ -80,6 +80,9 @@ public class TestExecDriver extends Test static { try { conf = new HiveConf(ExecDriver.class); + conf.setBoolVar(HiveConf.ConfVars.SUBMITVIACHILD, true); + conf.setBoolVar(HiveConf.ConfVars.SUBMITLOCALTASKVIACHILD, true); + SessionState.start(conf); //convert possible incompatible Windows path in config Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (original) +++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java Sat Jun 28 00:23:54 2014 @@ -624,7 +624,7 @@ public class TestRCFile { int writeCount = 2500; Configuration cloneConf = new Configuration(conf); RCFileOutputFormat.setColumnNumber(cloneConf, bytesArray.length); - cloneConf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, intervalRecordCount); + cloneConf.setInt(HiveConf.ConfVars.HIVE_RCFILE_RECORD_INTERVAL.varname, intervalRecordCount); RCFile.Writer writer = new RCFile.Writer(fs, cloneConf, testFile, null, codec); BytesRefArrayWritable bytes = new BytesRefArrayWritable(bytesArray.length); @@ -690,7 +690,7 @@ public class TestRCFile { fs.delete(testFile, true); Configuration cloneConf = new Configuration(conf); RCFileOutputFormat.setColumnNumber(cloneConf, bytesArray.length); - cloneConf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, intervalRecordCount); + cloneConf.setInt(HiveConf.ConfVars.HIVE_RCFILE_RECORD_INTERVAL.varname, intervalRecordCount); RCFile.Writer writer = new RCFile.Writer(fs, cloneConf, testFile, null, codec); Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,7 @@ set hive.archive.enabled = true; set hive.enforce.bucketing = true; +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact1.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact1.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact1.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact1.q Sat Jun 28 00:23:54 2014 @@ -1,10 +1,8 @@ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -set hive.txn.testing=true; create table T1(key string, val string) stored as textfile; -set hive.txn.testing=true; alter table T1 compact 'major'; alter table T1 compact 'minor'; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q Sat Jun 28 00:23:54 2014 @@ -1,6 +1,5 @@ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -set hive.txn.testing=true; create table T1(key string, val string) partitioned by (ds string) stored as textfile; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact3.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact3.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact3.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_compact3.q Sat Jun 28 00:23:54 2014 @@ -1,6 +1,5 @@ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -set hive.txn.testing=true; create database D1; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_showlocks.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_showlocks.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_showlocks.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/dbtxnmgr_showlocks.q Sat Jun 28 00:23:54 2014 @@ -1,6 +1,5 @@ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -set hive.txn.testing=true; show locks; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q Sat Jun 28 00:23:54 2014 @@ -6,6 +6,8 @@ set hive.exec.dynamic.partition.mode=non set hive.vectorized.execution.enabled=true; set hive.enforce.bucketing=false; set hive.enforce.sorting=false; +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; create table over1k( t tinyint, Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q Sat Jun 28 00:23:54 2014 @@ -5,6 +5,8 @@ set hive.exec.max.dynamic.partitions.per set hive.exec.dynamic.partition.mode=nonstrict; set hive.enforce.bucketing=false; set hive.enforce.sorting=false; +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; create table over1k( t tinyint, Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/fetch_aggregation.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/fetch_aggregation.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/fetch_aggregation.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/fetch_aggregation.q Sat Jun 28 00:23:54 2014 @@ -1,4 +1,6 @@ set hive.fetch.task.aggr=true; +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; explain select count(key),sum(key),avg(key),min(key),max(key),std(key),variance(key) from src; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/index_bitmap_compression.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/index_bitmap_compression.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/index_bitmap_compression.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/index_bitmap_compression.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,5 @@ set hive.stats.dbclass=fs; -SET hive.exec.compress.result=true; +SET hive.exec.compress.output=true; -- SORT_QUERY_RESULTS Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/index_compression.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/index_compression.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/index_compression.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/index_compression.q Sat Jun 28 00:23:54 2014 @@ -1,4 +1,4 @@ -SET hive.exec.compress.result=true; +SET hive.exec.compress.output=true; SET hive.stats.dbclass=fs; -- SORT_QUERY_RESULTS Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/join25.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/join25.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/join25.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/join25.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,3 @@ -set hive.mapjoin.numrows = 2; - -- SORT_QUERY_RESULTS CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/join36.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/join36.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/join36.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/join36.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,3 @@ -set hive.mapjoin.numrows = 2; - -- SORT_QUERY_RESULTS CREATE TABLE tmp1(key INT, cnt INT); Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/join37.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/join37.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/join37.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/join37.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,3 @@ -set hive.mapjoin.numrows = 2; - -- SORT_QUERY_RESULTS CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/join_nulls.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/join_nulls.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/join_nulls.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/join_nulls.q Sat Jun 28 00:23:54 2014 @@ -49,8 +49,8 @@ LOAD DATA LOCAL INPATH '../../data/files LOAD DATA LOCAL INPATH '../../data/files/in1.txt' into table smb_input2; LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table smb_input2; -SET hive.optimize.bucketmapJOIN = true; -SET hive.optimize.bucketmapJOIN.sortedmerge = true; +SET hive.optimize.bucketmapjoin = true; +SET hive.optimize.bucketmapjoin.sortedmerge = true; SET hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key = b.key; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/join_nullsafe.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/join_nullsafe.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/join_nullsafe.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/join_nullsafe.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,3 @@ -set hive.nullsafe.equijoin=true; - -- SORT_QUERY_RESULTS CREATE TABLE myinput1(key int, value int); @@ -30,16 +28,23 @@ SELECT * FROM myinput1 a FULL OUTER JOIN SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value; SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value; +CREATE TABLE smb_input(key int, value int); +LOAD DATA LOCAL INPATH '../../data/files/in4.txt' into table smb_input; +LOAD DATA LOCAL INPATH '../../data/files/in5.txt' into table smb_input; + +set hive.enforce.sorting = true; +set hive.enforce.bucketing = true; + -- smbs CREATE TABLE smb_input1(key int, value int) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS; CREATE TABLE smb_input2(key int, value int) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS; -LOAD DATA LOCAL INPATH '../../data/files/in8.txt' into table smb_input1; -LOAD DATA LOCAL INPATH '../../data/files/in9.txt' into table smb_input1; -LOAD DATA LOCAL INPATH '../../data/files/in8.txt' into table smb_input2; -LOAD DATA LOCAL INPATH '../../data/files/in9.txt' into table smb_input2; -SET hive.optimize.bucketmapJOIN = true; -SET hive.optimize.bucketmapJOIN.sortedmerge = true; +from smb_input +insert overwrite table smb_input1 select * +insert overwrite table smb_input2 select *; + +SET hive.optimize.bucketmapjoin = true; +SET hive.optimize.bucketmapjoin.sortedmerge = true; SET hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/metadata_export_drop.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/metadata_export_drop.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/metadata_export_drop.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/metadata_export_drop.q Sat Jun 28 00:23:54 2014 @@ -2,7 +2,7 @@ create table tmp_meta_export_listener_dr dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/data/exports/HIVE-3427; set hive.metastore.pre.event.listeners=org.apache.hadoop.hive.ql.parse.MetaDataExportListener; set hive.metadata.export.location=${system:test.tmp.dir}/data/exports/HIVE-3427; -set hive.move.exported.metadata.to.trash=false; +set hive.metadata.move.exported.metadata.to.trash=false; drop table tmp_meta_export_listener_drop_test; dfs -rmr ${system:test.tmp.dir}/data/exports/HIVE-3427; set hive.metastore.pre.event.listeners=; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/nonmr_fetch.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/nonmr_fetch.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/nonmr_fetch.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/nonmr_fetch.q Sat Jun 28 00:23:54 2014 @@ -1,4 +1,6 @@ set hive.fetch.task.conversion=minimal; +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; -- backward compatible (minimal) explain select * from src limit 10; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/orc_analyze.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/orc_analyze.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/orc_analyze.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/orc_analyze.q Sat Jun 28 00:23:54 2014 @@ -1,3 +1,6 @@ +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; + CREATE TABLE orc_create_people_staging ( id int, first_name string, Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/overridden_confs.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/overridden_confs.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/overridden_confs.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/overridden_confs.q Sat Jun 28 00:23:54 2014 @@ -1,4 +1,4 @@ set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyOverriddenConfigsHook; -set hive.config.doesnt.exit=abc; +set some.hive.config.doesnt.exit=abc; select count(*) from src; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/quotedid_skew.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/quotedid_skew.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/quotedid_skew.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/quotedid_skew.q Sat Jun 28 00:23:54 2014 @@ -2,7 +2,6 @@ set hive.support.quoted.identifiers=column; set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(`!@#$%^&*()_q` string, `y&y` string) Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/sample10.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/sample10.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/sample10.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/sample10.q Sat Jun 28 00:23:54 2014 @@ -1,4 +1,5 @@ - +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; set hive.enforce.bucketing=true; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/sample_islocalmode_hook.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/sample_islocalmode_hook.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/sample_islocalmode_hook.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/sample_islocalmode_hook.q Sat Jun 28 00:23:54 2014 @@ -1,3 +1,5 @@ +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; set mapred.max.split.size=300; set mapred.min.split.size=300; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoin_union_remove_1.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoin_union_remove_1.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoin_union_remove_1.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoin_union_remove_1.q Sat Jun 28 00:23:54 2014 @@ -1,4 +1,3 @@ -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; set hive.mapred.supports.subdirectories=true; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoin_union_remove_2.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoin_union_remove_2.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoin_union_remove_2.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoin_union_remove_2.q Sat Jun 28 00:23:54 2014 @@ -1,4 +1,3 @@ -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; set hive.mapred.supports.subdirectories=true; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt1.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt1.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt1.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt1.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, val STRING) Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt10.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt10.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt10.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt10.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, value STRING) STORED AS TEXTFILE; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt11.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt11.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt11.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt11.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, val STRING) Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt12.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt12.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt12.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt12.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, val STRING) Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt13.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt13.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt13.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt13.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, val STRING) STORED AS TEXTFILE; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt14.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt14.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt14.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt14.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, val STRING) Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt15.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt15.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt15.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt15.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE tmpT1(key STRING, val STRING) STORED AS TEXTFILE; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt16.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt16.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt16.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt16.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, val STRING) Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt17.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt17.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt17.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt17.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, val STRING) Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt18.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt18.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt18.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt18.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE tmpT1(key STRING, val STRING) STORED AS TEXTFILE; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt19.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt19.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt19.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt19.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, val STRING) Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt2.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt2.q?rev=1606275&r1=1606274&r2=1606275&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt2.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/skewjoinopt2.q Sat Jun 28 00:23:54 2014 @@ -1,5 +1,4 @@ set hive.mapred.supports.subdirectories=true; -set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; CREATE TABLE T1(key STRING, val STRING)