hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hive git commit: HIVE-15181 : buildQueryWithINClause didn't properly handle multiples of ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE (Wei Zheng, reviewed by Eugene Koifman)
Date Wed, 23 Nov 2016 06:42:26 GMT
Repository: hive
Updated Branches:
  refs/heads/master f9c6d1858 -> 76f1f41f0


HIVE-15181 : buildQueryWithINClause didn't properly handle multiples of ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE
(Wei Zheng, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/76f1f41f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/76f1f41f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/76f1f41f

Branch: refs/heads/master
Commit: 76f1f41f0a970a820b1dff3f342591608a346eac
Parents: f9c6d18
Author: Wei Zheng <weiz@apache.org>
Authored: Tue Nov 22 22:42:52 2016 -0800
Committer: Wei Zheng <weiz@apache.org>
Committed: Tue Nov 22 22:42:52 2016 -0800

----------------------------------------------------------------------
 .../metastore/txn/CompactionTxnHandler.java     |  4 ++
 .../hadoop/hive/metastore/txn/TxnHandler.java   |  2 +-
 .../hadoop/hive/metastore/txn/TxnUtils.java     | 12 ++++--
 .../hadoop/hive/metastore/txn/TestTxnUtils.java | 40 ++++++++++++++++++--
 4 files changed, 49 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/76f1f41f/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index 9145fcc..fde1b54 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -726,6 +726,10 @@ class CompactionTxnHandler extends TxnHandler {
         }
         close(rs);
 
+        if (deleteSet.size() <= 0) {
+          return;
+        }
+
         List<String> queries = new ArrayList<String>();
 
         StringBuilder prefix = new StringBuilder();

http://git-wip-us.apache.org/repos/asf/hive/blob/76f1f41f/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 97d09df..382df1f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -2844,11 +2844,11 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
         List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
         timedOutTxns.add(currentBatch);
         do {
-          currentBatch.add(rs.getLong(1));
           if(currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
             currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
             timedOutTxns.add(currentBatch);
           }
+          currentBatch.add(rs.getLong(1));
         } while(rs.next());
         dbConn.commit();
         close(rs, stmt, null);

http://git-wip-us.apache.org/repos/asf/hive/blob/76f1f41f/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
index 2ffa1da..517eec3 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
@@ -145,6 +145,9 @@ public class TxnUtils {
   public static void buildQueryWithINClause(HiveConf conf, List<String> queries, StringBuilder
prefix,
                                             StringBuilder suffix, List<Long> inList,
                                             String inColumn, boolean addParens, boolean notIn)
{
+    if (inList == null || inList.size() == 0) {
+      throw new IllegalArgumentException("The IN list is empty!");
+    }
     int batchSize = conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE);
     int numWholeBatches = inList.size() / batchSize;
     StringBuilder buf = new StringBuilder();
@@ -160,6 +163,11 @@ public class TxnUtils {
     }
 
     for (int i = 0; i <= numWholeBatches; i++) {
+      if (i * batchSize == inList.size()) {
+        // At this point we just realized we don't need another query
+        break;
+      }
+
       if (needNewQuery(conf, buf)) {
         // Wrap up current query string
         if (addParens) {
@@ -198,10 +206,6 @@ public class TxnUtils {
         }
       }
 
-      if (i * batchSize == inList.size()) {
-        // At this point we just realized we don't need another query
-        return;
-      }
       for (int j = i * batchSize; j < (i + 1) * batchSize && j < inList.size();
j++) {
         buf.append(inList.get(j)).append(",");
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/76f1f41f/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
index 8fada2c..4c3b824 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
@@ -70,24 +70,56 @@ public class TestTxnUtils {
     Assert.assertEquals(2, queries.size());
     runAgainstDerby(queries);
 
-    // Case 3 - Max in list members: 1000; Max query string length: 5KB
+    // Case 3.1 - Max in list members: 1000, Max query string length: 1KB, and exact 1000
members in a single IN clause
+    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 1);
+    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 1000);
+    queries.clear();
+    for (long i = 202; i <= 1000; i++) {
+      inList.add(i);
+    }
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true,
false);
+    Assert.assertEquals(1, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 3.2 - Max in list members: 1000, Max query string length: 10KB, and exact 1000
members in a single IN clause
     conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 10);
     conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 1000);
     queries.clear();
-    for (long i = 202; i <= 4321; i++) {
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true,
false);
+    Assert.assertEquals(1, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 3.3 - Now with 2000 entries, try the above settings
+    for (long i = 1001; i <= 2000; i++) {
+      inList.add(i);
+    }
+    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 1);
+    queries.clear();
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true,
false);
+    Assert.assertEquals(2, queries.size());
+    runAgainstDerby(queries);
+    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 10);
+    queries.clear();
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true,
false);
+    Assert.assertEquals(1, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 4 - Max in list members: 1000; Max query string length: 10KB
+    queries.clear();
+    for (long i = 2001; i <= 4321; i++) {
       inList.add(i);
     }
     TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true,
false);
     Assert.assertEquals(3, queries.size());
     runAgainstDerby(queries);
 
-    // Case 4 - NOT IN list
+    // Case 5 - NOT IN list
     queries.clear();
     TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true,
true);
     Assert.assertEquals(3, queries.size());
     runAgainstDerby(queries);
 
-    // Case 5 - No parenthesis
+    // Case 6 - No parenthesis
     queries.clear();
     suffix.setLength(0);
     suffix.append("");


Mime
View raw message