hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vihan...@apache.org
Subject hive git commit: HIVE-16143: Improve msck repair batching (Vihang Karajgaonkar, reviewed by Sahil Takiar & Aihua Xu)
Date Wed, 04 Oct 2017 17:39:23 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2 cc7f21381 -> 3f25911a6


HIVE-16143: Improve msck repair batching (Vihang Karajgaonkar, reviewed by Sahil Takiar & Aihua Xu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3f25911a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3f25911a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3f25911a

Branch: refs/heads/branch-2
Commit: 3f25911a6ad3183bfa66460d258f2de44039a9af
Parents: cc7f213
Author: Aihua Xu <aihuaxu@apache.org>
Authored: Mon May 15 14:08:32 2017 -0400
Committer: Vihang Karajgaonkar <vihang@cloudera.com>
Committed: Wed Oct 4 10:39:12 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  13 +-
 .../apache/hive/common/util/RetryUtilities.java | 112 ++++++
 .../hive/common/util/TestRetryUtilities.java    | 150 ++++++++
 .../test/queries/clientpositive/create_like.q   |   3 +-
 .../results/clientpositive/create_like.q.out    |  15 +-
 .../org/apache/hadoop/hive/ql/QTestUtil.java    |   3 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |  89 ++---
 .../exec/TestMsckCreatePartitionsInBatches.java | 340 +++++++++++++++++++
 .../test/queries/clientpositive/msck_repair_0.q |  10 +
 .../test/queries/clientpositive/msck_repair_1.q |   8 +
 .../test/queries/clientpositive/msck_repair_2.q |   5 +
 .../test/queries/clientpositive/msck_repair_3.q |   4 +
 .../clientpositive/msck_repair_batchsize.q      |   4 +
 .../results/clientpositive/msck_repair_0.q.out  |  38 ++-
 .../results/clientpositive/msck_repair_1.q.out  |  28 +-
 .../results/clientpositive/msck_repair_2.q.out  |  28 +-
 .../results/clientpositive/msck_repair_3.q.out  |  28 +-
 .../clientpositive/msck_repair_batchsize.q.out  |  34 +-
 ql/src/test/results/clientpositive/repair.q.out |   3 +-
 19 files changed, 857 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 7420310..e17e73f 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3324,9 +3324,16 @@ public class HiveConf extends Configuration {
        " others; 'ignore' will skip the validation (legacy behavior, causes bugs in many cases)"),
     HIVE_MSCK_REPAIR_BATCH_SIZE(
         "hive.msck.repair.batch.size", 0,
-        "Batch size for the msck repair command. If the value is greater than zero, "
-            + "it will execute batch wise with the configured batch size. "
-            + "The default value is zero. Zero means it will execute directly (Not batch wise)"),
+        "Batch size for the msck repair command. If the value is greater than zero,\n "
+            + "it will execute batch wise with the configured batch size. In case of errors while\n"
+            + "adding unknown partitions the batch size is automatically reduced by half in the subsequent\n"
+            + "retry attempt. The default value is zero which means it will execute directly (not batch wise)"),
+    HIVE_MSCK_REPAIR_BATCH_MAX_RETRIES("hive.msck.repair.batch.max.retries", 0,
+        "Maximum number of retries for the msck repair command when adding unknown partitions.\n "
+        + "If the value is greater than zero it will retry adding unknown partitions until the maximum\n"
+        + "number of attempts is reached or batch size is reduced to 0, whichever is earlier.\n"
+        + "In each retry attempt it will reduce the batch size by a factor of 2 until it reaches zero.\n"
+        + "If the value is set to zero it will retry until the batch size becomes zero as described above."),
     HIVE_SERVER2_LLAP_CONCURRENT_QUERIES("hive.server2.llap.concurrent.queries", -1,
         "The number of queries allowed in parallel via llap. Negative number implies 'infinite'."),
     HIVE_TEZ_ENABLE_MEMORY_MANAGER("hive.tez.enable.memory.manager", true,

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/common/src/java/org/apache/hive/common/util/RetryUtilities.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/common/util/RetryUtilities.java b/common/src/java/org/apache/hive/common/util/RetryUtilities.java
new file mode 100644
index 0000000..3a20f2c
--- /dev/null
+++ b/common/src/java/org/apache/hive/common/util/RetryUtilities.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.common.util;
+
+import java.util.concurrent.Callable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RetryUtilities {
+  public static class RetryException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    public RetryException(Exception ex) {
+      super(ex);
+    }
+
+    public RetryException(String msg) {
+      super(msg);
+    }
+  }
+
+  /**
+   * Interface used to create a ExponentialBackOffRetry policy
+   */
+  public static interface ExponentialBackOffRetry<T> {
+    /**
+     * This method should be called by implementations of this ExponentialBackOffRetry policy
+     * It represents the actual work which needs to be done based on a given batch size
+     * @param batchSize The batch size for the work which needs to be executed
+     * @return
+     * @throws Exception
+     */
+    public T execute(int batchSize) throws Exception;
+  }
+
+  /**
+   * This class is a base implementation of a simple exponential back retry policy. The batch size
+   * and decaying factor are provided with the constructor. It reduces the batch size by dividing
+   * it by the decaying factor every time there is an exception in the execute method.
+   */
+  public static abstract class ExponentiallyDecayingBatchWork<T>
+      implements ExponentialBackOffRetry<T> {
+    private int batchSize;
+    private final int decayingFactor;
+    private int maxRetries;
+    private static final Logger LOG = LoggerFactory.getLogger(ExponentiallyDecayingBatchWork.class);
+
+    public ExponentiallyDecayingBatchWork(int batchSize, int reducingFactor, int maxRetries) {
+      if (batchSize <= 0) {
+        throw new IllegalArgumentException(String.format(
+            "Invalid batch size %d provided. Batch size must be greater than 0", batchSize));
+      }
+      this.batchSize = batchSize;
+      if (reducingFactor <= 1) {
+        throw new IllegalArgumentException(String.format(
+            "Invalid decaying factor %d provided. Decaying factor must be greater than 1",
+            batchSize));
+      }
+      if (maxRetries < 0) {
+        throw new IllegalArgumentException(String.format(
+            "Invalid number of maximum retries %d provided. It must be a non-negative integer value",
+            maxRetries));
+      }
+      //if maxRetries is 0 code retries until batch decays to zero
+      this.maxRetries = maxRetries;
+      this.decayingFactor = reducingFactor;
+    }
+
+    public T run() throws Exception {
+      int attempt = 0;
+      while (true) {
+        int size = getNextBatchSize();
+        if (size == 0) {
+          throw new RetryException("Batch size reduced to zero");
+        }
+        try {
+          return execute(size);
+        } catch (Exception ex) {
+          LOG.warn(String.format("Exception thrown while processing using a batch size %d", size),
+              ex);
+        } finally {
+          attempt++;
+          if (attempt == maxRetries) {
+            throw new RetryException(String.format("Maximum number of retry attempts %d exhausted", maxRetries));
+          }
+        }
+      }
+    }
+
+    private int getNextBatchSize() {
+      int ret = batchSize;
+      batchSize /= decayingFactor;
+      return ret;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/common/src/test/org/apache/hive/common/util/TestRetryUtilities.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hive/common/util/TestRetryUtilities.java b/common/src/test/org/apache/hive/common/util/TestRetryUtilities.java
new file mode 100644
index 0000000..4187be2
--- /dev/null
+++ b/common/src/test/org/apache/hive/common/util/TestRetryUtilities.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.common.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hive.common.util.RetryUtilities.ExponentiallyDecayingBatchWork;
+import org.apache.hive.common.util.RetryUtilities.RetryException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestRetryUtilities {
+
+  private class DummyExponentiallyDecayingBatchWork extends ExponentiallyDecayingBatchWork<Void> {
+    public DummyExponentiallyDecayingBatchWork(int batchSize, int reducingFactor,
+        int throwException) {
+      super(batchSize, reducingFactor, 0);
+      this.exceptionCount = throwException;
+    }
+
+    public DummyExponentiallyDecayingBatchWork(int batchSize, int reducingFactor,
+        int throwException, int maxRetries) {
+      super(batchSize, reducingFactor, maxRetries);
+      this.exceptionCount = throwException;
+    }
+
+    final List<Integer> batchSizes = new ArrayList<>();
+    int exceptionCount = 0;
+
+    @Override
+    public Void execute(int size) throws Exception {
+      batchSizes.add(size);
+      if (exceptionCount > 0) {
+        exceptionCount--;
+        throw new Exception("Dummy exception");
+      }
+      return null;
+    }
+
+    public int getCount() {
+      return batchSizes.size();
+    }
+
+    public int[] getBatchSizes() {
+      int[] ret = new int[batchSizes.size()];
+      int i = 0;
+      for (int b : batchSizes) {
+        ret[i++] = b;
+      }
+      return ret;
+    }
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testZeroBatchSize() {
+    new DummyExponentiallyDecayingBatchWork(0, 2, 0);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testNegativeBatchSize() {
+    new DummyExponentiallyDecayingBatchWork(-1, 2, 0);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testZeroDecayingFactor() {
+    new DummyExponentiallyDecayingBatchWork(5, 0, 0);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testOneDecayingFactor() {
+    new DummyExponentiallyDecayingBatchWork(10, 1, 0);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testNegativeMaxRetries() {
+    new DummyExponentiallyDecayingBatchWork(10, 2, 0, -1);
+  }
+
+  @Test
+  public void testNumberOfAttempts() throws Exception {
+    // test perfectly divisible batchsize and decaying factor
+    DummyExponentiallyDecayingBatchWork dummy = new DummyExponentiallyDecayingBatchWork(10, 2, 0);
+    dummy.run();
+    Assert.assertEquals("Unexpected number of executions of execute method", 1, dummy.getCount());
+    // there were no exception. Batchsize doesn't change until there is an exception
+    Assert.assertArrayEquals(new int[] { 10 }, dummy.getBatchSizes());
+    // test batchsize is not divisible by decaying factor
+    dummy = new DummyExponentiallyDecayingBatchWork(11, 2, 0);
+    dummy.run();
+    Assert.assertEquals("Unexpected number of executions of execute method", 1, dummy.getCount());
+    // there were no exception. Batchsize doesn't change until there is an exception
+    Assert.assertArrayEquals(new int[] { 11 }, dummy.getBatchSizes());
+
+    dummy = new DummyExponentiallyDecayingBatchWork(11, 3, 1);
+    // batches will be sized 11,3
+    dummy.run();
+    Assert.assertEquals("Unexpected number of executions of execute method", 2, dummy.getCount());
+    Assert.assertArrayEquals(new int[] { 11, 3 }, dummy.getBatchSizes());
+
+    dummy = new DummyExponentiallyDecayingBatchWork(11, 3, 2);
+    // batches will be sized 11,3,1
+    dummy.run();
+    Assert.assertEquals("Unexpected number of executions of execute method", 3, dummy.getCount());
+    Assert.assertArrayEquals(new int[] { 11, 3, 1 }, dummy.getBatchSizes());
+
+    dummy = new DummyExponentiallyDecayingBatchWork(12, 3, 2);
+    // batches will be sized 12,4,1
+    dummy.run();
+    Assert.assertEquals("Unexpected number of executions of execute method", 3, dummy.getCount());
+    Assert.assertArrayEquals(new int[] { 12, 4, 1 }, dummy.getBatchSizes());
+  }
+
+  @Test
+  public void testZeroMaxRetriesValue() throws Exception {
+    DummyExponentiallyDecayingBatchWork dummy = new DummyExponentiallyDecayingBatchWork(10, 2, 3, 0);
+    dummy.run();
+    // batches will be sized 10, 5, 2, 1
+    Assert.assertEquals("Unexpected number of executions of execute method", 4, dummy.getCount());
+    Assert.assertArrayEquals(new int[] { 10, 5, 2, 1 }, dummy.getBatchSizes());
+
+    dummy = new DummyExponentiallyDecayingBatchWork(17, 2, 4, 0);
+    // batches will be sized 17, 8, 4, 2, 1
+    dummy.run();
+    Assert.assertEquals("Unexpected number of executions of execute method", 5, dummy.getCount());
+    Assert.assertArrayEquals(new int[] { 17, 8, 4, 2, 1 }, dummy.getBatchSizes());
+  }
+
+  @Test(expected = RetryException.class)
+  public void testRetriesExhausted() throws Exception {
+    // attempts at execute will be made using batchsizes 11, 3, 1, throws retry exception
+    DummyExponentiallyDecayingBatchWork dummy = new DummyExponentiallyDecayingBatchWork(11, 3, 3);
+    dummy.run();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/itests/hive-blobstore/src/test/queries/clientpositive/create_like.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/create_like.q b/itests/hive-blobstore/src/test/queries/clientpositive/create_like.q
index 38f384e..63ea3f6 100644
--- a/itests/hive-blobstore/src/test/queries/clientpositive/create_like.q
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/create_like.q
@@ -21,10 +21,11 @@ DROP TABLE like_table;
 CREATE EXTERNAL TABLE like_table LIKE blobstore_partitioned_source_table LOCATION '${hiveconf:test.blobstore.path.unique}/create_like/data';
 
 MSCK REPAIR TABLE like_table;
+show partitions like_table;
 
 SELECT * FROM blobstore_partitioned_source_table;
 SELECT * FROM like_table;
 
 DROP TABLE like_table;
 
-SELECT * FROM blobstore_partitioned_source_table;
\ No newline at end of file
+SELECT * FROM blobstore_partitioned_source_table;

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/itests/hive-blobstore/src/test/results/clientpositive/create_like.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/create_like.q.out b/itests/hive-blobstore/src/test/results/clientpositive/create_like.q.out
index 0d362a7..8dcbb93 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/create_like.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/create_like.q.out
@@ -91,10 +91,17 @@ POSTHOOK: query: MSCK REPAIR TABLE like_table
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@like_table
 Partitions not in metastore:	like_table:dt=20110924/hour=1	like_table:dt=20110924/hour=2	like_table:dt=20110925/hour=1	like_table:dt=20110925/hour=2
-Repair: Added partition to metastore like_table:dt=20110924/hour=1
-Repair: Added partition to metastore like_table:dt=20110924/hour=2
-Repair: Added partition to metastore like_table:dt=20110925/hour=1
-Repair: Added partition to metastore like_table:dt=20110925/hour=2
+#### A masked pattern was here ####
+PREHOOK: query: show partitions like_table
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@like_table
+POSTHOOK: query: show partitions like_table
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@like_table
+dt=20110924/hour=1
+dt=20110924/hour=2
+dt=20110925/hour=1
+dt=20110925/hour=2
 PREHOOK: query: SELECT * FROM blobstore_partitioned_source_table
 PREHOOK: type: QUERY
 PREHOOK: Input: default@blobstore_partitioned_source_table

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 9631af4..551a6de 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1691,7 +1691,8 @@ public class QTestUtil {
       ".*at com\\.sun\\.proxy.*",
       ".*at com\\.jolbox.*",
       ".*at com\\.zaxxer.*",
-      "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*"
+      "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*",
+      "^Repair: Added partition to metastore.*"
   });
 
   private final Pattern[] partialReservedPlanMask = toPattern(new String[] {

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 2a43eb3..91f1b53 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -247,10 +247,12 @@ import org.apache.hadoop.util.ToolRunner;
 import org.apache.hive.common.util.AnnotationUtils;
 import org.apache.hive.common.util.HiveStringUtils;
 import org.apache.hive.common.util.ReflectionUtil;
+import org.apache.hive.common.util.RetryUtilities;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.stringtemplate.v4.ST;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 
 /**
@@ -1866,19 +1868,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
     }
   }
 
-  private void msckAddPartitionsOneByOne(Hive db, Table table,
-      Set<CheckResult.PartitionResult> partsNotInMs, List<String> repairOutput) {
-    for (CheckResult.PartitionResult part : partsNotInMs) {
-      try {
-        db.createPartition(table, Warehouse.makeSpecFromName(part.getPartitionName()));
-        repairOutput.add("Repair: Added partition to metastore "
-            + table.getTableName() + ':' + part.getPartitionName());
-      } catch (Exception e) {
-        LOG.warn("Repair error, could not add partition to metastore: ", e);
-      }
-    }
-  }
-
   private int compact(Hive db, AlterTableSimpleDesc desc) throws HiveException {
 
     Table tbl = db.getTable(desc.getTableName());
@@ -2006,34 +1995,18 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
           }
         }
         Table table = db.getTable(msckDesc.getTableName());
-        AddPartitionDesc apd = new AddPartitionDesc(
-            table.getDbName(), table.getTableName(), false);
+        int batchSize = conf.getIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_SIZE);
+        int maxRetries = conf.getIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_MAX_RETRIES);
+        int decayingFactor = 2;
+        if (batchSize == 0) {
+          //batching is not enabled. Try to add all the partitions in one call
+          batchSize = partsNotInMs.size();
+        }
         try {
-          int batch_size = conf.getIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_SIZE);
-          if (batch_size > 0 && partsNotInMs.size() > batch_size) {
-            int counter = 0;
-            for (CheckResult.PartitionResult part : partsNotInMs) {
-              counter++;
-              apd.addPartition(Warehouse.makeSpecFromName(part.getPartitionName()), null);
-              repairOutput.add("Repair: Added partition to metastore " + msckDesc.getTableName()
-                  + ':' + part.getPartitionName());
-              if (counter % batch_size == 0 || counter == partsNotInMs.size()) {
-                db.createPartitions(apd);
-                apd = new AddPartitionDesc(table.getDbName(), table.getTableName(), false);
-              }
-            }
-          } else {
-            for (CheckResult.PartitionResult part : partsNotInMs) {
-              apd.addPartition(Warehouse.makeSpecFromName(part.getPartitionName()), null);
-              repairOutput.add("Repair: Added partition to metastore " + msckDesc.getTableName()
-                  + ':' + part.getPartitionName());
-            }
-            db.createPartitions(apd);
-          }
+          createPartitionsInBatches(db, repairOutput, partsNotInMs, table, batchSize,
+              decayingFactor, maxRetries);
         } catch (Exception e) {
-          LOG.info("Could not bulk-add partitions to metastore; trying one by one", e);
-          repairOutput.clear();
-          msckAddPartitionsOneByOne(db, table, partsNotInMs, repairOutput);
+          throw new HiveException(e);
         }
       }
     } catch (HiveException e) {
@@ -2085,6 +2058,44 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
     return 0;
   }
 
+  @VisibleForTesting
+  void createPartitionsInBatches(final Hive db, final List<String> repairOutput,
+      Set<CheckResult.PartitionResult> partsNotInMs, final Table table, int batchSize, int decayingFactor, int maxRetries)
+      throws Exception {
+    final String addMsgFormat = "Repair: Added partition to metastore "
+        + table.getTableName() + ":%s";
+    final Set<CheckResult.PartitionResult> batchWork = new HashSet<>(partsNotInMs);
+    new RetryUtilities.ExponentiallyDecayingBatchWork<Void>(batchSize, decayingFactor, maxRetries) {
+      @Override
+      public Void execute(int size) throws Exception {
+        while (!batchWork.isEmpty()) {
+          //get the current batch size
+          int currentBatchSize = size;
+          AddPartitionDesc apd =
+              new AddPartitionDesc(table.getDbName(), table.getTableName(), true);
+          //store the partitions temporarily until processed
+          List<CheckResult.PartitionResult> lastBatch = new ArrayList<>(currentBatchSize);
+          List<String> addMsgs = new ArrayList<>(currentBatchSize);
+          //add the number of partitions given by the current batchsize
+          for (CheckResult.PartitionResult part : batchWork) {
+            if (currentBatchSize == 0) {
+              break;
+            }
+            apd.addPartition(Warehouse.makeSpecFromName(part.getPartitionName()), null);
+            lastBatch.add(part);
+            addMsgs.add(String.format(addMsgFormat, part.getPartitionName()));
+            currentBatchSize--;
+          }
+          db.createPartitions(apd);
+          // if last batch is successful remove it from partsNotInMs
+          batchWork.removeAll(lastBatch);
+          repairOutput.addAll(addMsgs);
+        }
+        return null;
+      }
+    }.run();
+  }
+
   /**
    * Write the result of msck to a writer.
    *

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java
new file mode 100644
index 0000000..6e69d4f
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java
@@ -0,0 +1,340 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec;
+
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.metadata.CheckResult.PartitionResult;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hive.common.util.RetryUtilities.RetryException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+public class TestMsckCreatePartitionsInBatches {
+  private static HiveConf hiveConf;
+  private static DDLTask ddlTask;
+  private final String tableName = "test_msck_batch";
+  private static Hive db;
+  private List<String> repairOutput;
+  private Table table;
+
+  @BeforeClass
+  public static void setupClass() throws HiveException {
+    hiveConf = new HiveConf(TestMsckCreatePartitionsInBatches.class);
+    hiveConf.setIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_SIZE, 5);
+    hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
+        "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
+    SessionState.start(hiveConf);
+    db = Hive.get(hiveConf);
+    ddlTask = new DDLTask();
+  }
+
+  @Before
+  public void before() throws Exception {
+    createPartitionedTable("default", tableName);
+    table = db.getTable(tableName);
+    repairOutput = new ArrayList<String>();
+  }
+
+  @After
+  public void after() throws Exception {
+    cleanUpTableQuietly("default", tableName);
+  }
+
+  private Table createPartitionedTable(String dbName, String tableName) throws Exception {
+    try {
+      db.dropTable(dbName, tableName);
+      db.createTable(tableName, Arrays.asList("key", "value"), // Data columns.
+          Arrays.asList("city"), // Partition columns.
+          TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class);
+      return db.getTable(dbName, tableName);
+    } catch (Exception exception) {
+      fail("Unable to drop and create table " + dbName + "." + tableName + " because "
+          + StringUtils.stringifyException(exception));
+      throw exception;
+    }
+  }
+
+  private void cleanUpTableQuietly(String dbName, String tableName) {
+    try {
+      db.dropTable(dbName, tableName, true, true, true);
+    } catch (Exception exception) {
+      fail("Unexpected exception: " + StringUtils.stringifyException(exception));
+    }
+  }
+
+  private Set<PartitionResult> createPartsNotInMs(int numOfParts) {
+    Set<PartitionResult> partsNotInMs = new HashSet<>();
+    for (int i = 0; i < numOfParts; i++) {
+      PartitionResult result = new PartitionResult();
+      result.setPartitionName("city=dummyCity_" + String.valueOf(i));
+      partsNotInMs.add(result);
+    }
+    return partsNotInMs;
+  }
+
+  /**
+   * Tests the number of times Hive.createPartitions calls are executed with total number of
+   * partitions to be added are equally divisible by batch size
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testNumberOfCreatePartitionCalls() throws Exception {
+    // create 10 dummy partitions
+    Set<PartitionResult> partsNotInMs = createPartsNotInMs(10);
+    Hive spyDb = Mockito.spy(db);
+    // batch size of 5 and decaying factor of 2
+    ddlTask.createPartitionsInBatches(spyDb, repairOutput, partsNotInMs, table, 5, 2, 0);
+    // there should be 2 calls to create partitions with each batch size of 5
+    ArgumentCaptor<AddPartitionDesc> argument = ArgumentCaptor.forClass(AddPartitionDesc.class);
+    Mockito.verify(spyDb, Mockito.times(2)).createPartitions(argument.capture());
+    // confirm the batch sizes were 5, 5 in the two calls to create partitions
+    List<AddPartitionDesc> apds = argument.getAllValues();
+    int retryAttempt = 1;
+    Assert.assertEquals(String.format("Unexpected batch size in retry attempt %d ", retryAttempt++),
+        5, apds.get(0).getPartitionCount());
+    Assert.assertEquals(String.format("Unexpected batch size in retry attempt %d ", retryAttempt++),
+        5, apds.get(1).getPartitionCount());
+  }
+
+  /**
+   * Tests the number of times Hive.createPartitions calls are executed with total number of
+   * partitions to be added are not exactly divisible by batch size
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testUnevenNumberOfCreatePartitionCalls() throws Exception {
+    // create 9 dummy partitions
+    Set<PartitionResult> partsNotInMs = createPartsNotInMs(9);
+    Hive spyDb = Mockito.spy(db);
+    // batch size of 5 and decaying factor of 2
+    ddlTask.createPartitionsInBatches(spyDb, repairOutput, partsNotInMs, table, 5, 2, 0);
+    // there should be 2 calls to create partitions with batch sizes of 5, 4
+    ArgumentCaptor<AddPartitionDesc> argument = ArgumentCaptor.forClass(AddPartitionDesc.class);
+    Mockito.verify(spyDb, Mockito.times(2)).createPartitions(argument.capture());
+    // confirm the batch sizes were 5, 4 in the two calls to create partitions
+    List<AddPartitionDesc> apds = argument.getAllValues();
+    int retryAttempt = 1;
+    Assert.assertEquals(String.format("Unexpected batch size in retry attempt %d ", retryAttempt++),
+        5, apds.get(0).getPartitionCount());
+    Assert.assertEquals(String.format("Unexpected batch size in retry attempt %d ", retryAttempt++),
+        4, apds.get(1).getPartitionCount());
+  }
+
+  /**
+   * Tests the number of times Hive.createPartitions calls are executed with total number of
+   * partitions exactly equal to batch size
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testEqualNumberOfPartitions() throws Exception {
+    // create 13 dummy partitions
+    Set<PartitionResult> partsNotInMs = createPartsNotInMs(13);
+    Hive spyDb = Mockito.spy(db);
+    // batch size of 13 and decaying factor of 2
+    ddlTask.createPartitionsInBatches(spyDb, repairOutput, partsNotInMs, table, 13, 2, 0);
+    // there should be 1 call to create partitions with batch sizes of 13
+    ArgumentCaptor<AddPartitionDesc> argument = ArgumentCaptor.forClass(AddPartitionDesc.class);
+    Mockito.verify(spyDb, Mockito.times(1)).createPartitions(argument.capture());
+    Assert.assertEquals("Unexpected number of batch size", 13,
+        argument.getValue().getPartitionCount());
+  }
+
+  /**
+   * Tests the number of times Hive.createPartitions calls are executed with total number of
+   * partitions to is less than batch size
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testSmallNumberOfPartitions() throws Exception {
+    // create 10 dummy partitions
+    Set<PartitionResult> partsNotInMs = createPartsNotInMs(10);
+    Hive spyDb = Mockito.spy(db);
+    // batch size of 20 and decaying factor of 2
+    ddlTask.createPartitionsInBatches(spyDb, repairOutput, partsNotInMs, table, 20, 2, 0);
+    // there should be 1 call to create partitions with batch sizes of 10
+    Mockito.verify(spyDb, Mockito.times(1)).createPartitions(Mockito.any(AddPartitionDesc.class));
+    ArgumentCaptor<AddPartitionDesc> argument = ArgumentCaptor.forClass(AddPartitionDesc.class);
+    Mockito.verify(spyDb).createPartitions(argument.capture());
+    Assert.assertEquals("Unexpected number of batch size", 10,
+        argument.getValue().getPartitionCount());
+  }
+
+  /**
+   * Tests the number of calls to createPartitions and the respective batch sizes when first call to
+   * createPartitions throws HiveException. The batch size should be reduced by the decayingFactor
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testBatchingWhenException() throws Exception {
+    // create 13 dummy partitions
+    Set<PartitionResult> partsNotInMs = createPartsNotInMs(23);
+    Hive spyDb = Mockito.spy(db);
+    // first call to createPartitions should throw exception
+    Mockito.doThrow(HiveException.class).doCallRealMethod().doCallRealMethod().when(spyDb)
+        .createPartitions(Mockito.any(AddPartitionDesc.class));
+
+    // test with a batch size of 30 and decaying factor of 2
+    ddlTask.createPartitionsInBatches(spyDb, repairOutput, partsNotInMs, table, 30, 2, 0);
+    // confirm the batch sizes were 23, 15, 8 in the three calls to create partitions
+    ArgumentCaptor<AddPartitionDesc> argument = ArgumentCaptor.forClass(AddPartitionDesc.class);
+    // there should be 3 calls to create partitions with batch sizes of 23, 15, 8
+    Mockito.verify(spyDb, Mockito.times(3)).createPartitions(argument.capture());
+    List<AddPartitionDesc> apds = argument.getAllValues();
+    int retryAttempt = 1;
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 23,
+        apds.get(0).getPartitionCount());
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 15,
+        apds.get(1).getPartitionCount());
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 8,
+        apds.get(2).getPartitionCount());
+  }
+
+  /**
+   * Tests the retries exhausted case when Hive.createPartitions method call always keep throwing
+   * HiveException. The batch sizes should exponentially decreased based on the decaying factor and
+   * ultimately give up when it reaches 0
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testRetriesExhaustedBatchSize() throws Exception {
+    Set<PartitionResult> partsNotInMs = createPartsNotInMs(17);
+    Hive spyDb = Mockito.spy(db);
+    Mockito.doThrow(HiveException.class).when(spyDb)
+        .createPartitions(Mockito.any(AddPartitionDesc.class));
+    // batch size of 5 and decaying factor of 2
+    Exception ex = null;
+    try {
+      ddlTask.createPartitionsInBatches(spyDb, repairOutput, partsNotInMs, table, 30, 2, 0);
+    } catch (Exception retryEx) {
+      ex = retryEx;
+    }
+    Assert.assertFalse("Exception was expected but was not thrown", ex == null);
+    Assert.assertTrue("Unexpected class of exception thrown", ex instanceof RetryException);
+    // there should be 5 calls to create partitions with batch sizes of 17, 15, 7, 3, 1
+    ArgumentCaptor<AddPartitionDesc> argument = ArgumentCaptor.forClass(AddPartitionDesc.class);
+    Mockito.verify(spyDb, Mockito.times(5)).createPartitions(argument.capture());
+    List<AddPartitionDesc> apds = argument.getAllValues();
+    int retryAttempt = 1;
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 17,
+        apds.get(0).getPartitionCount());
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 15,
+        apds.get(1).getPartitionCount());
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 7,
+        apds.get(2).getPartitionCount());
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 3,
+        apds.get(3).getPartitionCount());
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 1,
+        apds.get(4).getPartitionCount());
+  }
+
+  /**
+   * Tests the maximum retry attempts provided by configuration
+   * @throws Exception
+   */
+  @Test
+  public void testMaxRetriesReached() throws Exception {
+    Set<PartitionResult> partsNotInMs = createPartsNotInMs(17);
+    Hive spyDb = Mockito.spy(db);
+    Mockito.doThrow(HiveException.class).when(spyDb)
+        .createPartitions(Mockito.any(AddPartitionDesc.class));
+    // batch size of 5 and decaying factor of 2
+    Exception ex = null;
+    try {
+      ddlTask.createPartitionsInBatches(spyDb, repairOutput, partsNotInMs, table, 30, 2, 2);
+    } catch (Exception retryEx) {
+      ex = retryEx;
+    }
+    Assert.assertFalse("Exception was expected but was not thrown", ex == null);
+    Assert.assertTrue("Unexpected class of exception thrown", ex instanceof RetryException);
+    ArgumentCaptor<AddPartitionDesc> argument = ArgumentCaptor.forClass(AddPartitionDesc.class);
+    Mockito.verify(spyDb, Mockito.times(2)).createPartitions(argument.capture());
+    List<AddPartitionDesc> apds = argument.getAllValues();
+    int retryAttempt = 1;
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 17,
+        apds.get(0).getPartitionCount());
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 15,
+        apds.get(1).getPartitionCount());
+  }
+
+  /**
+   * Tests when max number of retries is set to 1. In this case the number of retries should
+   * be specified
+   * @throws Exception
+   */
+  @Test
+  public void testOneMaxRetries() throws Exception {
+    Set<PartitionResult> partsNotInMs = createPartsNotInMs(17);
+    Hive spyDb = Mockito.spy(db);
+    Mockito.doThrow(HiveException.class).when(spyDb)
+        .createPartitions(Mockito.any(AddPartitionDesc.class));
+    // batch size of 5 and decaying factor of 2
+    Exception ex = null;
+    try {
+      ddlTask.createPartitionsInBatches(spyDb, repairOutput, partsNotInMs, table, 30, 2, 1);
+    } catch (Exception retryEx) {
+      ex = retryEx;
+    }
+    Assert.assertFalse("Exception was expected but was not thrown", ex == null);
+    Assert.assertTrue("Unexpected class of exception thrown", ex instanceof RetryException);
+    // there should be 5 calls to create partitions with batch sizes of 17, 15, 7, 3, 1
+    ArgumentCaptor<AddPartitionDesc> argument = ArgumentCaptor.forClass(AddPartitionDesc.class);
+    Mockito.verify(spyDb, Mockito.times(1)).createPartitions(argument.capture());
+    List<AddPartitionDesc> apds = argument.getAllValues();
+    int retryAttempt = 1;
+    Assert.assertEquals(
+        String.format("Unexpected batch size in retry attempt %d ", retryAttempt++), 17,
+        apds.get(0).getPartitionCount());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/queries/clientpositive/msck_repair_0.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/msck_repair_0.q b/ql/src/test/queries/clientpositive/msck_repair_0.q
index 2254233..cb291fe 100644
--- a/ql/src/test/queries/clientpositive/msck_repair_0.q
+++ b/ql/src/test/queries/clientpositive/msck_repair_0.q
@@ -7,15 +7,23 @@ CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING, p2 STRING);
 
 MSCK TABLE repairtable;
 
+show partitions repairtable;
+
 dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b;
 dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b/datafile;
 
 MSCK TABLE default.repairtable;
 
+show partitions default.repairtable;
+
 MSCK REPAIR TABLE default.repairtable;
 
+show partitions default.repairtable;
+
 MSCK TABLE repairtable;
 
+show partitions repairtable;
+
 set hive.mapred.mode=strict;
 
 dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=e/p2=f/p3=g;
@@ -23,4 +31,6 @@ dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=e/p2=f/p3=g/datafile;
 
 MSCK REPAIR TABLE default.repairtable;
 
+show partitions default.repairtable;
+
 DROP TABLE default.repairtable;

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/queries/clientpositive/msck_repair_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/msck_repair_1.q b/ql/src/test/queries/clientpositive/msck_repair_1.q
index ea596cb..21aca3b 100644
--- a/ql/src/test/queries/clientpositive/msck_repair_1.q
+++ b/ql/src/test/queries/clientpositive/msck_repair_1.q
@@ -6,13 +6,21 @@ CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING, p2 STRING);
 
 MSCK TABLE repairtable;
 
+SHOW PARTITIONS repairtable;
+
 dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b;
 dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b/datafile;
 
 MSCK TABLE default.repairtable;
 
+SHOW PARTITIONS default.repairtable;
+
 MSCK REPAIR TABLE default.repairtable;
 
+SHOW PARTITIONS default.repairtable;
+
 MSCK TABLE repairtable;
 
+SHOW PARTITIONS repairtable;
+
 DROP TABLE default.repairtable;

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/queries/clientpositive/msck_repair_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/msck_repair_2.q b/ql/src/test/queries/clientpositive/msck_repair_2.q
index d833821..77785e9 100644
--- a/ql/src/test/queries/clientpositive/msck_repair_2.q
+++ b/ql/src/test/queries/clientpositive/msck_repair_2.q
@@ -7,14 +7,19 @@ CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING, p2 STRING);
 
 MSCK TABLE repairtable;
 
+show partitions repairtable;
+
 dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b;
 dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b/datafile;
 dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/datafile;
 
 MSCK TABLE default.repairtable;
+show partitions repairtable;
 
 MSCK REPAIR TABLE default.repairtable;
+show partitions repairtable;
 
 MSCK TABLE repairtable;
+show partitions repairtable;
 
 DROP TABLE default.repairtable;

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/queries/clientpositive/msck_repair_3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/msck_repair_3.q b/ql/src/test/queries/clientpositive/msck_repair_3.q
index fdefca1..f42443f 100644
--- a/ql/src/test/queries/clientpositive/msck_repair_3.q
+++ b/ql/src/test/queries/clientpositive/msck_repair_3.q
@@ -5,13 +5,17 @@ DROP TABLE IF EXISTS repairtable;
 CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING, p2 STRING);
 
 MSCK TABLE repairtable;
+show partitions repairtable;
 
 dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/p3=b;
 
 MSCK TABLE default.repairtable;
+show partitions repairtable;
 
 MSCK REPAIR TABLE default.repairtable;
+show partitions repairtable;
 
 MSCK TABLE repairtable;
+show partitions repairtable;
 
 DROP TABLE default.repairtable;

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/queries/clientpositive/msck_repair_batchsize.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/msck_repair_batchsize.q b/ql/src/test/queries/clientpositive/msck_repair_batchsize.q
index e56e97a..a44c00e 100644
--- a/ql/src/test/queries/clientpositive/msck_repair_batchsize.q
+++ b/ql/src/test/queries/clientpositive/msck_repair_batchsize.q
@@ -14,10 +14,13 @@ dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=b/p2=a/datafile;
 dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=c/p2=a/datafile;
 
 MSCK TABLE default.repairtable;
+show partitions default.repairtable;
 
 MSCK REPAIR TABLE default.repairtable;
+show partitions default.repairtable;
 
 MSCK TABLE repairtable;
+show partitions repairtable;
 
 DROP TABLE default.repairtable;
 
@@ -28,5 +31,6 @@ CREATE TABLE `repairtable`( `col` string) PARTITIONED BY (  `p1` string,  `p2` s
 dfs -touchz ${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable/p1=c/p2=a/p3=b/datafile;
 set hive.mv.files.thread=1;
 MSCK TABLE repairtable;
+show partitions repairtable;
 
 DROP TABLE default.repairtable;

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/results/clientpositive/msck_repair_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/msck_repair_0.q.out b/ql/src/test/results/clientpositive/msck_repair_0.q.out
index 2e0d9dc..f48cc78 100644
--- a/ql/src/test/results/clientpositive/msck_repair_0.q.out
+++ b/ql/src/test/results/clientpositive/msck_repair_0.q.out
@@ -16,6 +16,12 @@ PREHOOK: Output: default@repairtable
 POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: MSCK TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -23,6 +29,12 @@ POSTHOOK: query: MSCK TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=c/p2=a
+PREHOOK: query: show partitions default.repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions default.repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: MSCK REPAIR TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -30,13 +42,27 @@ POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=c/p2=a
-Repair: Added partition to metastore default.repairtable:p1=c/p2=a
+#### A masked pattern was here ####
+PREHOOK: query: show partitions default.repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions default.repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=c/p2=a
 PREHOOK: query: MSCK TABLE repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
 POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=c/p2=a
 PREHOOK: query: MSCK REPAIR TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -44,7 +70,15 @@ POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=e/p2=f
-Repair: Added partition to metastore default.repairtable:p1=e/p2=f
+#### A masked pattern was here ####
+PREHOOK: query: show partitions default.repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions default.repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=c/p2=a
+p1=e/p2=f
 PREHOOK: query: DROP TABLE default.repairtable
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@repairtable

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/results/clientpositive/msck_repair_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/msck_repair_1.q.out b/ql/src/test/results/clientpositive/msck_repair_1.q.out
index 3f2fe75..5f94246 100644
--- a/ql/src/test/results/clientpositive/msck_repair_1.q.out
+++ b/ql/src/test/results/clientpositive/msck_repair_1.q.out
@@ -16,6 +16,12 @@ PREHOOK: Output: default@repairtable
 POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
+PREHOOK: query: SHOW PARTITIONS repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: SHOW PARTITIONS repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: MSCK TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -23,6 +29,12 @@ POSTHOOK: query: MSCK TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=c/p2=a
+PREHOOK: query: SHOW PARTITIONS default.repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: SHOW PARTITIONS default.repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: MSCK REPAIR TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -30,13 +42,27 @@ POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=c/p2=a
-Repair: Added partition to metastore default.repairtable:p1=c/p2=a
+#### A masked pattern was here ####
+PREHOOK: query: SHOW PARTITIONS default.repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: SHOW PARTITIONS default.repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=c/p2=a
 PREHOOK: query: MSCK TABLE repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
 POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
+PREHOOK: query: SHOW PARTITIONS repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: SHOW PARTITIONS repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=c/p2=a
 PREHOOK: query: DROP TABLE default.repairtable
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@repairtable

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/results/clientpositive/msck_repair_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/msck_repair_2.q.out b/ql/src/test/results/clientpositive/msck_repair_2.q.out
index 3f2fe75..c702f37 100644
--- a/ql/src/test/results/clientpositive/msck_repair_2.q.out
+++ b/ql/src/test/results/clientpositive/msck_repair_2.q.out
@@ -16,6 +16,12 @@ PREHOOK: Output: default@repairtable
 POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: MSCK TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -23,6 +29,12 @@ POSTHOOK: query: MSCK TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=c/p2=a
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: MSCK REPAIR TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -30,13 +42,27 @@ POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=c/p2=a
-Repair: Added partition to metastore default.repairtable:p1=c/p2=a
+#### A masked pattern was here ####
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=c/p2=a
 PREHOOK: query: MSCK TABLE repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
 POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=c/p2=a
 PREHOOK: query: DROP TABLE default.repairtable
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@repairtable

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/results/clientpositive/msck_repair_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/msck_repair_3.q.out b/ql/src/test/results/clientpositive/msck_repair_3.q.out
index 3f2fe75..c702f37 100644
--- a/ql/src/test/results/clientpositive/msck_repair_3.q.out
+++ b/ql/src/test/results/clientpositive/msck_repair_3.q.out
@@ -16,6 +16,12 @@ PREHOOK: Output: default@repairtable
 POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: MSCK TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -23,6 +29,12 @@ POSTHOOK: query: MSCK TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=c/p2=a
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: MSCK REPAIR TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -30,13 +42,27 @@ POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=c/p2=a
-Repair: Added partition to metastore default.repairtable:p1=c/p2=a
+#### A masked pattern was here ####
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=c/p2=a
 PREHOOK: query: MSCK TABLE repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
 POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=c/p2=a
 PREHOOK: query: DROP TABLE default.repairtable
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@repairtable

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out b/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out
index ba99024..aed9271 100644
--- a/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out
+++ b/ql/src/test/results/clientpositive/msck_repair_batchsize.q.out
@@ -23,6 +23,12 @@ POSTHOOK: query: MSCK TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=a/p2=a	repairtable:p1=b/p2=a	repairtable:p1=c/p2=a
+PREHOOK: query: show partitions default.repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions default.repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: MSCK REPAIR TABLE default.repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
@@ -30,15 +36,31 @@ POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=a/p2=a	repairtable:p1=b/p2=a	repairtable:p1=c/p2=a
-Repair: Added partition to metastore default.repairtable:p1=a/p2=a
-Repair: Added partition to metastore default.repairtable:p1=b/p2=a
-Repair: Added partition to metastore default.repairtable:p1=c/p2=a
+#### A masked pattern was here ####
+PREHOOK: query: show partitions default.repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions default.repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=a/p2=a
+p1=b/p2=a
+p1=c/p2=a
 PREHOOK: query: MSCK TABLE repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable
 POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
+p1=a/p2=a
+p1=b/p2=a
+p1=c/p2=a
 PREHOOK: query: DROP TABLE default.repairtable
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@repairtable
@@ -64,6 +86,12 @@ POSTHOOK: query: MSCK TABLE repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=c/p2=a
+PREHOOK: query: show partitions repairtable
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable
+POSTHOOK: query: show partitions repairtable
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable
 PREHOOK: query: DROP TABLE default.repairtable
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@repairtable

http://git-wip-us.apache.org/repos/asf/hive/blob/3f25911a/ql/src/test/results/clientpositive/repair.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/repair.q.out b/ql/src/test/results/clientpositive/repair.q.out
index c183464..581afe8 100644
--- a/ql/src/test/results/clientpositive/repair.q.out
+++ b/ql/src/test/results/clientpositive/repair.q.out
@@ -30,8 +30,7 @@ POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
 POSTHOOK: type: MSCK
 POSTHOOK: Output: default@repairtable
 Partitions not in metastore:	repairtable:p1=a/p2=a	repairtable:p1=b/p2=a
-Repair: Added partition to metastore default.repairtable:p1=a/p2=a
-Repair: Added partition to metastore default.repairtable:p1=b/p2=a
+#### A masked pattern was here ####
 PREHOOK: query: MSCK TABLE repairtable
 PREHOOK: type: MSCK
 PREHOOK: Output: default@repairtable


Mime
View raw message