hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ekoif...@apache.org
Subject [11/11] hive git commit: HIVE-15844 Make ReduceSinkOperator independent of Acid (Eugene Koifman, reviewed by Prasanth Jayachandran, Matt McCline)
Date Wed, 01 Mar 2017 19:26:49 GMT
HIVE-15844 Make ReduceSinkOperator independent of Acid (Eugene Koifman, reviewed by Prasanth Jayachandran, Matt McCline)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/917221e8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/917221e8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/917221e8

Branch: refs/heads/master
Commit: 917221e8378ec48ea05ef6b6c7d9515609b8ec01
Parents: 03216f5
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Wed Mar 1 11:26:16 2017 -0800
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Wed Mar 1 11:26:16 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    4 +-
 .../insert_into_dynamic_partitions.q.out        |    1 +
 .../insert_overwrite_dynamic_partitions.q.out   |    1 +
 .../hadoop/hive/ql/exec/ReduceSinkOperator.java |   89 +-
 .../BucketingSortingReduceSinkOptimizer.java    |    3 -
 .../optimizer/SortedDynPartitionOptimizer.java  |   45 +-
 ...tedDynPartitionTimeGranularityOptimizer.java |    8 +-
 .../hive/ql/optimizer/physical/Vectorizer.java  |    8 +-
 .../hadoop/hive/ql/plan/FileSinkDesc.java       |   10 +-
 .../apache/hadoop/hive/ql/plan/PlanUtils.java   |    2 +-
 .../hadoop/hive/ql/plan/ReduceSinkDesc.java     |   16 +-
 .../hive/ql/plan/VectorReduceSinkDesc.java      |    7 -
 .../hive/ql/txn/compactor/CompactorMR.java      |    2 +-
 .../dynpart_sort_optimization_acid.q            |  162 +--
 .../clientpositive/autoColumnStats_4.q.out      |    1 +
 .../dynpart_sort_optimization_acid2.q.out       |    1 +
 .../llap/dynamic_semijoin_reduction_3.q.out     |    7 +
 .../llap/dynpart_sort_opt_vectorization.q.out   |   12 +
 .../llap/dynpart_sort_optimization.q.out        |   18 +
 .../llap/dynpart_sort_optimization2.q.out       |    2 +
 .../llap/dynpart_sort_optimization_acid.q.out   | 1211 +++++++++---------
 .../results/clientpositive/llap/sqlmerge.q.out  |    4 +
 .../llap/vector_adaptor_usage_mode.q.out        |    2 +-
 .../llap/vector_auto_smb_mapjoin_14.q.out       |   10 +-
 .../llap/vector_between_columns.q.out           |    4 +-
 .../clientpositive/llap/vector_between_in.q.out |   32 +-
 .../llap/vector_binary_join_groupby.q.out       |    8 +-
 .../clientpositive/llap/vector_char_2.q.out     |    8 +-
 .../llap/vector_char_mapjoin1.q.out             |   12 +-
 .../llap/vector_char_simple.q.out               |    2 +-
 .../clientpositive/llap/vector_coalesce.q.out   |   10 +-
 .../clientpositive/llap/vector_coalesce_2.q.out |    2 +-
 .../clientpositive/llap/vector_count.q.out      |    8 +-
 .../llap/vector_count_distinct.q.out            |    4 +-
 .../clientpositive/llap/vector_data_types.q.out |    2 +-
 .../llap/vector_decimal_aggregate.q.out         |    2 +-
 .../llap/vector_decimal_expressions.q.out       |    2 +-
 .../llap/vector_decimal_mapjoin.q.out           |    2 +-
 .../llap/vector_decimal_round.q.out             |    4 +-
 .../llap/vector_decimal_round_2.q.out           |    8 +-
 .../clientpositive/llap/vector_distinct_2.q.out |    2 +-
 .../clientpositive/llap/vector_groupby4.q.out   |    4 +-
 .../clientpositive/llap/vector_groupby6.q.out   |    4 +-
 .../clientpositive/llap/vector_groupby_3.q.out  |    2 +-
 .../llap/vector_groupby_mapjoin.q.out           |    4 +-
 .../llap/vector_groupby_reduce.q.out            |   14 +-
 .../clientpositive/llap/vector_if_expr.q.out    |    2 +-
 .../llap/vector_include_no_sel.q.out            |    4 +-
 .../clientpositive/llap/vector_inner_join.q.out |   18 +-
 .../clientpositive/llap/vector_interval_1.q.out |   16 +-
 .../clientpositive/llap/vector_interval_2.q.out |   20 +-
 .../llap/vector_interval_arithmetic.q.out       |   14 +-
 .../llap/vector_interval_mapjoin.q.out          |    2 +-
 .../clientpositive/llap/vector_join30.q.out     |   84 +-
 .../llap/vector_left_outer_join2.q.out          |    8 +-
 .../llap/vector_leftsemi_mapjoin.q.out          |  376 +++---
 .../llap/vector_nullsafe_join.q.out             |   36 +-
 .../llap/vector_number_compare_projection.q.out |    4 +-
 .../clientpositive/llap/vector_orderby_5.q.out  |    4 +-
 .../llap/vector_outer_join0.q.out               |    4 +-
 .../llap/vector_outer_join1.q.out               |   10 +-
 .../llap/vector_outer_join2.q.out               |    6 +-
 .../llap/vector_partition_diff_num_cols.q.out   |   10 +-
 .../llap/vector_partitioned_date_time.q.out     |   18 +-
 .../clientpositive/llap/vector_reduce1.q.out    |    2 +-
 .../clientpositive/llap/vector_reduce2.q.out    |    2 +-
 .../clientpositive/llap/vector_reduce3.q.out    |    2 +-
 .../llap/vector_reduce_groupby_decimal.q.out    |    4 +-
 .../llap/vector_string_concat.q.out             |    4 +-
 .../llap/vector_varchar_simple.q.out            |    2 +-
 .../llap/vector_when_case_null.q.out            |    2 +-
 .../clientpositive/llap/vectorization_0.q.out   |   24 +-
 .../clientpositive/llap/vectorization_7.q.out   |    4 +-
 .../clientpositive/llap/vectorization_8.q.out   |    4 +-
 .../llap/vectorization_short_regress.q.out      |   24 +-
 .../llap/vectorized_bucketmapjoin1.q.out        |    8 +-
 .../llap/vectorized_date_funcs.q.out            |    4 +-
 .../vectorized_dynamic_semijoin_reduction.q.out |   58 +-
 .../llap/vectorized_mapjoin.q.out               |    2 +-
 .../llap/vectorized_shufflejoin.q.out           |    4 +-
 .../llap/vectorized_timestamp_funcs.q.out       |   10 +-
 .../spark/vector_between_in.q.out               |   32 +-
 .../spark/vector_count_distinct.q.out           |    4 +-
 .../spark/vector_data_types.q.out               |    2 +-
 .../spark/vector_decimal_aggregate.q.out        |    2 +-
 .../spark/vector_distinct_2.q.out               |    2 +-
 .../clientpositive/spark/vector_groupby_3.q.out |    2 +-
 .../clientpositive/spark/vector_orderby_5.q.out |    4 +-
 .../spark/vector_outer_join1.q.out              |    2 +-
 .../spark/vector_outer_join2.q.out              |    2 +-
 .../spark/vector_string_concat.q.out            |    4 +-
 .../clientpositive/spark/vectorization_0.q.out  |   24 +-
 .../spark/vectorization_div0.q.out              |    4 +-
 .../spark/vectorization_short_regress.q.out     |   24 +-
 .../spark/vectorized_shufflejoin.q.out          |    4 +-
 .../spark/vectorized_timestamp_funcs.q.out      |   10 +-
 .../tez/vector_non_string_partition.q.out       |    4 +-
 .../clientpositive/tez/vectorization_div0.q.out |    4 +-
 .../tez/vectorization_limit.q.out               |   10 +-
 .../vector_binary_join_groupby.q.out            |    4 +-
 .../results/clientpositive/vector_char_2.q.out  |    4 +-
 .../clientpositive/vector_char_mapjoin1.q.out   |    6 +-
 .../clientpositive/vector_char_simple.q.out     |    2 +-
 .../clientpositive/vector_coalesce.q.out        |   10 +-
 .../clientpositive/vector_coalesce_2.q.out      |    2 +-
 .../results/clientpositive/vector_count.q.out   |    8 +-
 .../clientpositive/vector_data_types.q.out      |    2 +-
 .../vector_decimal_aggregate.q.out              |    2 +-
 .../vector_decimal_expressions.q.out            |    2 +-
 .../clientpositive/vector_decimal_round.q.out   |    4 +-
 .../clientpositive/vector_decimal_round_2.q.out |    8 +-
 .../clientpositive/vector_distinct_2.q.out      |    2 +-
 .../clientpositive/vector_empty_where.q.out     |    8 +-
 .../clientpositive/vector_groupby4.q.out        |    2 +-
 .../clientpositive/vector_groupby6.q.out        |    2 +-
 .../clientpositive/vector_groupby_3.q.out       |    2 +-
 .../clientpositive/vector_groupby_reduce.q.out  |    8 +-
 .../results/clientpositive/vector_if_expr.q.out |    2 +-
 .../clientpositive/vector_include_no_sel.q.out  |    2 +-
 .../clientpositive/vector_interval_1.q.out      |   16 +-
 .../vector_interval_arithmetic.q.out            |   14 +-
 .../vector_non_string_partition.q.out           |    4 +-
 .../clientpositive/vector_orderby_5.q.out       |    2 +-
 .../clientpositive/vector_outer_join1.q.out     |    2 +-
 .../clientpositive/vector_outer_join2.q.out     |    2 +-
 .../clientpositive/vector_outer_join3.q.out     |    6 +-
 .../clientpositive/vector_outer_join4.q.out     |    2 +-
 .../results/clientpositive/vector_reduce1.q.out |    2 +-
 .../results/clientpositive/vector_reduce2.q.out |    2 +-
 .../results/clientpositive/vector_reduce3.q.out |    2 +-
 .../vector_reduce_groupby_decimal.q.out         |    2 +-
 .../clientpositive/vector_string_concat.q.out   |    2 +-
 .../clientpositive/vector_varchar_simple.q.out  |    2 +-
 .../clientpositive/vector_when_case_null.q.out  |    2 +-
 .../clientpositive/vectorization_7.q.out        |    4 +-
 .../clientpositive/vectorization_8.q.out        |    4 +-
 .../clientpositive/vectorization_div0.q.out     |    4 +-
 .../clientpositive/vectorization_limit.q.out    |    8 +-
 .../vectorization_offset_limit.q.out            |    2 +-
 .../clientpositive/vectorized_date_funcs.q.out  |    2 +-
 .../clientpositive/vectorized_mapjoin2.q.out    |    2 +-
 .../clientpositive/vectorized_timestamp.q.out   |    2 +-
 .../vectorized_timestamp_funcs.q.out            |   10 +-
 .../objectinspector/ObjectInspectorUtils.java   |    5 +
 144 files changed, 1445 insertions(+), 1444 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 71bd19a..d13e03d 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1794,8 +1794,8 @@ public class HiveConf extends Configuration {
         "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager",
         "Set to org.apache.hadoop.hive.ql.lockmgr.DbTxnManager as part of turning on Hive\n" +
         "transactions, which also requires appropriate settings for hive.compactor.initiator.on,\n" +
-        "hive.compactor.worker.threads, hive.support.concurrency (true), hive.enforce.bucketing\n" +
-        "(true), and hive.exec.dynamic.partition.mode (nonstrict).\n" +
+        "hive.compactor.worker.threads, hive.support.concurrency (true),\n" +
+        "and hive.exec.dynamic.partition.mode (nonstrict).\n" +
         "The default DummyTxnManager replicates pre-Hive-0.13 behavior and provides\n" +
         "no transactions."),
     HIVE_TXN_STRICT_LOCKING_MODE("hive.txn.strict.locking.mode", true, "In strict mode non-ACID\n" +

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
index ce1ac16..e55b1c2 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
@@ -149,6 +149,7 @@ STAGE PLANS:
             compressed: false
             GlobalTableId: 1
             directory: ### BLOBSTORE_STAGING_PATH ###
+            Dp Sort State: PARTITION_BUCKET_SORTED
             NumFilesPerFileSink: 1
             Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
             Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ###

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
index 1ca24bc..660cebb 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
@@ -167,6 +167,7 @@ STAGE PLANS:
             compressed: false
             GlobalTableId: 1
             directory: ### BLOBSTORE_STAGING_PATH ###
+            Dp Sort State: PARTITION_BUCKET_SORTED
             NumFilesPerFileSink: 1
             Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
             Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ###

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
index 789d2a3..e03f4b7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
@@ -18,16 +18,15 @@
 
 package org.apache.hadoop.hive.ql.exec;
 
+import static org.apache.hadoop.hive.ql.optimizer.SortedDynPartitionOptimizer.BUCKET_NUMBER_COL_NAME;
 import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.UNIFORM;
 
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.List;
 import java.util.Random;
-import java.util.concurrent.Future;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -35,7 +34,6 @@ import org.apache.hadoop.hive.ql.CompilationOpContext;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.io.HiveKey;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
@@ -44,15 +42,12 @@ import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.Serializer;
-import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
 import org.apache.hadoop.io.BinaryComparable;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.LongWritable;
@@ -80,11 +75,13 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
   private transient ObjectInspector[] partitionObjectInspectors;
   private transient ObjectInspector[] bucketObjectInspectors;
   private transient int buckColIdxInKey;
-  private transient int buckColIdxInKeyForAcid = -1;
+  /**
+   * {@link org.apache.hadoop.hive.ql.optimizer.SortedDynPartitionOptimizer}
+   */
+  private transient int buckColIdxInKeyForSdpo = -1;
   private boolean firstRow;
   private transient int tag;
   private boolean skipTag = false;
-  private transient InspectableObject tempInspectableObject = new InspectableObject();
   private transient int[] valueIndex; // index for value(+ from keys, - from values)
 
   protected transient OutputCollector out;
@@ -143,12 +140,6 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
   // TODO: we only ever use one row of these at a time. Why do we need to cache multiple?
   protected transient Object[][] cachedKeys;
 
-  private StructField recIdField; // field to look for record identifier in
-  private StructField bucketField; // field to look for bucket in record identifier
-  private StructObjectInspector acidRowInspector; // row inspector used by acid options
-  private StructObjectInspector recIdInspector; // OI for the record identifier
-  private IntObjectInspector bucketInspector; // OI for the bucket field in the record id
-
   protected transient long numRows = 0;
   protected transient long cntr = 1;
   protected transient long logEveryNRows = 0;
@@ -186,8 +177,8 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
       keyEval = new ExprNodeEvaluator[keys.size()];
       int i = 0;
       for (ExprNodeDesc e : keys) {
-        if (e instanceof ExprNodeConstantDesc && ("_bucket_number").equals(((ExprNodeConstantDesc)e).getValue())) {
-          buckColIdxInKeyForAcid = i;
+        if (e instanceof ExprNodeConstantDesc && (BUCKET_NUMBER_COL_NAME).equals(((ExprNodeConstantDesc)e).getValue())) {
+          buckColIdxInKeyForSdpo = i;
         }
         keyEval[i++] = ExprNodeEvaluatorFactory.get(e);
       }
@@ -319,20 +310,6 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
         // TODO: this is fishy - we init object inspectors based on first tag. We
         //       should either init for each tag, or if rowInspector doesn't really
         //       matter, then we can create this in ctor and get rid of firstRow.
-        if (conf.getWriteType() == AcidUtils.Operation.UPDATE ||
-            conf.getWriteType() == AcidUtils.Operation.DELETE) {
-          assert rowInspector instanceof StructObjectInspector :
-              "Expected rowInspector to be instance of StructObjectInspector but it is a " +
-                  rowInspector.getClass().getName();
-          acidRowInspector = (StructObjectInspector)rowInspector;
-          // The record identifier is always in the first column
-          recIdField = acidRowInspector.getAllStructFieldRefs().get(0);
-          recIdInspector = (StructObjectInspector)recIdField.getFieldObjectInspector();
-          // The bucket field is in the second position
-          bucketField = recIdInspector.getAllStructFieldRefs().get(1);
-          bucketInspector = (IntObjectInspector)bucketField.getFieldObjectInspector();
-        }
-
         if (isLogInfoEnabled) {
           LOG.info("keys are " + conf.getOutputKeyColumnNames() + " num distributions: " +
               conf.getNumDistributionKeys());
@@ -360,14 +337,9 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
       if (bucketEval != null) {
         bucketNumber = computeBucketNumber(row, conf.getNumBuckets());
         cachedKeys[0][buckColIdxInKey] = new Text(String.valueOf(bucketNumber));
-      } else if (conf.getWriteType() == AcidUtils.Operation.UPDATE ||
-          conf.getWriteType() == AcidUtils.Operation.DELETE) {
-        // In the non-partitioned case we still want to compute the bucket number for updates and
-        // deletes.
-        bucketNumber = computeBucketNumber(row, conf.getNumBuckets());
-        if (buckColIdxInKeyForAcid != -1) {
-          cachedKeys[0][buckColIdxInKeyForAcid] = new Text(String.valueOf(bucketNumber));
-        }
+      }
+      if (buckColIdxInKeyForSdpo != -1) {
+        cachedKeys[0][buckColIdxInKeyForSdpo] = new Text(String.valueOf(bucketNumber));
       }
 
       HiveKey firstKey = toHiveKey(cachedKeys[0], tag, null);
@@ -427,24 +399,11 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
   }
 
   private int computeBucketNumber(Object row, int numBuckets) throws HiveException {
-    if (conf.getWriteType() == AcidUtils.Operation.UPDATE ||
-        conf.getWriteType() == AcidUtils.Operation.DELETE) {
-      // We don't need to evaluate the hash code.  Instead read the bucket number directly from
-      // the row.  I don't need to evaluate any expressions as I know I am reading the ROW__ID
-      // column directly.
-      Object recIdValue = acidRowInspector.getStructFieldData(row, recIdField);
-      int buckNum = bucketInspector.get(recIdInspector.getStructFieldData(recIdValue, bucketField));
-      if (isLogTraceEnabled) {
-        LOG.trace("Acid choosing bucket number " + buckNum);
-      }
-      return buckNum;
-    } else {
-      Object[] bucketFieldValues = new Object[bucketEval.length];
-      for (int i = 0; i < bucketEval.length; i++) {
-        bucketFieldValues[i] = bucketEval[i].evaluate(row);
-      }
-      return ObjectInspectorUtils.getBucketNumber(bucketFieldValues, bucketObjectInspectors, numBuckets);
+    Object[] bucketFieldValues = new Object[bucketEval.length];
+    for (int i = 0; i < bucketEval.length; i++) {
+      bucketFieldValues[i] = bucketEval[i].evaluate(row);
     }
+    return ObjectInspectorUtils.getBucketNumber(bucketFieldValues, bucketObjectInspectors, numBuckets);
   }
 
   private void populateCachedDistributionKeys(Object row, int index) throws HiveException {
@@ -477,23 +436,23 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
     return hash.hash(firstKey.getBytes(), firstKey.getDistKeyLength(), 0);
   }
 
+  /**
+   * For Acid Update/Delete case, we expect a single partitionEval of the form
+   * UDFToInteger(ROW__ID) and buckNum == -1 so that the result of this method
+   * is to return the bucketId extracted from ROW__ID unless it optimized by
+   * {@link org.apache.hadoop.hive.ql.optimizer.SortedDynPartitionOptimizer} 
+   */
   private int computeHashCode(Object row, int buckNum) throws HiveException {
     // Evaluate the HashCode
     int keyHashCode = 0;
     if (partitionEval.length == 0) {
-      // If no partition cols and not doing an update or delete, just distribute the data uniformly
+      // If no partition cols, just distribute the data uniformly
       // to provide better load balance. If the requirement is to have a single reducer, we should
       // set the number of reducers to 1. Use a constant seed to make the code deterministic.
-      // For acid operations make sure to send all records with the same key to the same
-      // FileSinkOperator, as the RecordUpdater interface can't manage multiple writers for a file.
-      if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
-        if (random == null) {
-          random = new Random(12345);
-        }
-        keyHashCode = random.nextInt();
-      } else {
-        keyHashCode = 1;
+      if (random == null) {
+        random = new Random(12345);
       }
+      keyHashCode = random.nextInt();
     } else {
       Object[] bucketFieldValues = new Object[partitionEval.length];
       for(int i = 0; i < partitionEval.length; i++) {

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
index 8f40998..ac1c803 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
@@ -401,9 +401,6 @@ public class BucketingSortingReduceSinkOptimizer extends Transform {
         return null;
       }
 
-      assert fsOp.getConf().getWriteType() == rsOp.getConf().getWriteType() :
-        "WriteType mismatch. fsOp is " + fsOp.getConf().getWriteType() +
-          "; rsOp is " + rsOp.getConf().getWriteType();
       // Don't do this optimization with updates or deletes
       if (fsOp.getConf().getWriteType() == AcidUtils.Operation.UPDATE ||
         fsOp.getConf().getWriteType() == AcidUtils.Operation.DELETE) {

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
index 926386b..76aa39f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.hive.ql.optimizer;
 
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -40,7 +40,7 @@ import org.apache.hadoop.hive.ql.exec.SelectOperator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.Utilities.ReduceField;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
+import org.apache.hadoop.hive.ql.io.RecordIdentifier;
 import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
 import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
 import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
 import org.apache.hadoop.hive.ql.lib.Rule;
 import org.apache.hadoop.hive.ql.lib.RuleRegExp;
 import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
 import org.apache.hadoop.hive.ql.parse.ParseContext;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
@@ -81,7 +82,7 @@ import com.google.common.collect.Sets;
  */
 public class SortedDynPartitionOptimizer extends Transform {
 
-  private static final String BUCKET_NUMBER_COL_NAME = "_bucket_number";
+  public static final String BUCKET_NUMBER_COL_NAME = "_bucket_number";
   @Override
   public ParseContext transform(ParseContext pCtx) throws SemanticException {
 
@@ -191,9 +192,21 @@ public class SortedDynPartitionOptimizer extends Transform {
         // When doing updates and deletes we always want to sort on the rowid because the ACID
         // reader will expect this sort order when doing reads.  So
         // ignore whatever comes from the table and enforce this sort order instead.
-        sortPositions = Arrays.asList(0);
-        sortOrder = Arrays.asList(1); // 1 means asc, could really use enum here in the thrift if
-        bucketColumns = new ArrayList<>(); // Bucketing column is already present in ROW__ID, which is specially handled in ReduceSink
+        sortPositions = Collections.singletonList(0);
+        sortOrder = Collections.singletonList(1); // 1 means asc, could really use enum here in the thrift if
+        bucketColumns = new ArrayList<>();
+        /**
+         * ROW__ID is always the 1st column of Insert representing Update/Delete operation
+         * (set up in {@link org.apache.hadoop.hive.ql.parse.UpdateDeleteSemanticAnalyzer})
+         * and we wrap it in UDFToInteger 
+         * (in {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer#getPartitionColsFromBucketColsForUpdateDelete(Operator, boolean)})
+         * which extracts bucketId from it
+         * see {@link org.apache.hadoop.hive.ql.udf.UDFToInteger#evaluate(RecordIdentifier)}*/
+        ColumnInfo ci = fsParent.getSchema().getSignature().get(0);
+        if(!VirtualColumn.ROWID.getTypeInfo().equals(ci.getType())) {
+          throw new IllegalStateException("expected 1st column to be ROW__ID but got wrong type: " + ci.toString());
+        }
+        bucketColumns.add(new ExprNodeColumnDesc(ci));
       } else {
         if (!destTable.getSortCols().isEmpty()) {
           // Sort columns specified by table
@@ -231,7 +244,7 @@ public class SortedDynPartitionOptimizer extends Transform {
 
       // Create ReduceSink operator
       ReduceSinkOperator rsOp = getReduceSinkOp(partitionPositions, sortPositions, sortOrder, sortNullOrder,
-          allRSCols, bucketColumns, numBuckets, fsParent, fsOp.getConf().getWriteType());
+          allRSCols, bucketColumns, numBuckets, fsParent);
 
       List<ExprNodeDesc> descs = new ArrayList<ExprNodeDesc>(allRSCols.size());
       List<String> colNames = new ArrayList<String>();
@@ -247,7 +260,7 @@ public class SortedDynPartitionOptimizer extends Transform {
         }
       }
       RowSchema selRS = new RowSchema(fsParent.getSchema());
-      if (!bucketColumns.isEmpty() || fsOp.getConf().getWriteType() == Operation.DELETE || fsOp.getConf().getWriteType() == Operation.UPDATE) {
+      if (!bucketColumns.isEmpty()) {
         descs.add(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, ReduceField.KEY.toString()+".'"+BUCKET_NUMBER_COL_NAME+"'", null, false));
         colNames.add("'"+BUCKET_NUMBER_COL_NAME+"'");
         ColumnInfo ci = new ColumnInfo(BUCKET_NUMBER_COL_NAME, TypeInfoFactory.stringTypeInfo, selRS.getSignature().get(0).getTabAlias(), true, true);
@@ -268,7 +281,7 @@ public class SortedDynPartitionOptimizer extends Transform {
 
       // Set if partition sorted or partition bucket sorted
       fsOp.getConf().setDpSortState(FileSinkDesc.DPSortState.PARTITION_SORTED);
-      if (bucketColumns.size() > 0 || fsOp.getConf().getWriteType() == Operation.DELETE || fsOp.getConf().getWriteType() == Operation.UPDATE) {
+      if (!bucketColumns.isEmpty()) {
         fsOp.getConf().setDpSortState(FileSinkDesc.DPSortState.PARTITION_BUCKET_SORTED);
       }
 
@@ -428,7 +441,7 @@ public class SortedDynPartitionOptimizer extends Transform {
     public ReduceSinkOperator getReduceSinkOp(List<Integer> partitionPositions,
         List<Integer> sortPositions, List<Integer> sortOrder, List<Integer> sortNullOrder,
         ArrayList<ExprNodeDesc> allCols, ArrayList<ExprNodeDesc> bucketColumns, int numBuckets,
-        Operator<? extends OperatorDesc> parent, AcidUtils.Operation writeType) throws SemanticException {
+        Operator<? extends OperatorDesc> parent) throws SemanticException {
 
       // Order of KEY columns
       // 1) Partition columns
@@ -441,7 +454,7 @@ public class SortedDynPartitionOptimizer extends Transform {
       int numPartAndBuck = partitionPositions.size();
 
       keyColsPosInVal.addAll(partitionPositions);
-      if (!bucketColumns.isEmpty() || writeType == Operation.DELETE || writeType == Operation.UPDATE) {
+      if (!bucketColumns.isEmpty()) {
         keyColsPosInVal.add(-1);
         numPartAndBuck += 1;
       }
@@ -450,7 +463,7 @@ public class SortedDynPartitionOptimizer extends Transform {
       // by default partition and bucket columns are sorted in ascending order
       Integer order = 1;
       if (sortOrder != null && !sortOrder.isEmpty()) {
-        if (sortOrder.get(0).intValue() == 0) {
+        if (sortOrder.get(0) == 0) {
           order = 0;
         }
       }
@@ -461,7 +474,7 @@ public class SortedDynPartitionOptimizer extends Transform {
 
       String orderStr = "";
       for (Integer i : newSortOrder) {
-        if(i.intValue() == 1) {
+        if(i == 1) {
           orderStr += "+";
         } else {
           orderStr += "-";
@@ -472,7 +485,7 @@ public class SortedDynPartitionOptimizer extends Transform {
       // nulls come first; otherwise nulls come last
       Integer nullOrder = order == 1 ? 0 : 1;
       if (sortNullOrder != null && !sortNullOrder.isEmpty()) {
-        if (sortNullOrder.get(0).intValue() == 0) {
+        if (sortNullOrder.get(0) == 0) {
           nullOrder = 0;
         } else {
           nullOrder = 1;
@@ -485,7 +498,7 @@ public class SortedDynPartitionOptimizer extends Transform {
 
       String nullOrderStr = "";
       for (Integer i : newSortNullOrder) {
-        if(i.intValue() == 0) {
+        if(i == 0) {
           nullOrderStr += "a";
         } else {
           nullOrderStr += "z";
@@ -563,7 +576,7 @@ public class SortedDynPartitionOptimizer extends Transform {
       // Number of reducers is set to default (-1)
       ReduceSinkDesc rsConf = new ReduceSinkDesc(keyCols, keyCols.size(), valCols,
           keyColNames, distinctColumnIndices, valColNames, -1, partCols, -1, keyTable,
-          valueTable, writeType);
+          valueTable);
       rsConf.setBucketCols(bucketColumns);
       rsConf.setNumBuckets(numBuckets);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
index 7a4f22a..7670dc1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
@@ -170,7 +170,7 @@ public class SortedDynPartitionTimeGranularityOptimizer extends Transform {
       List<Integer> sortNullOrder = new ArrayList<Integer>(1);
       sortNullOrder.add(0); // nulls first
       ReduceSinkOperator rsOp = getReduceSinkOp(keyPositions, sortOrder,
-          sortNullOrder, allRSCols, granularitySelOp, fsOp.getConf().getWriteType());
+          sortNullOrder, allRSCols, granularitySelOp);
 
       // Create backtrack SelectOp
       List<ExprNodeDesc> descs = new ArrayList<ExprNodeDesc>(allRSCols.size());
@@ -295,8 +295,8 @@ public class SortedDynPartitionTimeGranularityOptimizer extends Transform {
     }
 
     private ReduceSinkOperator getReduceSinkOp(List<Integer> keyPositions, List<Integer> sortOrder,
-        List<Integer> sortNullOrder, ArrayList<ExprNodeDesc> allCols, Operator<? extends OperatorDesc> parent,
-        AcidUtils.Operation writeType) throws SemanticException {
+        List<Integer> sortNullOrder, ArrayList<ExprNodeDesc> allCols, Operator<? extends OperatorDesc> parent
+    ) throws SemanticException {
 
       ArrayList<ExprNodeDesc> keyCols = Lists.newArrayList();
       // we will clone here as RS will update bucket column key with its
@@ -353,7 +353,7 @@ public class SortedDynPartitionTimeGranularityOptimizer extends Transform {
       // Number of reducers is set to default (-1)
       ReduceSinkDesc rsConf = new ReduceSinkDesc(keyCols, keyCols.size(), valCols,
           keyColNames, distinctColumnIndices, valColNames, -1, partCols, -1, keyTable,
-          valueTable, writeType);
+          valueTable);
 
       ArrayList<ColumnInfo> signature = new ArrayList<>();
       for (int index = 0; index < parent.getSchema().getSignature().size(); index++) {

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index fadbc20..f09bfa4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -2980,11 +2980,7 @@ public class Vectorizer implements PhysicalPlanResolver {
         HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCESINK_NEW_ENABLED);
 
     String engine = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE);
-
-    boolean acidChange =
-        desc.getWriteType() == AcidUtils.Operation.UPDATE ||
-            desc.getWriteType() == AcidUtils.Operation.DELETE;
-
+    
     boolean hasBuckets = desc.getBucketCols() != null && !desc.getBucketCols().isEmpty();
 
     boolean hasTopN = desc.getTopN() >= 0;
@@ -3004,7 +3000,6 @@ public class Vectorizer implements PhysicalPlanResolver {
     // Remember the condition variables for EXPLAIN regardless.
     vectorDesc.setIsVectorizationReduceSinkNativeEnabled(isVectorizationReduceSinkNativeEnabled);
     vectorDesc.setEngine(engine);
-    vectorDesc.setAcidChange(acidChange);
     vectorDesc.setHasBuckets(hasBuckets);
     vectorDesc.setHasTopN(hasTopN);
     vectorDesc.setUseUniformHash(useUniformHash);
@@ -3015,7 +3010,6 @@ public class Vectorizer implements PhysicalPlanResolver {
     // Many restrictions.
     if (!isVectorizationReduceSinkNativeEnabled ||
         !isTezOrSpark ||
-        acidChange ||
         hasBuckets ||
         hasTopN ||
         !useUniformHash ||

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 5cc1c45..4716adc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -438,7 +438,10 @@ public class FileSinkDesc extends AbstractOperatorDesc {
   public DPSortState getDpSortState() {
     return dpSortState;
   }
-
+  @Explain(displayName = "Dp Sort State")
+  public String getDpSortStateString() {
+    return getDpSortState() == DPSortState.NONE ? null : getDpSortState().toString();
+  }
   public void setDpSortState(DPSortState dpSortState) {
     this.dpSortState = dpSortState;
   }
@@ -450,7 +453,10 @@ public class FileSinkDesc extends AbstractOperatorDesc {
   public AcidUtils.Operation getWriteType() {
     return writeType;
   }
-
+  @Explain(displayName = "Write Type")
+  public String getWriteTypeString() {
+    return getWriteType() == AcidUtils.Operation.NOT_ACID ? null : getWriteType().toString();
+  }
   public void setTransactionId(long id) {
     txnId = id;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index 05d2c81..14f2a12 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -765,7 +765,7 @@ public final class PlanUtils {
     return new ReduceSinkDesc(keyCols, numKeys, valueCols, outputKeyCols,
         distinctColIndices, outputValCols,
         tag, partitionCols, numReducers, keyTable,
-        valueTable, writeType);
+        valueTable);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
index d77a223..9ae30ab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
@@ -21,13 +21,11 @@ package org.apache.hadoop.hive.ql.plan;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
-import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
 import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
 import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc.ReduceSinkKeyType;
@@ -123,9 +121,6 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
   // Is reducer auto-parallelism unset (FIXED, UNIFORM, PARALLEL)
   private EnumSet<ReducerTraits> reduceTraits = EnumSet.of(ReducerTraits.UNSET);
 
-  // Write type, since this needs to calculate buckets differently for updates and deletes
-  private AcidUtils.Operation writeType;
-
   // whether this RS is deduplicated
   private transient boolean isDeduplicated = false;
 
@@ -144,8 +139,7 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
       List<List<Integer>> distinctColumnIndices,
       ArrayList<String> outputValueColumnNames, int tag,
       ArrayList<ExprNodeDesc> partitionCols, int numReducers,
-      final TableDesc keySerializeInfo, final TableDesc valueSerializeInfo,
-      AcidUtils.Operation writeType) {
+      final TableDesc keySerializeInfo, final TableDesc valueSerializeInfo) {
     this.keyCols = keyCols;
     this.numDistributionKeys = numDistributionKeys;
     this.valueCols = valueCols;
@@ -159,7 +153,6 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
     this.distinctColumnIndices = distinctColumnIndices;
     this.setNumBuckets(-1);
     this.setBucketCols(null);
-    this.writeType = writeType;
     this.vectorDesc = null;
   }
 
@@ -472,10 +465,6 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
     }
   }
 
-  public AcidUtils.Operation getWriteType() {
-    return writeType;
-  }
-
   public boolean isDeduplicated() {
     return isDeduplicated;
   }
@@ -545,9 +534,6 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
               engineInSupported,
               engineInSupportedCondName),
           new VectorizationCondition(
-              !vectorReduceSinkDesc.getAcidChange(),
-              "Not ACID UPDATE or DELETE"),
-          new VectorizationCondition(
               !vectorReduceSinkDesc.getHasBuckets(),
               "No buckets"),
           new VectorizationCondition(

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java
index 288a440..2eb44b8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java
@@ -64,7 +64,6 @@ public class VectorReduceSinkDesc extends AbstractVectorDesc  {
 
   private boolean isVectorizationReduceSinkNativeEnabled;
   private String engine;
-  private boolean acidChange;
   private boolean hasBuckets;
   private boolean hasTopN;
   private boolean useUniformHash;
@@ -87,12 +86,6 @@ public class VectorReduceSinkDesc extends AbstractVectorDesc  {
   public String getEngine() {
     return engine;
   }
-  public void setAcidChange(boolean acidChange) {
-    this.acidChange = acidChange;
-  }
-  public boolean getAcidChange() {
-    return acidChange;
-  }
   public void setHasBuckets(boolean hasBuckets) {
     this.hasBuckets = hasBuckets;
   }  

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 1aef7ac..05b6fc4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -266,7 +266,7 @@ public class CompactorMR {
 
     if (parsedDeltas.size() == 0 && dir.getOriginalFiles() == null) {
       // Skip compaction if there's no delta files AND there's no original files
-      LOG.error("No delta files or original files found to compact in " + sd.getLocation());
+      LOG.error("No delta files or original files found to compact in " + sd.getLocation() + " for compactionId=" + ci.id);
       return;
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q
index 414d070..bf8cbc8 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q
@@ -7,131 +7,131 @@ set hive.exec.dynamic.partition.mode=nonstrict;
 set hive.optimize.sort.dynamic.partition=false;
 
 -- single level partition, sorted dynamic partition disabled
-drop table acid;
-CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
-insert into table acid partition(ds)  select key,value,ds from srcpart;
+drop table if exists acid_part;
+CREATE TABLE acid_part(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
+insert into table acid_part partition(ds)  select key,value,ds from srcpart;
 -- explicitly set statistics to avoid flakiness
-alter table acid partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
-select count(*) from acid where ds='2008-04-08';
+alter table acid_part partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
+select count(*) from acid_part where ds='2008-04-08';
 
-insert into table acid partition(ds='2008-04-08') values("foo", "bar");
-select count(*) from acid where ds='2008-04-08';
+insert into table acid_part partition(ds='2008-04-08') values("foo", "bar");
+select count(*) from acid_part where ds='2008-04-08';
 
-explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08';
-update acid set value = 'bar' where key = 'foo' and ds='2008-04-08';
-select count(*) from acid where ds='2008-04-08';
+explain update acid_part set value = 'bar' where key = 'foo' and ds='2008-04-08';
+update acid_part set value = 'bar' where key = 'foo' and ds='2008-04-08';
+select count(*) from acid_part where ds='2008-04-08';
 
-explain update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08');
-update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08');
-select count(*) from acid where ds in ('2008-04-08');
+explain update acid_part set value = 'bar' where key = 'foo' and ds in ('2008-04-08');
+update acid_part set value = 'bar' where key = 'foo' and ds in ('2008-04-08');
+select count(*) from acid_part where ds in ('2008-04-08');
 
-delete from acid where key = 'foo' and ds='2008-04-08';
-select count(*) from acid where ds='2008-04-08';
+delete from acid_part where key = 'foo' and ds='2008-04-08';
+select count(*) from acid_part where ds='2008-04-08';
 
 set hive.optimize.sort.dynamic.partition=true;
 
 -- single level partition, sorted dynamic partition enabled
-drop table acid;
-CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
-insert into table acid partition(ds)  select key,value,ds from srcpart;
-alter table acid partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
-select count(*) from acid where ds='2008-04-08';
+drop table if exists acid_part_sdpo;
+CREATE TABLE acid_part_sdpo(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
+insert into table acid_part_sdpo partition(ds)  select key,value,ds from srcpart;
+alter table acid_part_sdpo partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
+select count(*) from acid_part_sdpo where ds='2008-04-08';
 
-insert into table acid partition(ds='2008-04-08') values("foo", "bar");
-select count(*) from acid where ds='2008-04-08';
+insert into table acid_part_sdpo partition(ds='2008-04-08') values("foo", "bar");
+select count(*) from acid_part_sdpo where ds='2008-04-08';
 
-explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08';
-update acid set value = 'bar' where key = 'foo' and ds='2008-04-08';
-select count(*) from acid where ds='2008-04-08';
+explain update acid_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08';
+update acid_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08';
+select count(*) from acid_part_sdpo where ds='2008-04-08';
 
-explain update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08');
-update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08');
-select count(*) from acid where ds in ('2008-04-08');
+explain update acid_part_sdpo set value = 'bar' where key = 'foo' and ds in ('2008-04-08');
+update acid_part_sdpo set value = 'bar' where key = 'foo' and ds in ('2008-04-08');
+select count(*) from acid_part_sdpo where ds in ('2008-04-08');
 
-delete from acid where key = 'foo' and ds='2008-04-08';
-select count(*) from acid where ds='2008-04-08';
+delete from acid_part_sdpo where key = 'foo' and ds='2008-04-08';
+select count(*) from acid_part_sdpo where ds='2008-04-08';
 
 set hive.optimize.sort.dynamic.partition=false;
 
 -- 2 level partition, sorted dynamic partition disabled
-drop table acid;
-CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
-insert into table acid partition(ds,hr)  select * from srcpart;
-alter table acid partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
-select count(*) from acid where ds='2008-04-08' and hr=11;
+drop table if exists acid_2L_part;
+CREATE TABLE acid_2L_part(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
+insert into table acid_2L_part partition(ds,hr)  select * from srcpart;
+alter table acid_2L_part partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
+select count(*) from acid_2L_part where ds='2008-04-08' and hr=11;
 
-insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar");
-select count(*) from acid where ds='2008-04-08' and hr=11;
+insert into table acid_2L_part partition(ds='2008-04-08',hr=11) values("foo", "bar");
+select count(*) from acid_2L_part where ds='2008-04-08' and hr=11;
 
-explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
-update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
-select count(*) from acid where ds='2008-04-08' and hr=11;
+explain update acid_2L_part set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
+update acid_2L_part set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
+select count(*) from acid_2L_part where ds='2008-04-08' and hr=11;
 
-explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
-update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
-select count(*) from acid where ds='2008-04-08' and hr>=11;
+explain update acid_2L_part set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
+update acid_2L_part set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
+select count(*) from acid_2L_part where ds='2008-04-08' and hr>=11;
 
-delete from acid where key = 'foo' and ds='2008-04-08' and hr=11;
-select count(*) from acid where ds='2008-04-08' and hr=11;
+delete from acid_2L_part where key = 'foo' and ds='2008-04-08' and hr=11;
+select count(*) from acid_2L_part where ds='2008-04-08' and hr=11;
 
 -- test with bucketing column not in select list
 explain
-delete from acid where value = 'bar';
-delete from acid where value = 'bar';
-select count(*) from acid;
+delete from acid_2L_part where value = 'bar';
+delete from acid_2L_part where value = 'bar';
+select count(*) from acid_2L_part;
 
 set hive.optimize.sort.dynamic.partition=true;
 
 -- 2 level partition, sorted dynamic partition enabled
-drop table acid;
-CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
-insert into table acid partition(ds,hr)  select * from srcpart;
-alter table acid partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
-select count(*) from acid where ds='2008-04-08' and hr=11;
+drop table if exists acid_2L_part_sdpo;
+CREATE TABLE acid_2L_part_sdpo(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
+insert into table acid_2L_part_sdpo partition(ds,hr)  select * from srcpart;
+alter table acid_2L_part_sdpo partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
+select count(*) from acid_2L_part_sdpo where ds='2008-04-08' and hr=11;
 
-insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar");
-select count(*) from acid where ds='2008-04-08' and hr=11;
+insert into table acid_2L_part_sdpo partition(ds='2008-04-08',hr=11) values("foo", "bar");
+select count(*) from acid_2L_part_sdpo where ds='2008-04-08' and hr=11;
 
-explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
-update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
-select count(*) from acid where ds='2008-04-08' and hr=11;
+explain update acid_2L_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
+update acid_2L_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
+select count(*) from acid_2L_part_sdpo where ds='2008-04-08' and hr=11;
 
-explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
-update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
-select count(*) from acid where ds='2008-04-08' and hr>=11;
+explain update acid_2L_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
+update acid_2L_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
+select count(*) from acid_2L_part_sdpo where ds='2008-04-08' and hr>=11;
 
-delete from acid where key = 'foo' and ds='2008-04-08' and hr=11;
-select count(*) from acid where ds='2008-04-08' and hr=11;
+delete from acid_2L_part_sdpo where key = 'foo' and ds='2008-04-08' and hr=11;
+select count(*) from acid_2L_part_sdpo where ds='2008-04-08' and hr=11;
 
 -- test with bucketing column not in select list
 explain
-delete from acid where value = 'bar';
-delete from acid where value = 'bar';
-select count(*) from acid;
+delete from acid_2L_part_sdpo where value = 'bar';
+delete from acid_2L_part_sdpo where value = 'bar';
+select count(*) from acid_2L_part_sdpo;
 
 
 set hive.optimize.sort.dynamic.partition=true;
 set hive.optimize.constant.propagation=false;
 
 -- 2 level partition, sorted dynamic partition enabled, constant propagation disabled
-drop table acid;
-CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
-insert into table acid partition(ds,hr)  select * from srcpart;
-alter table acid partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
-select count(*) from acid where ds='2008-04-08' and hr=11;
+drop table if exists acid_2L_part_sdpo_no_cp;
+CREATE TABLE acid_2L_part_sdpo_no_cp(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true');
+insert into table acid_2L_part_sdpo_no_cp partition(ds,hr)  select * from srcpart;
+alter table acid_2L_part_sdpo_no_cp partition(ds='2008-04-08') update statistics set('numRows'='1600', 'rawDataSize'='18000');
+select count(*) from acid_2L_part_sdpo_no_cp where ds='2008-04-08' and hr=11;
 
-insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar");
-select count(*) from acid where ds='2008-04-08' and hr=11;
+insert into table acid_2L_part_sdpo_no_cp partition(ds='2008-04-08',hr=11) values("foo", "bar");
+select count(*) from acid_2L_part_sdpo_no_cp where ds='2008-04-08' and hr=11;
 
-explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
-update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
-select count(*) from acid where ds='2008-04-08' and hr=11;
+explain update acid_2L_part_sdpo_no_cp set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
+update acid_2L_part_sdpo_no_cp set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11;
+select count(*) from acid_2L_part_sdpo_no_cp where ds='2008-04-08' and hr=11;
 
-explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
-update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
-select count(*) from acid where ds='2008-04-08' and hr>=11;
+explain update acid_2L_part_sdpo_no_cp set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
+update acid_2L_part_sdpo_no_cp set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11;
+select count(*) from acid_2L_part_sdpo_no_cp where ds='2008-04-08' and hr>=11;
 
-delete from acid where key = 'foo' and ds='2008-04-08' and hr=11;
-select count(*) from acid where ds='2008-04-08' and hr=11;
+delete from acid_2L_part_sdpo_no_cp where key = 'foo' and ds='2008-04-08' and hr=11;
+select count(*) from acid_2L_part_sdpo_no_cp where ds='2008-04-08' and hr=11;
 
 set hive.optimize.sort.dynamic.partition=true;

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
index c7b9b4f..3ae2f20 100644
--- a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
+++ b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
@@ -110,6 +110,7 @@ STAGE PLANS:
                 output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
                 serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
                 name: default.acid_dtt
+            Write Type: INSERT
           Select Operator
             expressions: _col0 (type: int), _col1 (type: varchar(128))
             outputColumnNames: a, b

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out
index 0b6e992..76d0b7b 100644
--- a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out
+++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out
@@ -47,6 +47,7 @@ STAGE PLANS:
           Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false
+            Dp Sort State: PARTITION_BUCKET_SORTED
             Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
             table:
                 input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
index ef5a4b2..829f28f 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
@@ -174,6 +174,7 @@ STAGE PLANS:
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
                       serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
                       name: default.acidtbl
+                  Write Type: DELETE
         Reducer 4 
             Execution mode: llap
             Reduce Operator Tree:
@@ -189,6 +190,7 @@ STAGE PLANS:
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
                       serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
                       name: default.acidtbl
+                  Write Type: UPDATE
         Reducer 5 
             Execution mode: llap
             Reduce Operator Tree:
@@ -228,6 +230,7 @@ STAGE PLANS:
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
                       serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
                       name: default.acidtbl
+                  Write Type: INSERT
         Reducer 8 
             Execution mode: llap
             Reduce Operator Tree:
@@ -398,6 +401,7 @@ STAGE PLANS:
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
                       serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
                       name: default.acidtbl
+                  Write Type: INSERT
         Reducer 5 
             Execution mode: llap
             Reduce Operator Tree:
@@ -639,6 +643,7 @@ STAGE PLANS:
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
                       serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
                       name: default.acidtbl
+                  Write Type: DELETE
         Reducer 5 
             Execution mode: llap
             Reduce Operator Tree:
@@ -654,6 +659,7 @@ STAGE PLANS:
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
                       serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
                       name: default.acidtbl
+                  Write Type: UPDATE
         Reducer 6 
             Execution mode: llap
             Reduce Operator Tree:
@@ -693,6 +699,7 @@ STAGE PLANS:
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
                       serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
                       name: default.acidtbl
+                  Write Type: INSERT
         Reducer 8 
             Execution mode: llap
             Reduce Operator Tree:

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
index eaa394d..0e16ff1 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
@@ -198,6 +198,7 @@ STAGE PLANS:
                 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
@@ -290,6 +291,7 @@ STAGE PLANS:
                 Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
@@ -363,6 +365,7 @@ STAGE PLANS:
                 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
@@ -435,6 +438,7 @@ STAGE PLANS:
                 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
@@ -575,6 +579,7 @@ STAGE PLANS:
                 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
@@ -667,6 +672,7 @@ STAGE PLANS:
                 Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
@@ -740,6 +746,7 @@ STAGE PLANS:
                 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
@@ -812,6 +819,7 @@ STAGE PLANS:
                 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
@@ -1426,6 +1434,7 @@ STAGE PLANS:
                 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -1520,6 +1529,7 @@ STAGE PLANS:
                 Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -1676,6 +1686,7 @@ STAGE PLANS:
                   Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
                   File Output Operator
                     compressed: false
+                    Dp Sort State: PARTITION_SORTED
                     Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
                     table:
                         input format: org.apache.hadoop.mapred.TextInputFormat
@@ -2133,6 +2144,7 @@ STAGE PLANS:
                 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
index 95a4e0f..1ef0740 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
@@ -155,6 +155,7 @@ STAGE PLANS:
                 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -247,6 +248,7 @@ STAGE PLANS:
                 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -320,6 +322,7 @@ STAGE PLANS:
                 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -392,6 +395,7 @@ STAGE PLANS:
                 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -532,6 +536,7 @@ STAGE PLANS:
                 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -624,6 +629,7 @@ STAGE PLANS:
                 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -697,6 +703,7 @@ STAGE PLANS:
                 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -769,6 +776,7 @@ STAGE PLANS:
                 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -1383,6 +1391,7 @@ STAGE PLANS:
                 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -1477,6 +1486,7 @@ STAGE PLANS:
                 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -1633,6 +1643,7 @@ STAGE PLANS:
                   Statistics: Num rows: 2221 Data size: 53305 Basic stats: COMPLETE Column stats: NONE
                   File Output Operator
                     compressed: false
+                    Dp Sort State: PARTITION_SORTED
                     Statistics: Num rows: 2221 Data size: 53305 Basic stats: COMPLETE Column stats: NONE
                     table:
                         input format: org.apache.hadoop.mapred.TextInputFormat
@@ -2090,6 +2101,7 @@ STAGE PLANS:
                 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_BUCKET_SORTED
                   Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -2464,6 +2476,7 @@ STAGE PLANS:
                 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -2538,6 +2551,7 @@ STAGE PLANS:
                 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -2612,6 +2626,7 @@ STAGE PLANS:
                 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -2686,6 +2701,7 @@ STAGE PLANS:
                 Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -2760,6 +2776,7 @@ STAGE PLANS:
                 Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -2834,6 +2851,7 @@ STAGE PLANS:
                 Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/917221e8/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
index 41a7709..667d980 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
@@ -120,6 +120,7 @@ STAGE PLANS:
                   Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                   File Output Operator
                     compressed: false
+                    Dp Sort State: PARTITION_SORTED
                     Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                     table:
                         input format: org.apache.hadoop.mapred.TextInputFormat
@@ -357,6 +358,7 @@ STAGE PLANS:
                 Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat


Mime
View raw message