hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hive git commit: HIVE-14878 : integrate MM tables into ACID: add separate ACID type
Date Tue, 18 Oct 2016 22:13:35 GMT
Repository: hive
Updated Branches:
  refs/heads/hive-14535 9ecffcb1b -> c7fb2dbaa


HIVE-14878 : integrate MM tables into ACID: add separate ACID type


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c7fb2dba
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c7fb2dba
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c7fb2dba

Branch: refs/heads/hive-14535
Commit: c7fb2dbaa35de53f352a0cc32cfe8312a40fa637
Parents: 9ecffcb
Author: Wei Zheng <weiz@apache.org>
Authored: Tue Oct 18 15:13:20 2016 -0700
Committer: Wei Zheng <weiz@apache.org>
Committed: Tue Oct 18 15:13:20 2016 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  10 +-
 .../TransactionalValidationListener.java        |  39 ++++---
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |  15 ++-
 .../apache/hadoop/hive/ql/exec/MoveTask.java    |   3 +-
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |  44 ++++++-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  34 +++---
 .../queries/clientpositive/mm_insertonly_acid.q |  16 +++
 .../clientpositive/mm_insertonly_acid.q.out     | 115 +++++++++++++++++++
 8 files changed, 236 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 23314ed..ccc29f8 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1787,10 +1787,12 @@ public class HiveConf extends Configuration {
 
     HIVE_TXN_OPERATIONAL_PROPERTIES("hive.txn.operational.properties", 0,
         "Sets the operational properties that control the appropriate behavior for various\n"
-        + "versions of the Hive ACID subsystem. Setting it to zero will turn on the legacy
mode\n"
-        + "for ACID, while setting it to one will enable a split-update feature found in
the newer\n"
-        + "version of Hive ACID subsystem. Mostly it is intended to be used as an internal
property\n"
-        + "for future versions of ACID. (See HIVE-14035 for details.)"),
+        + "versions of the Hive ACID subsystem. Mostly it is intended to be used as an internal
property\n"
+        + "for future versions of ACID. (See HIVE-14035 for details.)\n"
+        + "0: Turn on the legacy mode for ACID\n"
+        + "1: Enable split-update feature found in the newer version of Hive ACID subsystem\n"
+        + "2: Hash-based merge, which combines delta files using GRACE hash join based approach
(not implemented)\n"
+        + "3: Make the table 'quarter-acid' as it only supports insert. But it doesn't require
ORC or bucketing."),
 
     HIVE_MAX_OPEN_TXNS("hive.max.open.txns", 100000, "Maximum number of open transactions.
If \n" +
         "current open transactions reach this limit, future open transaction requests will
be \n" +

http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 0f08f43..f942479 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -40,6 +40,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
   // These constants are also imported by org.apache.hadoop.hive.ql.io.AcidUtils.
   public static final String DEFAULT_TRANSACTIONAL_PROPERTY = "default";
   public static final String LEGACY_TRANSACTIONAL_PROPERTY = "legacy";
+  public static final String INSERTONLY_TRANSACTIONAL_PROPERTY = "insert_only";
 
   TransactionalValidationListener(Configuration conf) {
     super(conf);
@@ -105,8 +106,11 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
     }
     if ("true".equalsIgnoreCase(transactionalValue)) {
       if (!conformToAcid(newTable)) {
-        throw new MetaException("The table must be bucketed and stored using an ACID compliant"
+
-            " format (such as ORC)");
+        // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing
+        if (transactionalPropertiesValue == null || !"insert_only".equalsIgnoreCase(transactionalPropertiesValue))
{
+          throw new MetaException("The table must be bucketed and stored using an ACID compliant"
+
+              " format (such as ORC)");
+        }
       }
 
       if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
@@ -172,32 +176,40 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
     if (parameters == null || parameters.isEmpty()) {
       return;
     }
-    String transactionalValue = null;
-    boolean transactionalPropFound = false;
+    String transactional = null;
+    String transactionalProperties = null;
     Set<String> keys = new HashSet<>(parameters.keySet());
     for(String key : keys) {
-      if(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) {
-        transactionalPropFound = true;
-        transactionalValue = parameters.get(key);
+      // Get the "transactional" tblproperties value
+      if (hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) {
+        transactional = parameters.get(key);
         parameters.remove(key);
       }
+
+      // Get the "transactional_properties" tblproperties value
+      if (hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES.equalsIgnoreCase(key)) {
+        transactionalProperties = parameters.get(key);
+      }
     }
 
-    if (!transactionalPropFound) {
+    if (transactional == null) {
       return;
     }
 
-    if ("false".equalsIgnoreCase(transactionalValue)) {
+    if ("false".equalsIgnoreCase(transactional)) {
       // just drop transactional=false.  For backward compatibility in case someone has scripts
       // with transactional=false
       LOG.info("'transactional'='false' is no longer a valid property and will be ignored");
       return;
     }
 
-    if ("true".equalsIgnoreCase(transactionalValue)) {
+    if ("true".equalsIgnoreCase(transactional)) {
       if (!conformToAcid(newTable)) {
-        throw new MetaException("The table must be bucketed and stored using an ACID compliant"
+
-            " format (such as ORC)");
+        // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing
+        if (transactionalProperties == null || !"insert_only".equalsIgnoreCase(transactionalProperties))
{
+          throw new MetaException("The table must be bucketed and stored using an ACID compliant"
+
+              " format (such as ORC)");
+        }
       }
 
       if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
@@ -211,7 +223,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
       return;
     }
 
-    // transactional prop is found, but the value is not in expected range
+    // transactional is found, but the value is not in expected range
     throw new MetaException("'transactional' property of TBLPROPERTIES may only have value
'true'");
   }
 
@@ -277,6 +289,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
     switch (transactionalProperties) {
       case DEFAULT_TRANSACTIONAL_PROPERTY:
       case LEGACY_TRANSACTIONAL_PROPERTY:
+      case INSERTONLY_TRANSACTIONAL_PROPERTY:
         isValid = true;
         break;
       default:

http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index ef6473a..c54187f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -678,7 +678,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc>
implements
       Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), jc);
       // only create bucket files only if no dynamic partitions,
       // buckets of dynamic partitions will be created for each newly created partition
-      if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
+      if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID ||
+          conf.getWriteType() == AcidUtils.Operation.INSERT_ONLY) {
         fsp.outWriters[filesIdx] = HiveFileFormatUtils.getHiveRecordWriter(jc, conf.getTableInfo(),
             outputClass, conf, fsp.outPaths[filesIdx], reporter);
         // If the record writer provides stats, get it from there instead of the serde
@@ -821,7 +822,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc>
implements
       // for a given operator branch prediction should work quite nicely on it.
       // RecordUpdateer expects to get the actual row, not a serialized version of it.  Thus
we
       // pass the row rather than recordValue.
-      if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
+      if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID ||
+          conf.getWriteType() == AcidUtils.Operation.INSERT_ONLY) {
         rowOutWriters[writerOffset].write(recordValue);
       } else if (conf.getWriteType() == AcidUtils.Operation.INSERT) {
         fpaths.updaters[writerOffset].insert(conf.getTransactionId(), row);
@@ -865,7 +867,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc>
implements
   protected boolean areAllTrue(boolean[] statsFromRW) {
     // If we are doing an acid operation they will always all be true as RecordUpdaters always
     // collect stats
-    if (conf.getWriteType() != AcidUtils.Operation.NOT_ACID) {
+    if (conf.getWriteType() != AcidUtils.Operation.NOT_ACID &&
+        conf.getWriteType() != AcidUtils.Operation.INSERT_ONLY) {
       return true;
     }
     for(boolean b : statsFromRW) {
@@ -1008,7 +1011,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc>
implements
           // stats from the record writer and store in the previous fsp that is cached
           if (conf.isGatherStats() && isCollectRWStats) {
             SerDeStats stats = null;
-            if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
+            if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID ||
+                conf.getWriteType() == AcidUtils.Operation.INSERT_ONLY) {
               RecordWriter outWriter = prevFsp.outWriters[0];
               if (outWriter != null) {
                 stats = ((StatsProvidingRecordWriter) outWriter).getStats();
@@ -1112,7 +1116,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc>
implements
         // record writer already gathers the statistics, it can simply return the
         // accumulated statistics which will be aggregated in case of spray writers
         if (conf.isGatherStats() && isCollectRWStats) {
-          if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
+          if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID ||
+              conf.getWriteType() == AcidUtils.Operation.INSERT_ONLY) {
             for (int idx = 0; idx < fsp.outWriters.length; idx++) {
               RecordWriter outWriter = fsp.outWriters[idx];
               if (outWriter != null) {

http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index f2b8ca3..74a650d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -389,7 +389,8 @@ public class MoveTask extends Task<MoveWork> implements Serializable
{
     db.loadSinglePartition(tbd.getSourcePath(), tbd.getTable().getTableName(),
         tbd.getPartitionSpec(), tbd.getReplace(),
         tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(),
-        work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID,
+        (work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
+         work.getLoadTableWork().getWriteType() != AcidUtils.Operation.INSERT_ONLY),
         hasFollowingStatsTask(), tbd.getMmWriteId());
     Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 7351bbe..ecbc216 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -275,8 +275,9 @@ public class AcidUtils {
     return result;
   }
 
+  // INSERT_ONLY is a special operation which we only support INSERT operations, no UPDATE/DELETE
   public enum Operation {
-    NOT_ACID, INSERT, UPDATE, DELETE;
+    NOT_ACID, INSERT, UPDATE, DELETE, INSERT_ONLY
   }
 
   /**
@@ -344,8 +345,11 @@ public class AcidUtils {
     public static final String SPLIT_UPDATE_STRING = "split_update";
     public static final int HASH_BASED_MERGE_BIT = 0x02;
     public static final String HASH_BASED_MERGE_STRING = "hash_merge";
+    public static final int INSERT_ONLY_BIT = 0x03;
+    public static final String INSERT_ONLY_STRING = "insert_only";
     public static final String DEFAULT_VALUE_STRING = TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY;
     public static final String LEGACY_VALUE_STRING = TransactionalValidationListener.LEGACY_TRANSACTIONAL_PROPERTY;
+    public static final String INSERTONLY_VALUE_STRING = TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY;
 
     private AcidOperationalProperties() {
     }
@@ -374,6 +378,17 @@ public class AcidUtils {
     }
 
     /**
+     * Returns an acidOperationalProperties object for tables that uses ACID framework but
only
+     * supports INSERT operation and does not require ORC or bucketing
+     * @return the acidOperationalProperties object
+     */
+    public static AcidOperationalProperties getInsertOnly() {
+      AcidOperationalProperties obj = new AcidOperationalProperties();
+      obj.setInsertOnly(true);
+      return obj;
+    }
+
+    /**
      * Returns an acidOperationalProperties object that is represented by an encoded string.
      * @param propertiesStr an encoded string representing the acidOperationalProperties.
      * @return the acidOperationalProperties object.
@@ -388,6 +403,9 @@ public class AcidUtils {
       if (propertiesStr.equalsIgnoreCase(LEGACY_VALUE_STRING)) {
         return AcidOperationalProperties.getLegacy();
       }
+      if (propertiesStr.equalsIgnoreCase(INSERTONLY_VALUE_STRING)) {
+        return AcidOperationalProperties.getInsertOnly();
+      }
       AcidOperationalProperties obj = new AcidOperationalProperties();
       String[] options = propertiesStr.split("\\|");
       for (String option : options) {
@@ -448,6 +466,12 @@ public class AcidUtils {
       return this;
     }
 
+    public AcidOperationalProperties setInsertOnly(boolean isInsertOnly) {
+      description = (isInsertOnly
+              ? (description | INSERT_ONLY_BIT) : (description & ~INSERT_ONLY_BIT));
+      return this;
+    }
+
     public boolean isSplitUpdate() {
       return (description & SPLIT_UPDATE_BIT) > 0;
     }
@@ -456,6 +480,10 @@ public class AcidUtils {
       return (description & HASH_BASED_MERGE_BIT) > 0;
     }
 
+    public boolean isInsertOnly() {
+      return (description & INSERT_ONLY_BIT) > 0;
+    }
+
     public int toInt() {
       return description;
     }
@@ -469,6 +497,9 @@ public class AcidUtils {
       if (isHashBasedMerge()) {
         str.append("|" + HASH_BASED_MERGE_STRING);
       }
+      if (isInsertOnly()) {
+        str.append("|" + INSERT_ONLY_STRING);
+      }
       return str.toString();
     }
   }
@@ -1078,6 +1109,17 @@ public class AcidUtils {
   }
 
   /**
+   * Checks if a table is an ACID table that only supports INSERT, but not UPDATE/DELETE
+   * @param table table
+   * @return true if table is an INSERT_ONLY table, false otherwise
+   */
+  public static boolean isInsertOnlyTable(Table table) {
+    String transactionalProp = table.getProperty(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+    return transactionalProp != null &&
+        AcidUtils.AcidOperationalProperties.INSERT_ONLY_STRING.equals(transactionalProp);
+  }
+
+  /**
    * Sets the acidOperationalProperties in the configuration object argument.
    * @param conf Mutable configuration object
    * @param properties An acidOperationalProperties object to initialize from.

http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 62faf89..f74c0a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6455,7 +6455,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         nullOrder.append(sortOrder == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_ASC ? 'a' :
'z');
       }
       input = genReduceSinkPlan(input, partnCols, sortCols, order.toString(), nullOrder.toString(),
-              maxReducers, (AcidUtils.isAcidTable(dest_tab) ? getAcidType() : AcidUtils.Operation.NOT_ACID));
+              maxReducers, (AcidUtils.isAcidTable(dest_tab) ?
+              getAcidType(dest_tab, table_desc.getOutputFileFormatClass()) : AcidUtils.Operation.NOT_ACID));
       reduceSinkOperatorsAddedByEnforceBucketingSorting.add((ReduceSinkOperator)input.getParentOperators().get(0));
       ctx.setMultiFileSpray(multiFileSpray);
       ctx.setNumFiles(numFiles);
@@ -6588,8 +6589,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
       if (!isNonNativeTable) {
         AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID;
         if (destTableIsAcid) {
-          acidOp = getAcidType(table_desc.getOutputFileFormatClass());
-          checkAcidConstraints(qb, table_desc, dest_tab);
+          acidOp = getAcidType(dest_tab, table_desc.getOutputFileFormatClass());
+          checkAcidConstraints(qb, table_desc, dest_tab, acidOp);
         }
         try {
           mmWriteId = getMmWriteId(dest_tab, isMmTable);
@@ -6648,8 +6649,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
           dest_part.isStoredAsSubDirectories(), conf);
       AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID;
       if (destTableIsAcid) {
-        acidOp = getAcidType(table_desc.getOutputFileFormatClass());
-        checkAcidConstraints(qb, table_desc, dest_tab);
+        acidOp = getAcidType(dest_tab, table_desc.getOutputFileFormatClass());
+        checkAcidConstraints(qb, table_desc, dest_tab, acidOp);
       }
       try {
         mmWriteId = getMmWriteId(dest_tab, isMmTable);
@@ -6945,7 +6946,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     fileSinkDesc.setHiveServerQuery(SessionState.get().isHiveServerQuery());
     // If this is an insert, update, or delete on an ACID table then mark that so the
     // FileSinkOperator knows how to properly write to it.
-    if (destTableIsAcid) {
+    if (destTableIsAcid && !AcidUtils.isInsertOnlyTable(dest_part.getTable())) {
       AcidUtils.Operation wt = updating() ? AcidUtils.Operation.UPDATE :
           (deleting() ? AcidUtils.Operation.DELETE : AcidUtils.Operation.INSERT);
       fileSinkDesc.setWriteType(wt);
@@ -7141,7 +7142,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
   // This method assumes you have already decided that this is an Acid write.  Don't call
it if
   // that isn't true.
   private void checkAcidConstraints(QB qb, TableDesc tableDesc,
-                                    Table table) throws SemanticException {
+                                    Table table, AcidUtils.Operation acidOp) throws SemanticException
{
     String tableName = tableDesc.getTableName();
     if (!qb.getParseInfo().isInsertIntoTable(tableName)) {
       LOG.debug("Couldn't find table " + tableName + " in insertIntoTable");
@@ -7158,15 +7159,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     */
     conf.set(AcidUtils.CONF_ACID_KEY, "true");
 
-    if (table.getNumBuckets() < 1) {
-      throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, table.getTableName());
-    }
-    if (table.getSortCols() != null && table.getSortCols().size() > 0) {
-      throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName());
+    if (!Operation.NOT_ACID.equals(acidOp) && !Operation.INSERT_ONLY.equals(acidOp))
{
+      if (table.getNumBuckets() < 1) {
+        throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, table.getTableName());
+      }
+      if (table.getSortCols() != null && table.getSortCols().size() > 0) {
+        throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName());
+      }
     }
-
-
-
   }
 
   /**
@@ -13118,9 +13118,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
             AcidUtils.Operation.INSERT);
   }
 
-  private AcidUtils.Operation getAcidType(Class<? extends OutputFormat> of) {
+  private AcidUtils.Operation getAcidType(Table table, Class<? extends OutputFormat>
of) {
     if (SessionState.get() == null || !SessionState.get().getTxnMgr().supportsAcid()) {
       return AcidUtils.Operation.NOT_ACID;
+    } else if (AcidUtils.isInsertOnlyTable(table)) {
+      return AcidUtils.Operation.INSERT_ONLY;
     } else if (isAcidOutputFormat(of)) {
       return getAcidType();
     } else {

http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/test/queries/clientpositive/mm_insertonly_acid.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_insertonly_acid.q b/ql/src/test/queries/clientpositive/mm_insertonly_acid.q
new file mode 100644
index 0000000..7da99c5
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/mm_insertonly_acid.q
@@ -0,0 +1,16 @@
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.fetch.task.conversion=none;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+
+drop table qtr_acid;
+create table qtr_acid (key int) partitioned by (p int) tblproperties ("transactional"="true",
"transactional_properties"="insert_only");
+insert into table qtr_acid partition(p='123') select distinct key from src where key >
0 order by key asc limit 10;
+insert into table qtr_acid partition(p='456') select distinct key from src where key >
0 order by key desc limit 10;
+explain
+select * from qtr_acid order by key;
+select * from qtr_acid order by key;
+drop table qtr_acid;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/test/results/clientpositive/mm_insertonly_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/mm_insertonly_acid.q.out b/ql/src/test/results/clientpositive/mm_insertonly_acid.q.out
new file mode 100644
index 0000000..6f7d198
--- /dev/null
+++ b/ql/src/test/results/clientpositive/mm_insertonly_acid.q.out
@@ -0,0 +1,115 @@
+PREHOOK: query: drop table qtr_acid
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table qtr_acid
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table qtr_acid (key int) partitioned by (p int) tblproperties ("transactional"="true",
"transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@qtr_acid
+POSTHOOK: query: create table qtr_acid (key int) partitioned by (p int) tblproperties ("transactional"="true",
"transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@qtr_acid
+PREHOOK: query: insert into table qtr_acid partition(p='123') select distinct key from src
where key > 0 order by key asc limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@qtr_acid@p=123
+POSTHOOK: query: insert into table qtr_acid partition(p='123') select distinct key from src
where key > 0 order by key asc limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@qtr_acid@p=123
+POSTHOOK: Lineage: qtr_acid PARTITION(p=123).key EXPRESSION [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
+PREHOOK: query: insert into table qtr_acid partition(p='456') select distinct key from src
where key > 0 order by key desc limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@qtr_acid@p=456
+POSTHOOK: query: insert into table qtr_acid partition(p='456') select distinct key from src
where key > 0 order by key desc limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@qtr_acid@p=456
+POSTHOOK: Lineage: qtr_acid PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
+PREHOOK: query: explain
+select * from qtr_acid order by key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select * from qtr_acid order by key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: qtr_acid
+            Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE
+            Select Operator
+              expressions: key (type: int), p (type: int)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats:
NONE
+              Reduce Output Operator
+                key expressions: _col0 (type: int)
+                sort order: +
+                Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats:
NONE
+                value expressions: _col1 (type: int)
+      Reduce Operator Tree:
+        Select Operator
+          expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int)
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select * from qtr_acid order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@qtr_acid
+PREHOOK: Input: default@qtr_acid@p=123
+PREHOOK: Input: default@qtr_acid@p=456
+#### A masked pattern was here ####
+POSTHOOK: query: select * from qtr_acid order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@qtr_acid
+POSTHOOK: Input: default@qtr_acid@p=123
+POSTHOOK: Input: default@qtr_acid@p=456
+#### A masked pattern was here ####
+9	456
+10	123
+11	123
+85	456
+86	456
+87	456
+90	456
+92	456
+95	456
+96	456
+97	456
+98	456
+100	123
+103	123
+104	123
+105	123
+111	123
+113	123
+114	123
+116	123
+PREHOOK: query: drop table qtr_acid
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@qtr_acid
+PREHOOK: Output: default@qtr_acid
+POSTHOOK: query: drop table qtr_acid
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@qtr_acid
+POSTHOOK: Output: default@qtr_acid


Mime
View raw message