hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ekoif...@apache.org
Subject hive git commit: HIVE-16542 make merge that targets acid 2.0 table fail-fast (Eugene Koifman, reviwed by Wei Zheng)
Date Fri, 28 Apr 2017 16:15:20 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2 2ba121757 -> ec5b137ab


HIVE-16542 make merge that targets acid 2.0 table fail-fast (Eugene Koifman, reviwed by Wei
Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ec5b137a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ec5b137a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ec5b137a

Branch: refs/heads/branch-2
Commit: ec5b137abe880d2211495e8d5b354d5b802ad449
Parents: 2ba1217
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Fri Apr 28 09:15:14 2017 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Fri Apr 28 09:15:14 2017 -0700

----------------------------------------------------------------------
 .../ql/parse/UpdateDeleteSemanticAnalyzer.java  | 14 +++++++
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |  1 -
 .../apache/hadoop/hive/ql/TestTxnCommands2.java |  3 +-
 .../ql/TestTxnCommands2WithSplitUpdate.java     | 41 ++++++++++++++++++++
 4 files changed, 56 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ec5b137a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
index 64f1bdd..0541a40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
@@ -32,15 +32,18 @@ import org.antlr.runtime.TokenRewriteStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
 import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.QueryState;
 import org.apache.hadoop.hive.ql.hooks.Entity;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.HiveUtils;
@@ -662,9 +665,11 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
      */
     String extraPredicate = null;
     int numWhenMatchedUpdateClauses = 0, numWhenMatchedDeleteClauses = 0;
+    int numInsertClauses = 0;
     for(ASTNode whenClause : whenClauses) {
       switch (getWhenClauseOperation(whenClause).getType()) {
         case HiveParser.TOK_INSERT:
+          numInsertClauses++;
           handleInsert(whenClause, rewrittenQueryStr, target, onClause, targetTable, targetName,
onClauseAsText);
           break;
         case HiveParser.TOK_UPDATE:
@@ -691,6 +696,15 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
       if(numWhenMatchedUpdateClauses > 1) {
         throw new SemanticException(ErrorMsg.MERGE_TOO_MANY_UPDATE, ctx.getCmd());
       }
+      assert numInsertClauses < 2;
+      if(numInsertClauses == 1 && numWhenMatchedUpdateClauses == 1) {
+        if(AcidUtils.getAcidOperationalProperties(targetTable).isSplitUpdate()) {
+          throw new IllegalStateException("Tables with " +
+            hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "=" +
+            TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY + " currently
do not " +
+            "support MERGE with both Insert and Update clauses.");
+        }
+      }
     }
     if(numWhenMatchedDeleteClauses + numWhenMatchedUpdateClauses == 2 && extraPredicate
== null) {
       throw new SemanticException(ErrorMsg.MERGE_PREDIACTE_REQUIRED, ctx.getCmd());

http://git-wip-us.apache.org/repos/asf/hive/blob/ec5b137a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 6dae7ba..c8bc119 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -103,7 +103,6 @@ public class TestTxnCommands {
     hiveConf = new HiveConf(this.getClass());
     hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
     hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
-    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
     hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
     hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
     hiveConf

http://git-wip-us.apache.org/repos/asf/hive/blob/ec5b137a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 6718ae9..e2db5b7 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -113,7 +113,6 @@ public class TestTxnCommands2 {
     hiveConf = new HiveConf(this.getClass());
     hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
     hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
-    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
     hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
     hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
     hiveConf.setVar(HiveConf.ConfVars.HIVEINPUTFORMAT, HiveInputFormat.class.getName());
@@ -1676,7 +1675,7 @@ public class TestTxnCommands2 {
       return 0;
     }
   }
-  private String makeValuesClause(int[][] rows) {
+  String makeValuesClause(int[][] rows) {
     assert rows.length > 0;
     StringBuilder sb = new StringBuilder("values");
     for(int[] row : rows) {

http://git-wip-us.apache.org/repos/asf/hive/blob/ec5b137a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
index 67e917c..e516f18 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
@@ -540,4 +540,45 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2
{
     resultCount = 2;
     Assert.assertEquals(resultCount, Integer.parseInt(rs.get(0)));
   }
+  @Ignore("HIVE-14947")
+  @Test
+  @Override
+  public void testDynamicPartitionsMerge() throws Exception {}
+  @Ignore("HIVE-14947")
+  @Test
+  @Override
+  public void testDynamicPartitionsMerge2() throws Exception {}
+  @Ignore("HIVE-14947")
+  @Test
+  @Override
+  public void testMerge() throws Exception {}
+
+  /**
+   * todo: remove this test once HIVE-14947 is done (parent class has a better version)
+   */
+  @Test
+  @Override
+  public void testMerge2() throws Exception {
+    int[][] baseValsOdd = {{5,5},{11,11}};
+    int[][] baseValsEven = {{2,2},{4,44}};
+    runStatementOnDriver("insert into " + Table.NONACIDPART2 + " PARTITION(p2='odd') " +
makeValuesClause(baseValsOdd));
+    runStatementOnDriver("insert into " + Table.NONACIDPART2 + " PARTITION(p2='even') " +
makeValuesClause(baseValsEven));
+    int[][] vals = {{2,1},{4,3},{5,6},{7,8}};
+    runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals));
+    List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order
by a,b");
+    Assert.assertEquals(stringifyValues(vals), r);
+    String query = "merge into " + Table.ACIDTBL +
+      " using " + Table.NONACIDPART2 + " source ON " + Table.ACIDTBL + ".a = source.a2 "
+
+      "WHEN MATCHED THEN UPDATE set b = source.b2 ";
+    r = runStatementOnDriver(query);
+
+    r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+    int[][] rExpected = {{2,2},{4,44},{5,5},{7,8}};
+    Assert.assertEquals(stringifyValues(rExpected), r);
+
+  }
+  @Ignore("HIVE-14947")
+  @Test
+  @Override
+  public void testMergeWithPredicate() throws Exception {}
 }


Mime
View raw message