hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1626449 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/
Date Sat, 20 Sep 2014 14:52:10 GMT
Author: gates
Date: Sat Sep 20 14:52:09 2014
New Revision: 1626449

URL: http://svn.apache.org/r1626449
Log:
HIVE-8149 hive.optimize.reducededuplication should be set to false for IUD ops (Alan Gates,
reviewed by Eugene Koifman)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q
    hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
    hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q
    hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q
    hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q
    hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q
    hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q
    hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q
    hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
    hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q
    hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Sep 20
14:52:09 2014
@@ -6240,6 +6240,9 @@ public class SemanticAnalyzer extends Ba
   // Check constraints on acid tables.  This includes
   // * no insert overwrites
   // * no use of vectorization
+  // * turns off reduce deduplication optimization, as that sometimes breaks acid
+  // This method assumes you have already decided that this is an Acid write.  Don't call
it if
+  // that isn't true.
   private void checkAcidConstraints(QB qb, TableDesc tableDesc) throws SemanticException
{
     String tableName = tableDesc.getTableName();
     if (!qb.getParseInfo().isInsertIntoTable(tableName)) {
@@ -6250,6 +6253,9 @@ public class SemanticAnalyzer extends Ba
       LOG.info("Turning off vectorization for acid write operation");
       conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, false);
     }
+    LOG.info("Modifying config values for ACID write");
+    conf.setBoolVar(ConfVars.HIVEOPTREDUCEDEDUPLICATION, false);
+    conf.setBoolVar(ConfVars.HIVE_HADOOP_SUPPORTS_SUBDIRECTORIES, true);
   }
 
   /**

Modified: hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q Sat Sep 20 14:52:09
2014
@@ -4,7 +4,6 @@ set hive.input.format=org.apache.hadoop.
 set hive.enforce.bucketing=true;
 set hive.exec.dynamic.partition.mode=nonstrict;
 set hive.vectorized.execution.enabled=true;
-set hive.mapred.supports.subdirectories=true;
 
 CREATE TABLE acid_vectorized(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC;
 insert into table acid_vectorized select cint, cstring1 from alltypesorc where cint is not
null order by cint limit 10;

Modified: hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q Sat Sep 20
14:52:09 2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.exec.reducers.max = 1;
 
 create table acid_danp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc;
 

Modified: hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q Sat Sep 20 14:52:09
2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
 
 create table acid_dap(a int, b varchar(128)) partitioned by (ds string) clustered by (a)
into 2 buckets stored as orc;
 

Modified: hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q Sat Sep 20 14:52:09
2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
 
 create table acid_dwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a)
into 2 buckets stored as orc;
 

Modified: hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q Sat Sep 20 14:52:09
2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
 
 create table acid_dwhp(a int, b varchar(128)) partitioned by (ds string) clustered by (a)
into 2 buckets stored as orc;
 

Modified: hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q Sat Sep 20 14:52:09
2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
 
 create table acid_iud(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc;
 

Modified: hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q Sat
Sep 20 14:52:09 2014
@@ -3,7 +3,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
 
 create table ivdp(i int,
                  de decimal(5,2),

Modified: hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q Sat Sep 20 14:52:09
2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.exec.dynamic.partition.mode=nonstrict;
 
 create table acid_ivp(ti tinyint,
                  si smallint,

Modified: hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q Sat Sep
20 14:52:09 2014
@@ -3,7 +3,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
 
 create table acid_uami(i int,
                  de decimal(5,2),

Modified: hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q Sat Sep 20 14:52:09
2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
 
 create table acid_uap(a int, b varchar(128)) partitioned by (ds string) clustered by (a)
into 2 buckets stored as orc;
 

Modified: hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q Sat Sep 20 14:52:09
2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
 
 create table acid_uwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a)
into 2 buckets stored as orc;
 



Mime
View raw message