hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject svn commit: r1590853 - in /hive/trunk: itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbuc...
Date Tue, 29 Apr 2014 01:10:35 GMT
Author: prasanthj
Date: Tue Apr 29 01:10:34 2014
New Revision: 1590853

URL: http://svn.apache.org/r1590853
Log:
HIVE-6968: list bucketing feature does not update the location map for unpartitioned tables

Added:
    hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
Modified:
    hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java

Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
(original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
Tue Apr 29 01:10:34 2014
@@ -103,7 +103,7 @@ public class TestHiveHistory extends Tes
         db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
         db.createTable(src, cols, null, TextInputFormat.class,
             IgnoreKeyTextOutputFormat.class);
-        db.loadTable(hadoopDataFile[i], src, false, false, false);
+        db.loadTable(hadoopDataFile[i], src, false, false, false, false);
         i++;
       }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Tue Apr 29 01:10:34
2014
@@ -273,7 +273,8 @@ public class MoveTask extends Task<MoveW
         if (tbd.getPartitionSpec().size() == 0) {
           dc = new DataContainer(table.getTTable());
           db.loadTable(tbd.getSourcePath(), tbd.getTable()
-              .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime(), work.isSrcLocal());
+              .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime(), work.isSrcLocal(),
+              isSkewedStoredAsDirs(tbd));
           if (work.getOutputs() != null) {
             work.getOutputs().add(new WriteEntity(table,
                 (tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE :

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Apr 29 01:10:34
2014
@@ -1468,9 +1468,11 @@ private void constructOneLBLocationMap(F
    * @param holdDDLTime
    * @param isSrcLocal
    *          If the source directory is LOCAL
+   * @param isSkewedStoreAsSubdir
+   *          if list bucketing enabled
    */
   public void loadTable(Path loadPath, String tableName, boolean replace,
-      boolean holdDDLTime, boolean isSrcLocal) throws HiveException {
+      boolean holdDDLTime, boolean isSrcLocal, boolean isSkewedStoreAsSubdir) throws HiveException
{
     Table tbl = getTable(tableName);
     if (replace) {
       tbl.replaceFiles(loadPath, isSrcLocal);
@@ -1478,6 +1480,20 @@ private void constructOneLBLocationMap(F
       tbl.copyFiles(loadPath, isSrcLocal);
     }
 
+    try {
+      if (isSkewedStoreAsSubdir) {
+        SkewedInfo skewedInfo = tbl.getSkewedInfo();
+        // Construct list bucketing location mappings from sub-directory name.
+        Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap(
+            tbl.getPath(), skewedInfo);
+        // Add list bucketing location mappings.
+        skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps);
+      }
+    } catch (IOException e) {
+      LOG.error(StringUtils.stringifyException(e));
+      throw new HiveException(e);
+    }
+
     if (!holdDDLTime) {
       try {
         alterTable(tableName, tbl);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
Tue Apr 29 01:10:34 2014
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.UD
 import org.apache.hadoop.hive.ql.lib.NodeProcessor;
 import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.optimizer.PrunerOperatorFactory;
 import org.apache.hadoop.hive.ql.optimizer.pcr.PcrOpProcFactory;
 import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
@@ -51,23 +52,24 @@ public class LBPartitionProcFactory exte
     protected void generatePredicate(NodeProcessorCtx procCtx, FilterOperator fop,
         TableScanOperator top) throws SemanticException, UDFArgumentException {
       LBOpPartitionWalkerCtx owc = (LBOpPartitionWalkerCtx) procCtx;
-
-      //Run partition pruner to get partitions
-      ParseContext parseCtx = owc.getParseContext();
-      PrunedPartitionList prunedPartList;
-      try {
-        String alias = (String) parseCtx.getTopOps().keySet().toArray()[0];
-        prunedPartList = PartitionPruner.prune(top, parseCtx, alias);
-      } catch (HiveException e) {
-        // Has to use full name to make sure it does not conflict with
-        // org.apache.commons.lang.StringUtils
-        throw new SemanticException(e.getMessage(), e);
-      }
-
-      if (prunedPartList != null) {
-        owc.setPartitions(prunedPartList);
+      Table tbl = owc.getParseContext().getTopToTable().get(top);
+      if (tbl.isPartitioned()) {
+        // Run partition pruner to get partitions
+        ParseContext parseCtx = owc.getParseContext();
+        PrunedPartitionList prunedPartList;
+        try {
+          String alias = (String) parseCtx.getTopOps().keySet().toArray()[0];
+          prunedPartList = PartitionPruner.prune(top, parseCtx, alias);
+        } catch (HiveException e) {
+          // Has to use full name to make sure it does not conflict with
+          // org.apache.commons.lang.StringUtils
+          throw new SemanticException(e.getMessage(), e);
+        }
+
+        if (prunedPartList != null) {
+          owc.setPartitions(prunedPartList);
+        }
       }
-
     }
 
   }

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java Tue Apr 29 01:10:34
2014
@@ -132,7 +132,7 @@ public class TestExecDriver extends Test
         db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
         db.createTable(src, cols, null, TextInputFormat.class,
             IgnoreKeyTextOutputFormat.class);
-        db.loadTable(hadoopDataFile[i], src, false, false, true);
+        db.loadTable(hadoopDataFile[i], src, false, false, true, false);
         i++;
       }
 

Added: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q?rev=1590853&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q Tue Apr 29 01:10:34
2014
@@ -0,0 +1,38 @@
+set hive.mapred.supports.subdirectories=true;
+set hive.exec.dynamic.partition=true;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+set hive.merge.mapfiles=false;
+set hive.merge.mapredfiles=false;
+set mapred.input.dir.recursive=true;
+
+-- list bucketing DML : unpartitioned table and 2 stage query plan.
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+
+-- create a skewed table
+create table list_bucketing (key String, value String)
+skewed by (key) on ("484")
+stored as DIRECTORIES
+;
+
+-- list bucketing DML
+explain extended
+insert overwrite table list_bucketing select * from src;
+insert overwrite table list_bucketing select * from src;
+
+-- check DML result
+desc formatted list_bucketing;
+
+select count(1) from src;
+select count(1) from list_bucketing;
+
+select key, value from src where key = "484";
+set hive.optimize.listbucketing=true;
+explain extended
+select key, value from list_bucketing where key = "484";
+select key, value from list_bucketing where key = "484";
+
+-- clean up resources
+drop table list_bucketing;
+

Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out?rev=1590853&view=auto
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out (added) and hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
Tue Apr 29 01:10:34 2014 differ



Mime
View raw message