hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r999758 - in /hadoop/hive/trunk: CHANGES.txt metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
Date Wed, 22 Sep 2010 05:22:27 GMT
Author: namit
Date: Wed Sep 22 05:22:27 2010
New Revision: 999758

URL: http://svn.apache.org/viewvc?rev=999758&view=rev
Log:
HIVE-1655.  Adding consistency check at jobClose() when committing dynamic
partitions (Ning Zhang via namit)


Removed:
    hadoop/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=999758&r1=999757&r2=999758&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Sep 22 05:22:27 2010
@@ -288,6 +288,9 @@ Trunk -  Unreleased
     HIVE-1534. Join filters do not work correctly with outer joins
     (Amareshwari Sriramadasu via namit)
 
+    HIVE-1655.  Adding consistency check at jobClose() when committing dynamic
+    partitions (Ning Zhang via namit)
+
   TESTS
 
     HIVE-1464. improve  test query performance

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=999758&r1=999757&r2=999758&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Sep 22
05:22:27 2010
@@ -1134,6 +1134,16 @@ public final class Utilities {
       for (int i = 0; i < parts.length; ++i) {
         assert parts[i].isDir(): "dynamic partition " + parts[i].getPath() + " is not a direcgtory";
         FileStatus[] items = fs.listStatus(parts[i].getPath());
+
+        // remove empty directory since DP insert should not generate empty partitions.
+        // empty directories could be generated by crashed Task/ScriptOperator
+        if (items.length == 0) {
+          if (!fs.delete(parts[i].getPath(), true)) {
+            LOG.error("Cannot delete empty directory " + parts[i].getPath());
+            throw new IOException("Cannot delete empty directory " + parts[i].getPath());
+          }
+        }
+
         taskIDToFile = removeTempOrDuplicateFiles(items, fs);
         // if the table is bucketed and enforce bucketing, we should check and generate all
buckets
         if (dpCtx.getNumBuckets() > 0 && taskIDToFile != null) {



Mime
View raw message