incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1459476 - in /incubator/hcatalog/trunk: CHANGES.txt core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
Date Thu, 21 Mar 2013 19:01:12 GMT
Author: daijy
Date: Thu Mar 21 19:01:11 2013
New Revision: 1459476

URL: http://svn.apache.org/r1459476
Log:
HCAT-628 Committer deletes the original partition when a duplicate partition fails registration

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
    incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1459476&r1=1459475&r2=1459476&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Mar 21 19:01:11 2013
@@ -31,6 +31,9 @@ Trunk (unreleased changes)
   OPTIMIZATIONS
 
   BUG FIXES
+
+  HCAT-628 Committer deletes the original partition when a duplicate partition fails registration
(thiruvel via daijy)
+
   HCAT-554 Loading data using HCatLoader() from a table on non default namenode fails. (amalakar
via avandana)
 
   HCAT-629 First request always fails for webhcat (kamrul via gates)

Modified: incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1459476&r1=1459475&r2=1459476&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
(original)
+++ incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Thu Mar 21 19:01:11 2013
@@ -681,14 +681,14 @@ class FileOutputCommitterContainer exten
                 // no harProcessor, regular operation
                 updateTableSchema(client, table, jobInfo.getOutputSchema());
                 LOG.info("HAR not is not being used. The table {} has new partitions {}.",
table.getTableName(), ptnInfos);
-                partitionsAdded = partitionsToAdd;
-                if (dynamicPartitioningUsed && (partitionsAdded.size()>0)){
+                if (dynamicPartitioningUsed && (partitionsToAdd.size()>0)){
                     Path src = new Path(ptnRootLocation);
                     moveTaskOutputs(fs, src, src, tblPath, true);
                     moveTaskOutputs(fs, src, src, tblPath, false);
                     fs.delete(src, true);
                 }
                 client.add_partitions(partitionsToAdd);
+                partitionsAdded = partitionsToAdd;
             }
         } catch (Exception e) {
             if (partitionsAdded.size() > 0) {

Modified: incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java?rev=1459476&r1=1459475&r2=1459476&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
(original)
+++ incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
Thu Mar 21 19:01:11 2013
@@ -160,6 +160,24 @@ public class TestHCatDynamicPartitioned 
                             || (ErrorType.ERROR_MOVE_FAILED == ((HCatException) exc).getErrorType())
             );
         }
+
+        query = "show partitions " + tableName;
+        retCode = driver.run(query).getResponseCode();
+        if (retCode != 0) {
+            throw new Exception("Error " + retCode + " running query " + query);
+        }
+        res = new ArrayList<String>();
+        driver.getResults(res);
+        assertEquals(NUM_PARTITIONS, res.size());
+
+        query = "select * from " + tableName;
+        retCode = driver.run(query).getResponseCode();
+        if (retCode != 0) {
+            throw new Exception("Error " + retCode + " running query " + query);
+        }
+        res = new ArrayList<String>();
+        driver.getResults(res);
+        assertEquals(NUM_RECORDS, res.size());
     }
 
     //TODO 1.0 miniCluster is slow this test times out, make it work



Mime
View raw message