incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1329323 - in /incubator/hcatalog/branches/branch-0.4: ./ CHANGES.txt build.xml src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Date Mon, 23 Apr 2012 16:47:55 GMT
Author: toffer
Date: Mon Apr 23 16:47:54 2012
New Revision: 1329323

URL: http://svn.apache.org/viewvc?rev=1329323&view=rev
Log:
merged from trunk: HCATALOG-379 Fix mapred.out.dir hack in FileOutputCommitterContainer (toffer)

Modified:
    incubator/hcatalog/branches/branch-0.4/   (props changed)
    incubator/hcatalog/branches/branch-0.4/CHANGES.txt
    incubator/hcatalog/branches/branch-0.4/build.xml
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java

Propchange: incubator/hcatalog/branches/branch-0.4/
------------------------------------------------------------------------------
  Merged /incubator/hcatalog/trunk:r1329321

Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1329323&r1=1329322&r2=1329323&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Mon Apr 23 16:47:54 2012
@@ -79,6 +79,8 @@ Release 0.4.0 - Unreleased
   HCAT-2 Support nested schema conversion between Hive an Pig (julienledem via hashutosh)
 
   IMPROVEMENTS
+  HCAT-379 Fix mapred.out.dir hack in FileOutputCommitterContainer (toffer)
+
   HCAT-69 Fix token usage from HCat (toffer)
 
   HCAT-357 e2e harness: allow to run again pre-installed packages (thw via gates)

Modified: incubator/hcatalog/branches/branch-0.4/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/build.xml?rev=1329323&r1=1329322&r2=1329323&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/build.xml (original)
+++ incubator/hcatalog/branches/branch-0.4/build.xml Mon Apr 23 16:47:54 2012
@@ -69,7 +69,7 @@
   <property name="test.exclude.file" value="${test.src.dir}/excluded-tests"/>
   <property name="test.output" value="no"/>
   <property name="test.warehouse.dir" value="/tmp/hcat_junit_warehouse"/>
-  <property name="test.excludes" value="${test.src.dir}/e2e/**"/>
+  <property name="test.excludes" value="e2e/**"/>
 
   <!-- e2e test properties -->
   <property name="test.e2e.dir" value="${basedir}/src/test/e2e/hcatalog"/>

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1329323&r1=1329322&r2=1329323&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Mon Apr 23 16:47:54 2012
@@ -95,9 +95,6 @@ class FileOutputCommitterContainer exten
     @Override
     public void abortTask(TaskAttemptContext context) throws IOException {
         if (!dynamicPartitioningUsed){
-            // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the
same Pig script.
-            // Workaround: Set mapred.output.dir from OutputJobInfo.
-            resetMapRedOutputDirFromJobInfo(context.getConfiguration());
             getBaseOutputCommitter().abortTask(HCatMapRedUtil.createTaskAttemptContext(context));
         }
     }
@@ -105,11 +102,6 @@ class FileOutputCommitterContainer exten
     @Override
     public void commitTask(TaskAttemptContext context) throws IOException {
         if (!dynamicPartitioningUsed){
-            //TODO fix this hack, something wrong with pig
-            //running multiple storers in a single job, the real output dir got overwritten
or something
-            //the location in OutputJobInfo is still correct so we'll use that
-            //TestHCatStorer.testMultiPartColsInData() used to fail without this
-            resetMapRedOutputDirFromJobInfo(context.getConfiguration());
             getBaseOutputCommitter().commitTask(HCatMapRedUtil.createTaskAttemptContext(context));
         }
     }
@@ -117,9 +109,6 @@ class FileOutputCommitterContainer exten
     @Override
     public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
         if (!dynamicPartitioningUsed){
-            // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the
same Pig script.
-            // Workaround: Set mapred.output.dir from OutputJobInfo.
-            resetMapRedOutputDirFromJobInfo(context.getConfiguration());
             return getBaseOutputCommitter().needsTaskCommit(HCatMapRedUtil.createTaskAttemptContext(context));
         }else{
             // called explicitly through FileRecordWriterContainer.close() if dynamic - return
false by default
@@ -130,9 +119,6 @@ class FileOutputCommitterContainer exten
     @Override
     public void setupJob(JobContext context) throws IOException {
         if(getBaseOutputCommitter() != null && !dynamicPartitioningUsed) {
-            // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the
same Pig script.
-            // Workaround: Set mapred.output.dir from OutputJobInfo.
-            resetMapRedOutputDirFromJobInfo(context.getConfiguration());
             getBaseOutputCommitter().setupJob(HCatMapRedUtil.createJobContext(context));
         }
         // in dynamic usecase, called through FileRecordWriterContainer
@@ -141,9 +127,6 @@ class FileOutputCommitterContainer exten
     @Override
     public void setupTask(TaskAttemptContext context) throws IOException {
         if (!dynamicPartitioningUsed){
-            // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the
same Pig script.
-            // Workaround: Set mapred.output.dir from OutputJobInfo.
-            resetMapRedOutputDirFromJobInfo(context.getConfiguration());
             getBaseOutputCommitter().setupTask(HCatMapRedUtil.createTaskAttemptContext(context));
         }
     }
@@ -157,9 +140,6 @@ class FileOutputCommitterContainer exten
         }
 
         if(getBaseOutputCommitter() != null && !dynamicPartitioningUsed) {
-            // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the
same Pig script.
-            // Workaround: Set mapred.output.dir from OutputJobInfo.
-            resetMapRedOutputDirFromJobInfo(mapRedJobContext.getConfiguration());
             getBaseOutputCommitter().abortJob(mapRedJobContext, state);
         }
         else if (dynamicPartitioningUsed){
@@ -225,9 +205,6 @@ class FileOutputCommitterContainer exten
             discoverPartitions(jobContext);
         }
         if(getBaseOutputCommitter() != null && !dynamicPartitioningUsed) {
-            // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the
same Pig script.
-            // Workaround: Set mapred.output.dir from OutputJobInfo.
-            resetMapRedOutputDirFromJobInfo(jobContext.getConfiguration());
             getBaseOutputCommitter().commitJob(HCatMapRedUtil.createJobContext(jobContext));
         }
         // create _SUCCESS FILE if so requested.
@@ -265,9 +242,6 @@ class FileOutputCommitterContainer exten
         if( table.getPartitionKeys().size() == 0 ) {
             //non partitioned table
             if(getBaseOutputCommitter() != null && !dynamicPartitioningUsed) {
-               // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in
the same Pig script.
-               // Workaround: Set mapred.output.dir from OutputJobInfo.
-               resetMapRedOutputDirFromJobInfo(context.getConfiguration());
                getBaseOutputCommitter().cleanupJob(HCatMapRedUtil.createJobContext(context));
             }
             else if (dynamicPartitioningUsed){
@@ -683,15 +657,4 @@ class FileOutputCommitterContainer exten
         }
     }
 
-  /**
-   * TODO: Clean up this Hack! Resetting mapred.output.dir from OutputJobInfo.
-   * This works around PIG-2578, where Pig messes up output-directory
-   * if multiple storers are used in the same pig-script.
-   * @param config The configuration whose mapred.output.dir is to be reset.
-   */
-  private void resetMapRedOutputDirFromJobInfo(Configuration config) {
-    String outputLocation = jobInfo.getLocation();
-    if (outputLocation != null)
-      config.set("mapred.output.dir", outputLocation);
-  }
 }



Mime
View raw message