incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1239960 - in /incubator/hcatalog/trunk: CHANGES.txt src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Date Fri, 03 Feb 2012 00:57:43 GMT
Author: gates
Date: Fri Feb  3 00:57:42 2012
New Revision: 1239960

URL: http://svn.apache.org/viewvc?rev=1239960&view=rev
Log:
HCATALOG-232 Dynamic Partitioning broken: keys set to HIVE_DEFAULT_PARTITION

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1239960&r1=1239959&r2=1239960&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri Feb  3 00:57:42 2012
@@ -131,6 +131,8 @@ Release 0.3.0 - Unreleased
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-232. Dynamic Partitioning broken: keys set to HIVE_DEFAULT_PARTITION (mithun via gates)
+
   HCAT-229. bin/hcat should put hcatalog.jar into classpath (daiky via gates)
 
   HCAT-234. bin/hcat need to add hbase-site.xml in classpath (daijy via toffer)

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java?rev=1239960&r1=1239959&r2=1239960&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputFormatContainer.java
Fri Feb  3 00:57:42 2012
@@ -68,7 +68,12 @@ class FileOutputFormatContainer extends 
 
     @Override
     public RecordWriter<WritableComparable<?>, HCatRecord> getRecordWriter(TaskAttemptContext
context) throws IOException, InterruptedException {
-        return new FileRecordWriterContainer(of.getRecordWriter(context),context);
+        // When Dynamic partitioning is used, the RecordWriter instance initialized here
isn't used. Can use null.
+        // (That's because records can't be written until the values of the dynamic partitions
are deduced.
+        // By that time, a new local instance of RecordWriter, with the correct output-path,
will be constructed.)
+        return new FileRecordWriterContainer(HCatOutputFormat.getJobInfo(context)
+                                                   .isDynamicPartitioningUsed()? null : of.getRecordWriter(context),
+                                             context);
     }
 
     @Override

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java?rev=1239960&r1=1239959&r2=1239960&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Fri Feb  3 00:57:42 2012
@@ -69,7 +69,6 @@ class FileRecordWriterContainer extends 
         super(context,baseWriter);
         this.context = context;
         jobInfo = HCatOutputFormat.getJobInfo(context);
-        storageDriver = HCatOutputFormat.getOutputDriverInstance(context,jobInfo);
 
         // If partition columns occur in data, we want to remove them.
         partColsToDel = jobInfo.getPosOfPartCols();
@@ -84,12 +83,14 @@ class FileRecordWriterContainer extends 
 
 
         if (!dynamicPartitioningUsed) {
+            storageDriver = HCatOutputFormat.getOutputDriverInstance(context,jobInfo);
             this.baseDynamicStorageDrivers = null;
             this.baseDynamicWriters = null;
             this.baseDynamicCommitters = null;
             prepareForStorageDriverOutput(context);
         }
         else {
+            storageDriver = null;
             this.baseDynamicStorageDrivers = new HashMap<Integer,HCatOutputStorageDriver>();
             this.baseDynamicWriters = new HashMap<Integer,RecordWriter<? super WritableComparable<?>,
? super Writable>>();
             this.baseDynamicCommitters = new HashMap<Integer,OutputCommitter>();



Mime
View raw message