incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1388613 - /incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Date Fri, 21 Sep 2012 18:47:02 GMT
Author: toffer
Date: Fri Sep 21 18:47:01 2012
New Revision: 1388613

URL: http://svn.apache.org/viewvc?rev=1388613&view=rev
Log:
addenum to HCAT-490, added extra comment

Modified:
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java?rev=1388613&r1=1388612&r2=1388613&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Fri Sep 21 18:47:01 2012
@@ -195,6 +195,13 @@ class FileRecordWriterContainer extends 
                 //create base OutputFormat
                 org.apache.hadoop.mapred.OutputFormat baseOF =
                     ReflectionUtils.newInstance(storageHandler.getOutputFormatClass(), currTaskContext.getJobConf());
+
+                //We are skipping calling checkOutputSpecs() for each partition
+                //As it can throw a FileAlreadyExistsException when more than one mapper
is writing to a partition
+                //See HCATALOG-490, also to avoid contacting the namenode for each new FileOutputFormat
instance
+                //In general this should be ok for most FileOutputFormat implementations
+                //but may become an issue for cases when the method is used to perform other
setup tasks
+
                 //get Output Committer
                 org.apache.hadoop.mapred.OutputCommitter baseOutputCommitter = currTaskContext.getJobConf().getOutputCommitter();
                 //create currJobContext the latest so it gets all the config changes



Mime
View raw message