incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject svn commit: r1363523 - in /incubator/hcatalog/trunk: CHANGES.txt src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Date Thu, 19 Jul 2012 20:22:00 GMT
Author: khorgath
Date: Thu Jul 19 20:21:59 2012
New Revision: 1363523

URL: http://svn.apache.org/viewvc?rev=1363523&view=rev
Log:
HCATALOG-448 HCatStorer performance is 4x slower in HCat 0.4 than HCat 0.2 (rajesh.balamohan
via khorgath)

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1363523&r1=1363522&r2=1363523&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Jul 19 20:21:59 2012
@@ -61,6 +61,8 @@ Trunk (unreleased changes)
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-448 HCatStorer performance is 4x slower in HCat 0.4 than HCat 0.2 (rajesh.balamohan
via khorgath)
+
   HCAT-447 findbugs target doesn't work in trunk (traviscrawford)
 
   HCAT-437 HCatSchema's equals() implementation uses hashcode obtained from toString() method.
(avandana)

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java?rev=1363523&r1=1363522&r2=1363523&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Thu Jul 19 20:21:59 2012
@@ -62,6 +62,7 @@ class FileRecordWriterContainer extends 
     private final Map<String, org.apache.hadoop.mapred.OutputCommitter> baseDynamicCommitters;
     private final Map<String, org.apache.hadoop.mapred.TaskAttemptContext> dynamicContexts;
     private final Map<String, ObjectInspector> dynamicObjectInspectors;
+    private Map<String, OutputJobInfo> dynamicOutputJobInfo;
 
 
     private final List<Integer> partColsToDel;
@@ -110,6 +111,7 @@ class FileRecordWriterContainer extends 
             this.baseDynamicCommitters = null;
             this.dynamicContexts = null;
             this.dynamicObjectInspectors = null;
+            this.dynamicOutputJobInfo = null;
         }
         else {
             this.baseDynamicSerDe = new HashMap<String,SerDe>();
@@ -117,11 +119,12 @@ class FileRecordWriterContainer extends 
             this.baseDynamicCommitters = new HashMap<String, org.apache.hadoop.mapred.OutputCommitter>();
             this.dynamicContexts = new HashMap<String, org.apache.hadoop.mapred.TaskAttemptContext>();
             this.dynamicObjectInspectors = new HashMap<String, ObjectInspector>();
+            this.dynamicOutputJobInfo = new HashMap<String, OutputJobInfo>();
         }
     }
 
     /**
-     * @return the storagehandler
+     * @return the storageDriver
      */
     public HCatStorageHandler getStorageHandler() {
         return storageHandler;
@@ -158,7 +161,6 @@ class FileRecordWriterContainer extends 
         SerDe localSerDe;
         OutputJobInfo localJobInfo = null;
 
-
         if (dynamicPartitioningUsed){
             // calculate which writer to use from the remaining values - this needs to be
done before we delete cols
             List<String> dynamicPartValues = new ArrayList<String>();
@@ -223,18 +225,19 @@ class FileRecordWriterContainer extends 
                 baseDynamicCommitters.put(dynKey,baseOutputCommitter);
                 dynamicContexts.put(dynKey,currTaskContext);
                 dynamicObjectInspectors.put(dynKey,InternalUtil.createStructObjectInspector(jobInfo.getOutputSchema()));
+                dynamicOutputJobInfo.put(dynKey, HCatOutputFormat.getJobInfo(dynamicContexts.get(dynKey)));
             }
-            localJobInfo = HCatOutputFormat.getJobInfo(dynamicContexts.get(dynKey));
+            
+            localJobInfo = dynamicOutputJobInfo.get(dynKey);
             localWriter = baseDynamicWriters.get(dynKey);
             localSerDe = baseDynamicSerDe.get(dynKey);
             localContext = dynamicContexts.get(dynKey);
             localObjectInspector = dynamicObjectInspectors.get(dynKey);
         }
         else{
-            localJobInfo = HCatBaseOutputFormat.getJobInfo(context);
+            localJobInfo = jobInfo;
             localWriter = getBaseRecordWriter();
             localSerDe = serDe;
-            localContext = HCatMapRedUtil.createTaskAttemptContext(context);
             localObjectInspector = objectInspector;
         }
 



Mime
View raw message