hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nzh...@apache.org
Subject svn commit: r951256 - in /hadoop/hive/trunk: CHANGES.txt shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
Date Fri, 04 Jun 2010 03:55:45 GMT
Author: nzhang
Date: Fri Jun  4 03:55:45 2010
New Revision: 951256

URL: http://svn.apache.org/viewvc?rev=951256&view=rev
Log:
HIVE-1388. combinehiveinputformat does not work if files are of different types (Namit Jain
via Ning Zhang)

Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=951256&r1=951255&r2=951256&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Jun  4 03:55:45 2010
@@ -489,6 +489,10 @@ Trunk -  Unreleased
     handle matching by path
     (Paul Yang via Ning Zhang)
 
+    HIVE-1388. combinehiveinputformat does not work if files are of different
+    types
+    (Namit Jain via Ning Zhang)
+
 Release 0.5.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=951256&r1=951255&r2=951256&view=diff
==============================================================================
--- hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
(original)
+++ hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
Fri Jun  4 03:55:45 2010
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.shims;
 
 import java.io.IOException;
 import java.lang.reflect.Constructor;
+import java.io.DataOutput;
+import java.io.DataInput;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -43,6 +45,7 @@ import org.apache.hadoop.mapred.OutputCo
 import org.apache.hadoop.mapred.TaskAttemptContext;
 import org.apache.hadoop.mapred.JobContext;
 import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.io.WritableComparable;
 
 /**
  * Implemention of shims against Hadoop 0.20.0.
@@ -138,6 +141,35 @@ public class Hadoop20Shims implements Ha
     }
   }
 
+  public static class CombineHiveKey implements WritableComparable {
+    Object key;
+
+    public CombineHiveKey(Object key) {
+      this.key = key;
+    }
+
+    public Object getKey() {
+      return key;
+    }
+
+    public void setKey(Object key) {
+      this.key = key;
+    }
+
+    public void write(DataOutput out) throws IOException {
+      throw new IOException("Method not supported");
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      throw new IOException("Method not supported");
+    }
+
+    public int compareTo(Object w) {
+      assert false;
+      return 0;
+    }
+  }
+
   /* This class should be replaced with org.apache.hadoop.mapred.lib.CombineFileRecordReader
class, once
    * https://issues.apache.org/jira/browse/MAPREDUCE-955 is fixed. This code should be removed
- it is a copy
    * of org.apache.hadoop.mapred.lib.CombineFileRecordReader
@@ -164,8 +196,8 @@ public class Hadoop20Shims implements Ha
 
     public boolean next(K key, V value) throws IOException {
 
-      while ((curReader == null) || !curReader.next(key, value)) {
-        if (!initNextRecordReader()) {
+      while ((curReader == null) || !curReader.next((K)((CombineHiveKey)key).getKey(), value))
{
+        if (!initNextRecordReader(key)) {
           return false;
         }
       }
@@ -173,7 +205,8 @@ public class Hadoop20Shims implements Ha
     }
 
     public K createKey() {
-      return curReader.createKey();
+      K newKey = curReader.createKey();
+      return (K)(new CombineHiveKey(newKey));
     }
 
     public V createValue() {
@@ -224,13 +257,13 @@ public class Hadoop20Shims implements Ha
         throw new RuntimeException(rrClass.getName() +
             " does not have valid constructor", e);
       }
-      initNextRecordReader();
+      initNextRecordReader(null);
     }
 
     /**
      * Get the record reader for the next chunk in this CombineFileSplit.
      */
-    protected boolean initNextRecordReader() throws IOException {
+    protected boolean initNextRecordReader(K key) throws IOException {
 
       if (curReader != null) {
         curReader.close();
@@ -250,6 +283,12 @@ public class Hadoop20Shims implements Ha
         curReader = rrConstructor.newInstance(new Object[]
             {split, jc, reporter, Integer.valueOf(idx)});
 
+        // change the key if need be
+        if (key != null) {
+          K newKey = curReader.createKey();
+          ((CombineHiveKey)key).setKey(newKey);
+        }
+
         // setup some helper config variables.
         jc.set("map.input.file", split.getPath(idx).toString());
         jc.setLong("map.input.start", split.getOffset(idx));



Mime
View raw message