hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r986195 - in /hadoop/hive/branches/branch-0.6: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/io/
Date Tue, 17 Aug 2010 05:38:53 GMT
Author: namit
Date: Tue Aug 17 05:38:52 2010
New Revision: 986195

URL: http://svn.apache.org/viewvc?rev=986195&view=rev
Log:
HIVE-1543. Abort if record reader's next throws an exception
(Ning Zhang via namit)


Modified:
    hadoop/hive/branches/branch-0.6/CHANGES.txt
    hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
    hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java
    hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java
    hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java

Modified: hadoop/hive/branches/branch-0.6/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/CHANGES.txt?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.6/CHANGES.txt Tue Aug 17 05:38:52 2010
@@ -587,6 +587,9 @@ Release 0.6.0 -  Unreleased
     metastore
     (Pradeep Kamath via jvs)
 
+    HIVE-1543. Abort if record reader's next throws an exception
+    (Ning Zhang via namit)
+
 Release 0.5.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
(original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
Tue Aug 17 05:38:52 2010
@@ -51,7 +51,7 @@ public class ExecMapper extends MapReduc
   private Map<String, FetchOperator> fetchOperators;
   private OutputCollector oc;
   private JobConf jc;
-  private boolean abort = false;
+  private static boolean abort = false;
   private Reporter rp;
   public static final Log l4j = LogFactory.getLog("ExecMapper");
   private static boolean done;
@@ -213,6 +213,9 @@ public class ExecMapper extends MapReduc
 
       reportStats rps = new reportStats(rp);
       mo.preorderMap(rps);
+
+      // reset abort flag so that ExecMapper instance can be potentially reused
+      setAbort(false);
       return;
     } catch (Exception e) {
       if (!abort) {
@@ -231,8 +234,8 @@ public class ExecMapper extends MapReduc
     return abort;
   }
 
-  public void setAbort(boolean abort) {
-    this.abort = abort;
+  public static void setAbort(boolean abrt) {
+    abort = abrt;
   }
 
   public static void setDone(boolean done) {

Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java
(original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java
Tue Aug 17 05:38:52 2010
@@ -24,11 +24,9 @@ import org.apache.hadoop.hive.ql.exec.Ex
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RecordReader;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
 
 /**
  * BucketizedHiveRecordReader is a wrapper on a list of RecordReader. It behaves
@@ -87,12 +85,17 @@ public class BucketizedHiveRecordReader<
   }
 
   public boolean next(K key, V value) throws IOException {
-    while ((curReader == null) || !curReader.next(key, value)) {
-      if (!initNextRecordReader()) {
-        return false;
+    try {
+      while ((curReader == null) || !curReader.next(key, value)) {
+        if (!initNextRecordReader()) {
+       	  return false;
+        }
       }
+      return true;
+    } catch (IOException e) {
+      ExecMapper.setAbort(true);
+      throw e;
     }
-    return true;
   }
 
   /**

Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java
(original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java
Tue Aug 17 05:38:52 2010
@@ -92,6 +92,11 @@ public class CombineHiveRecordReader<K e
     if (ExecMapper.getDone()) {
       return false;
     }
-    return recordReader.next(key, value);
+    try {
+      return recordReader.next(key, value);
+    } catch (IOException e) {
+      ExecMapper.setAbort(true);
+      throw e;
+    }
   }
 }

Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java
(original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java
Tue Aug 17 05:38:52 2010
@@ -28,7 +28,7 @@ import org.apache.hadoop.mapred.RecordRe
 /**
  * HiveRecordReader is a simple wrapper on RecordReader.
  * It allows us to stop reading the data when some global flag
- * ExecMapper.getDone() is set. 
+ * ExecMapper.getDone() is set.
  */
 public class HiveRecordReader<K extends WritableComparable, V extends Writable>
     implements RecordReader<K, V> {
@@ -63,6 +63,11 @@ public class HiveRecordReader<K extends 
     if (ExecMapper.getDone()) {
       return false;
     }
-    return recordReader.next(key, value);
+    try {
+      return recordReader.next(key, value);
+    } catch (IOException e) {
+      ExecMapper.setAbort(true);
+      throw e;
+    }
   }
 }



Mime
View raw message