hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmhs...@apache.org
Subject svn commit: r1341203 - in /hbase/branches/0.90: ./ src/main/java/org/apache/hadoop/hbase/mapred/ src/main/java/org/apache/hadoop/hbase/mapreduce/ src/test/java/org/apache/hadoop/hbase/mapred/
Date Mon, 21 May 2012 21:02:28 GMT
Author: jmhsieh
Date: Mon May 21 21:02:27 2012
New Revision: 1341203

URL: http://svn.apache.org/viewvc?rev=1341203&view=rev
Log:
HBASE-5757 TableInputFormat should handle as many errors as possible (Jan Lukavsky)

Modified:
    hbase/branches/0.90/CHANGES.txt
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
    hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java

Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1341203&r1=1341202&r2=1341203&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Mon May 21 21:02:27 2012
@@ -30,6 +30,7 @@ Release 0.90.7 - Unreleased
                (Jimmy Xiang)
    HBASE-5894  Table deletion failed but HBaseAdmin#deletetable reports it as success (Xufeng)
    HBASE-5922  HalfStoreFileReader seekBefore causes StackOverflowError (Nate Putnam)
+   HBASE-5757  TableInputFormat should handle as many errors as possible (Jan Lukavsky)
 
   IMPROVEMENT
    HBASE-5588  Deprecate/remove AssignmentManager#clearRegionFromTransition

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java?rev=1341203&r1=1341202&r2=1341203&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
(original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
Mon May 21 21:02:27 2012
@@ -174,9 +174,9 @@ public class TableRecordReaderImpl {
     Result result;
     try {
       result = this.scanner.next();
-    } catch (DoNotRetryIOException e) {
-      throw e;
     } catch (IOException e) {
+      // try to handle all IOExceptions by restarting
+      // the scanner, if the second call fails, it will be rethrown
       LOG.debug("recovered from " + StringUtils.stringifyException(e));
       if (lastSuccessfulRow == null) {
         LOG.warn("We are restarting the first next() invocation," +

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java?rev=1341203&r1=1341202&r2=1341203&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
(original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
Mon May 21 21:02:27 2012
@@ -23,7 +23,6 @@ import java.io.IOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
@@ -131,9 +130,9 @@ public class TableRecordReaderImpl {
     if (value == null) value = new Result();
     try {
       value = this.scanner.next();
-    } catch (DoNotRetryIOException e) {
-      throw e;
     } catch (IOException e) {
+      // try to handle all IOExceptions by restarting
+      // the scanner, if the second call fails, it will be rethrown
       LOG.debug("recovered from " + StringUtils.stringifyException(e));
       if (lastSuccessfulRow == null) {
         LOG.warn("We are restarting the first next() invocation," +

Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java?rev=1341203&r1=1341202&r2=1341203&view=diff
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
(original)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
Mon May 21 21:02:27 2012
@@ -195,16 +195,16 @@ public class TestTableInputFormat {
    * 
    * @throws IOException
    */
-  static HTable createIOEScannerTable(byte[] name) throws IOException {
+  static HTable createIOEScannerTable(byte[] name, final int failCnt)
+      throws IOException {
     // build up a mock scanner stuff to fail the first time
     Answer<ResultScanner> a = new Answer<ResultScanner>() {
-      boolean first = true;
+      int cnt = 0;
 
       @Override
       public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
         // first invocation return the busted mock scanner
-        if (first) {
-          first = false;
+        if (cnt++ < failCnt) {
           // create mock ResultScanner that always fails.
           Scan scan = mock(Scan.class);
           doReturn("bogus".getBytes()).when(scan).getStartRow(); // avoid npe
@@ -230,16 +230,16 @@ public class TestTableInputFormat {
    * 
    * @throws IOException
    */
-  static HTable createDNRIOEScannerTable(byte[] name) throws IOException {
+  static HTable createDNRIOEScannerTable(byte[] name, final int failCnt)
+      throws IOException {
     // build up a mock scanner stuff to fail the first time
     Answer<ResultScanner> a = new Answer<ResultScanner>() {
-      boolean first = true;
+      int cnt = 0;
 
       @Override
       public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
         // first invocation return the busted mock scanner
-        if (first) {
-          first = false;
+        if (cnt++ < failCnt) {
           // create mock ResultScanner that always fails.
           Scan scan = mock(Scan.class);
           doReturn("bogus".getBytes()).when(scan).getStartRow(); // avoid npe
@@ -280,7 +280,18 @@ public class TestTableInputFormat {
    */
   @Test
   public void testTableRecordReaderScannerFail() throws IOException {
-    HTable htable = createIOEScannerTable("table2".getBytes());
+    HTable htable = createIOEScannerTable("table2".getBytes(), 1);
+    runTestMapred(htable);
+  }
+
+  /**
+   * Run test assuming Scanner IOException failure using mapred api,
+   * 
+   * @throws IOException
+   */
+  @Test(expected = IOException.class)
+  public void testTableRecordReaderScannerFailTwice() throws IOException {
+    HTable htable = createIOEScannerTable("table3".getBytes(), 2);
     runTestMapred(htable);
   }
 
@@ -290,9 +301,21 @@ public class TestTableInputFormat {
    * 
    * @throws DoNotRetryIOException
    */
-  @Test(expected = DoNotRetryIOException.class)
+  @Test
   public void testTableRecordReaderScannerTimeout() throws IOException {
-    HTable htable = createDNRIOEScannerTable("table3".getBytes());
+    HTable htable = createDNRIOEScannerTable("table4".getBytes(), 1);
+    runTestMapred(htable);
+  }
+
+  /**
+   * Run test assuming UnknownScannerException (which is a type of
+   * DoNotRetryIOException) using mapred api.
+   * 
+   * @throws DoNotRetryIOException
+   */
+  @Test(expected = DoNotRetryIOException.class)
+  public void testTableRecordReaderScannerTimeoutTwice() throws IOException {
+    HTable htable = createDNRIOEScannerTable("table5".getBytes(), 2);
     runTestMapred(htable);
   }
 
@@ -318,7 +341,20 @@ public class TestTableInputFormat {
   @Test
   public void testTableRecordReaderScannerFailMapreduce() throws IOException,
       InterruptedException {
-    HTable htable = createIOEScannerTable("table2-mr".getBytes());
+    HTable htable = createIOEScannerTable("table2-mr".getBytes(), 1);
+    runTestMapreduce(htable);
+  }
+
+  /**
+   * Run test assuming Scanner IOException failure using newer mapreduce api
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Test(expected = IOException.class)
+  public void testTableRecordReaderScannerFailMapreduceTwice() throws IOException,
+      InterruptedException {
+    HTable htable = createIOEScannerTable("table3-mr".getBytes(), 2);
     runTestMapreduce(htable);
   }
 
@@ -329,10 +365,24 @@ public class TestTableInputFormat {
    * @throws InterruptedException
    * @throws DoNotRetryIOException
    */
-  @Test(expected = DoNotRetryIOException.class)
+  @Test
   public void testTableRecordReaderScannerTimeoutMapreduce()
       throws IOException, InterruptedException {
-    HTable htable = createDNRIOEScannerTable("table3-mr".getBytes());
+    HTable htable = createDNRIOEScannerTable("table4-mr".getBytes(), 1);
+    runTestMapreduce(htable);
+  }
+
+  /**
+   * Run test assuming UnknownScannerException (which is a type of
+   * DoNotRetryIOException) using newer mapreduce api
+   * 
+   * @throws InterruptedException
+   * @throws DoNotRetryIOException
+   */
+  @Test(expected = DoNotRetryIOException.class)
+  public void testTableRecordReaderScannerTimeoutMapreduceTwice()
+      throws IOException, InterruptedException {
+    HTable htable = createDNRIOEScannerTable("table5-mr".getBytes(), 2);
     runTestMapreduce(htable);
   }
 }



Mime
View raw message