hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jdcry...@apache.org
Subject svn commit: r1055562 - in /hbase/branches/0.90: ./ src/main/java/org/apache/hadoop/hbase/regionserver/wal/ src/main/java/org/apache/hadoop/hbase/util/ src/test/java/org/apache/hadoop/hbase/regionserver/wal/
Date Wed, 05 Jan 2011 18:05:53 GMT
Author: jdcryans
Date: Wed Jan  5 18:05:52 2011
New Revision: 1055562

URL: http://svn.apache.org/viewvc?rev=1055562&view=rev
Log:
HBASE-3412  HLogSplitter should handle missing HLogs

Modified:
    hbase/branches/0.90/CHANGES.txt
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
    hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java

Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1055562&r1=1055561&r2=1055562&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Wed Jan  5 18:05:52 2011
@@ -777,6 +777,7 @@ Release 0.90.0 - Unreleased
    HBASE-3402  Web UI shows two META regions
    HBASE-3409  Failed server shutdown processing when retrying hlog split
    HBASE-3410  Unable to set/modify TTL on a column family using the shell
+   HBASE-3412  HLogSplitter should handle missing HLogs
 
 
   IMPROVEMENTS

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java?rev=1055562&r1=1055561&r2=1055562&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
(original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
Wed Jan  5 18:05:52 2011
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.regionse
 import static org.apache.hadoop.hbase.util.FSUtils.recoverFileLease;
 
 import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
@@ -262,7 +263,14 @@ public class HLogSplitter {
           processedLogs.add(logPath);
         } catch (EOFException eof) {
           // truncated files are expected if a RS crashes (see HBASE-2643)
-          LOG.info("EOF from hlog " + logPath + ".  continuing");
+          LOG.info("EOF from hlog " + logPath + ". Continuing");
+          processedLogs.add(logPath);
+        } catch (FileNotFoundException fnfe) {
+          // A file may be missing if the region server was able to archive it
+          // before shutting down. This means the edits were persisted already
+          LOG.info("A log was missing " + logPath +
+              ", probably because it was moved by the" +
+              " now dead region server. Continuing");
           processedLogs.add(logPath);
         } catch (IOException e) {
           // If the IOE resulted from bad file format,

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1055562&r1=1055561&r2=1055562&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Wed Jan  5
18:05:52 2011
@@ -36,10 +36,12 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
+import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
 import org.apache.hadoop.io.SequenceFile;
 
 import java.io.DataInputStream;
 import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -649,6 +651,11 @@ public class FSUtils {
           } catch (InterruptedException ex) {
             // ignore it and try again
           }
+        } else if (e instanceof LeaseExpiredException &&
+            e.getMessage().contains("File does not exist")) {
+          // This exception comes out instead of FNFE, fix it
+          throw new FileNotFoundException(
+              "The given HLog wasn't found at " + p.toString());
         } else {
           throw new IOException("Failed to open " + p + " for append", e);
         }

Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java?rev=1055562&r1=1055561&r2=1055562&view=diff
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
(original)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
Wed Jan  5 18:05:52 2011
@@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
 import org.apache.hadoop.ipc.RemoteException;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -672,6 +673,33 @@ public class TestHLogSplit {
       assertTrue(ioe.toString().contains("Injected"));
     }
   }
+
+  // Test for HBASE-3412
+  @Test
+  public void testMovedHLogDuringRecovery() throws Exception {
+    generateHLogs(-1);
+
+    fs.initialize(fs.getUri(), conf);
+
+    // This partial mock will throw LEE for every file simulating
+    // files that were moved
+    FileSystem spiedFs = Mockito.spy(fs);
+    // The "File does not exist" part is very important,
+    // that's how it comes out of HDFS
+    Mockito.doThrow(new LeaseExpiredException("Injected: File does not exist")).
+        when(spiedFs).append(Mockito.<Path>any());
+
+    HLogSplitter logSplitter = new HLogSplitter(
+        conf, hbaseDir, hlogDir, oldLogDir, spiedFs);
+
+    try {
+      logSplitter.splitLog();
+      assertEquals(NUM_WRITERS, fs.listStatus(oldLogDir).length);
+      assertFalse(fs.exists(hlogDir));
+    } catch (IOException e) {
+      fail("There shouldn't be any exception but: " + e.toString());
+    }
+  }
   
   /**
    * Test log split process with fake data and lots of edits to trigger threading



Mime
View raw message