hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1183173 - in /hbase/branches/0.92: CHANGES.txt src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java
Date Fri, 14 Oct 2011 00:20:38 GMT
Author: garyh
Date: Fri Oct 14 00:20:38 2011
New Revision: 1183173

URL: http://svn.apache.org/viewvc?rev=1183173&view=rev
Log:
HBASE-4282  RegionServer should abort when WAL close fails with unflushed edits

Added:
    hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java
Modified:
    hbase/branches/0.92/CHANGES.txt
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java

Modified: hbase/branches/0.92/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1183173&r1=1183172&r2=1183173&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Fri Oct 14 00:20:38 2011
@@ -701,6 +701,8 @@ Release 0.90.5 - Unreleased
    HBASE-4537  TestUser imports breaking build against secure Hadoop
    HBASE-4501  [replication] Shutting down a stream leaves recovered
                sources running
+   HBASE-4282  Ensure RegionServer aborts when WAL close fails with
+               deferred flush edits
 
   IMPROVEMENT
    HBASE-4205  Enhance HTable javadoc (Eric Charles)

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1183173&r1=1183172&r2=1183173&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Fri
Oct 14 00:20:38 2011
@@ -193,6 +193,8 @@ public class HLog implements Syncable {
 
   //number of transactions in the current Hlog.
   private final AtomicInteger numEntries = new AtomicInteger(0);
+  // last deferred-flush WAL edit sequence number
+  private long lastDeferredSeq;
   // If live datanode count is lower than the default replicas value,
   // RollWriter will be triggered in each sync(So the RollWriter will be
   // triggered one by one in a short time). Using it as a workaround to slow
@@ -789,9 +791,12 @@ public class HLog implements Syncable {
       } catch (IOException e) {
         LOG.error("Failed close of HLog writer", e);
         int errors = closeErrorCount.incrementAndGet();
-        if (errors <= closeErrorsTolerated) {
+        if (errors <= closeErrorsTolerated && lastDeferredSeq == 0) {
           LOG.warn("Riding over HLog close failure! error count="+errors);
         } else {
+          if (lastDeferredSeq > 0) {
+            LOG.error("Aborting due to unflushed edits in HLog");
+          }
           // Failed close of log file.  Means we're losing edits.  For now,
           // shut ourselves down to minimize loss.  Alternative is to try and
           // keep going.  See HBASE-930.
@@ -935,6 +940,9 @@ public class HLog implements Syncable {
         Long.valueOf(seqNum));
       doWrite(regionInfo, logKey, logEdit, htd);
       this.numEntries.incrementAndGet();
+      if (htd.isDeferredLogFlush()) {
+        lastDeferredSeq = seqNum;
+      }
     }
 
     // Sync if catalog region, and if not then check if that table supports
@@ -1007,6 +1015,9 @@ public class HLog implements Syncable {
         HLogKey logKey = makeKey(hriKey, tableName, seqNum, now, clusterId);
         doWrite(info, logKey, edits, htd);
         this.numEntries.incrementAndGet();
+        if (htd.isDeferredLogFlush()) {
+          lastDeferredSeq = seqNum;
+        }
       }
       // Sync if catalog region, and if not then check if that table supports
       // deferred log flushing
@@ -1053,10 +1064,12 @@ public class HLog implements Syncable {
   }
 
   private void syncer() throws IOException {
+    long currentDeferredSeq;
     synchronized (this.updateLock) {
       if (this.closed) {
         return;
       }
+      currentDeferredSeq = lastDeferredSeq;
     }
     try {
       long now = System.currentTimeMillis();
@@ -1072,6 +1085,9 @@ public class HLog implements Syncable {
           // HBASE-4387, retry with updateLock held
           this.writer.sync();
         }
+        if (currentDeferredSeq == lastDeferredSeq) {
+          lastDeferredSeq = 0;  // reset
+        }
         syncTime += System.currentTimeMillis() - now;
         syncOps++;
         if (!this.logRollRunning) {
@@ -1625,6 +1641,11 @@ public class HLog implements Syncable {
     return coprocessorHost;
   }
 
+  /** Provide access to currently deferred sequence num for tests */
+  long getLastDeferredSeq() {
+    return lastDeferredSeq;
+  }
+
   /**
    * Pass one or more log file names and it will either dump out a text version
    * on <code>stdout</code> or split the specified log files.

Added: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java?rev=1183173&view=auto
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java
(added)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java
Fri Oct 14 00:20:38 2011
@@ -0,0 +1,156 @@
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests for conditions that should trigger RegionServer aborts when
+ * rolling the current HLog fails.
+ */
+public class TestLogRollAbort {
+  private static final Log LOG = LogFactory.getLog(TestLogRolling.class);
+  private static MiniDFSCluster dfsCluster;
+  private static HBaseAdmin admin;
+  private static MiniHBaseCluster cluster;
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  // verbose logging on classes that are touched in these tests
+  {
+    ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem"))
+        .getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)HRegionServer.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)HRegion.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL);
+  }
+
+  // Need to override this setup so we can edit the config before it gets sent
+  // to the HDFS & HBase cluster startup.
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    // Tweak default timeout values down for faster recovery
+    TEST_UTIL.getConfiguration().setInt(
+        "hbase.regionserver.logroll.errors.tolerated", 2);
+    TEST_UTIL.getConfiguration().setInt("ipc.ping.interval", 10 * 1000);
+    TEST_UTIL.getConfiguration().setInt("ipc.socket.timeout", 10 * 1000);
+    TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 10 * 1000);
+
+    // Increase the amount of time between client retries
+    TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 5 * 1000);
+
+    // make sure log.hflush() calls syncFs() to open a pipeline
+    TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
+    // lower the namenode & datanode heartbeat so the namenode
+    // quickly detects datanode failures
+    TEST_UTIL.getConfiguration().setInt("heartbeat.recheck.interval", 5000);
+    TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
+    // the namenode might still try to choose the recently-dead datanode
+    // for a pipeline, so try to a new pipeline multiple times
+    TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 10);
+    // set periodic sync to 2 min so it doesn't run during test
+    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.optionallogflushinterval",
+        120 * 1000);
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    TEST_UTIL.startMiniCluster(2);
+
+    cluster = TEST_UTIL.getHBaseCluster();
+    dfsCluster = TEST_UTIL.getDFSCluster();
+    admin = TEST_UTIL.getHBaseAdmin();
+
+    // disable region rebalancing (interferes with log watching)
+    cluster.getMaster().balanceSwitch(false);
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Tests that RegionServer aborts if we hit an error closing the WAL when
+   * there are unsynced WAL edits.  See HBASE-4282.
+   */
+  @Test
+  public void testRSAbortWithUnflushedEdits() throws Exception {
+    LOG.info("Starting testRSAbortWithUnflushedEdits()");
+
+    // When the META table can be opened, the region servers are running
+    new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
+
+    // Create the test table and open it
+    String tableName = this.getClass().getSimpleName();
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
+    desc.setDeferredLogFlush(true);
+
+    admin.createTable(desc);
+    HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+
+    HRegionServer server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName));
+    HLog log = server.getWAL();
+
+    assertTrue("Need HDFS-826 for this test", log.canGetCurReplicas());
+    // don't run this test without append support (HDFS-200 & HDFS-142)
+    assertTrue("Need append support for this test",
+        FSUtils.isAppendSupported(TEST_UTIL.getConfiguration()));
+
+    Put p = new Put(Bytes.toBytes("row2001"));
+    p.add(HConstants.CATALOG_FAMILY, Bytes.toBytes("col"), Bytes.toBytes(2001));
+    table.put(p);
+
+    log.sync();
+    assertEquals("Last deferred edit should have been cleared by sync()",
+        log.getLastDeferredSeq(), 0);
+
+    p = new Put(Bytes.toBytes("row2002"));
+    p.add(HConstants.CATALOG_FAMILY, Bytes.toBytes("col"), Bytes.toBytes(2002));
+    table.put(p);
+
+    dfsCluster.restartDataNodes();
+    LOG.info("Restarted datanodes");
+
+    assertTrue("Should have an outstanding WAL edit",
+        log.getLastDeferredSeq() > 0);
+    try {
+      log.rollWriter(true);
+      fail("Log roll should have triggered FailedLogCloseException");
+    } catch (FailedLogCloseException flce) {
+      assertTrue("Should have deferred flush log edits outstanding",
+          log.getLastDeferredSeq() > 0);
+    }
+  }
+}



Mime
View raw message