hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1303219 - /hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
Date Tue, 20 Mar 2012 23:53:30 GMT
Author: stack
Date: Tue Mar 20 23:53:30 2012
New Revision: 1303219

URL: http://svn.apache.org/viewvc?rev=1303219&view=rev
Log:
HBASE-5270 Handle potential data loss due to concurrent processing of processFaileOver and
ServerShutdownHandler

Added:
    hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java

Added: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java?rev=1303219&view=auto
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
(added)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java
Tue Mar 20 23:53:30 2012
@@ -0,0 +1,257 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.TestMasterFailover;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestRSKilledWhenMasterInitializing {
+  private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
+
+  private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
+  private static final int NUM_MASTERS = 1;
+  private static final int NUM_RS = 4;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    // Set it so that this test runs with my custom master
+    TESTUTIL.getConfiguration().setClass(HConstants.MASTER_IMPL,
+        TestingMaster.class, HMaster.class);
+    // Start up the cluster.
+    TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    if (!TESTUTIL.getHBaseCluster().getMaster().isInitialized()) {
+      // master is not initialized and is waiting something forever.
+      for (MasterThread mt : TESTUTIL.getHBaseCluster().getLiveMasterThreads()) {
+        mt.interrupt();
+      }
+    }
+    TESTUTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * An HMaster instance used in this test. If 'TestingMaster.sleep' is set in
+   * the Configuration, then we'll sleep after log is split and we'll also
+   * return a custom RegionServerTracker.
+   */
+  public static class TestingMaster extends HMaster {
+    private boolean logSplit = false;
+
+    public TestingMaster(Configuration conf) throws IOException,
+        KeeperException, InterruptedException {
+      super(conf);
+    }
+
+    @Override
+    protected void splitLogAfterStartup(MasterFileSystem mfs,
+        Set<ServerName> onlineServers) {
+      super.splitLogAfterStartup(mfs, onlineServers);
+      logSplit = true;
+      // If "TestingMaster.sleep" is set, sleep after log split.
+      if (getConfiguration().getBoolean("TestingMaster.sleep", false)) {
+        int duration = getConfiguration().getInt(
+            "TestingMaster.sleep.duration", 0);
+        Threads.sleep(duration);
+      }
+    }
+
+
+    public boolean isLogSplitAfterStartup() {
+      return logSplit;
+    }
+  }
+
+  @Test(timeout = 120000)
+  public void testCorrectnessWhenMasterFailOver() throws Exception {
+    final byte[] TABLENAME = Bytes.toBytes("testCorrectnessWhenMasterFailOver");
+    final byte[] FAMILY = Bytes.toBytes("family");
+    final byte[][] SPLITKEYS = { Bytes.toBytes("b"), Bytes.toBytes("i") };
+
+    MiniHBaseCluster cluster = TESTUTIL.getHBaseCluster();
+
+    HTableDescriptor desc = new HTableDescriptor(TABLENAME);
+    desc.addFamily(new HColumnDescriptor(FAMILY));
+    HBaseAdmin hbaseAdmin = TESTUTIL.getHBaseAdmin();
+    hbaseAdmin.createTable(desc, SPLITKEYS);
+
+    assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
+
+    HTable table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
+    List<Put> puts = new ArrayList<Put>();
+    Put put1 = new Put(Bytes.toBytes("a"));
+    put1.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
+    Put put2 = new Put(Bytes.toBytes("h"));
+    put2.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
+    Put put3 = new Put(Bytes.toBytes("o"));
+    put3.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value"));
+    puts.add(put1);
+    puts.add(put2);
+    puts.add(put3);
+    table.put(puts);
+    ResultScanner resultScanner = table.getScanner(new Scan());
+    int count = 0;
+    while (resultScanner.next() != null) {
+      count++;
+    }
+    resultScanner.close();
+    table.close();
+    assertEquals(3, count);
+
+    /* Starting test */
+    cluster.getConfiguration().setBoolean("TestingMaster.sleep", true);
+    cluster.getConfiguration().setInt("TestingMaster.sleep.duration", 10000);
+
+    /* NO.1 .META. region correctness */
+    // First abort master
+    abortMaster(cluster);
+    TestingMaster master = startMasterAndWaitUntilLogSplit(cluster);
+
+    // Second kill meta server
+    int metaServerNum = cluster.getServerWithMeta();
+    int rootServerNum = cluster.getServerWith(HRegionInfo.ROOT_REGIONINFO
+        .getRegionName());
+    HRegionServer metaRS = cluster.getRegionServer(metaServerNum);
+    LOG.debug("Killing metaRS and carryingRoot = "
+        + (metaServerNum == rootServerNum));
+    metaRS.kill();
+    metaRS.join();
+
+    /*
+     * Sleep double time of TestingMaster.sleep.duration, so we can ensure that
+     * master has already assigned ROOTandMETA or is blocking on assigning
+     * ROOTandMETA
+     */
+    Thread.sleep(10000 * 2);
+
+    waitUntilMasterIsInitialized(master);
+
+    // Third check whether data is correct in meta region
+    assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
+
+    /*
+     * NO.2 -ROOT- region correctness . If the .META. server killed in the NO.1
+     * is also carrying -ROOT- region, it is not needed
+     */
+    if (rootServerNum != metaServerNum) {
+      // First abort master
+      abortMaster(cluster);
+      master = startMasterAndWaitUntilLogSplit(cluster);
+
+      // Second kill meta server
+      HRegionServer rootRS = cluster.getRegionServer(rootServerNum);
+      LOG.debug("Killing rootRS");
+      rootRS.kill();
+      rootRS.join();
+
+      /*
+       * Sleep double time of TestingMaster.sleep.duration, so we can ensure
+       * that master has already assigned ROOTandMETA or is blocking on
+       * assigning ROOTandMETA
+       */
+      Thread.sleep(10000 * 2);
+      waitUntilMasterIsInitialized(master);
+
+      // Third check whether data is correct in meta region
+      assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
+    }
+
+    /* NO.3 data region correctness */
+    ServerManager serverManager = cluster.getMaster().getServerManager();
+    while (serverManager.areDeadServersInProgress()) {
+      Thread.sleep(100);
+    }
+    table = new HTable(TESTUTIL.getConfiguration(), TABLENAME);
+    resultScanner = table.getScanner(new Scan());
+    count = 0;
+    while (resultScanner.next() != null) {
+      count++;
+    }
+    resultScanner.close();
+    table.close();
+    assertEquals(3, count);
+  }
+
+  private void abortMaster(MiniHBaseCluster cluster)
+      throws InterruptedException {
+    for (MasterThread mt : cluster.getLiveMasterThreads()) {
+      if (mt.getMaster().isActiveMaster()) {
+        mt.getMaster().abort("Aborting for tests", new Exception("Trace info"));
+        mt.join();
+        break;
+      }
+    }
+    LOG.debug("Master is aborted");
+  }
+
+  private TestingMaster startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster)
+      throws IOException, InterruptedException {
+    TestingMaster master = (TestingMaster) cluster.startMaster().getMaster();
+    while (!master.isLogSplitAfterStartup()) {
+      Thread.sleep(100);
+    }
+    LOG.debug("splitted:" + master.isLogSplitAfterStartup() + ",initialized:"
+        + master.isInitialized());
+    return master;
+  }
+
+  private void waitUntilMasterIsInitialized(HMaster master)
+      throws InterruptedException {
+    while (!master.isInitialized()) {
+      Thread.sleep(100);
+    }
+    LOG.debug("master isInitialized");
+  }
+
+}



Mime
View raw message