hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r927978 - in /hadoop/hbase/branches/0.20: CHANGES.txt src/test/org/apache/hadoop/hbase/HBaseTestingUtility.java src/test/org/apache/hadoop/hbase/TestFullLogReconstruction.java
Date Fri, 26 Mar 2010 17:38:40 GMT
Author: stack
Date: Fri Mar 26 17:38:40 2010
New Revision: 927978

URL: http://svn.apache.org/viewvc?rev=927978&view=rev
Log:
HBASE-2340 Add end-to-end test of sync/flush

Added:
    hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestFullLogReconstruction.java
Modified:
    hadoop/hbase/branches/0.20/CHANGES.txt
    hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestingUtility.java

Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=927978&r1=927977&r2=927978&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Fri Mar 26 17:38:40 2010
@@ -79,6 +79,7 @@ Release 0.20.4 - Unreleased
                on every heartbeat (Karthik Ranganathan via Stack)
    HBASE-2327  [EC2] Allocate elastic IP addresses for ZK and master nodes
    HBASE-2147  run zookeeper in the same jvm as master during non-distributed mode
+   HBASE-2340  Add end-to-end test of sync/flush
 
   NEW FEATURES
    HBASE-2257  [stargate] multiuser mode

Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=927978&r1=927977&r2=927978&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestingUtility.java Fri
Mar 26 17:38:40 2010
@@ -442,7 +442,7 @@ public class HBaseTestingUtility {
    *
    * @throws IOException When removing the rows fails.
    */
-  private void emptyMetaTable() throws IOException {
+  public void emptyMetaTable() throws IOException {
     HTable t = new HTable(this.conf, HConstants.META_TABLE_NAME);
     ArrayList<Delete> deletes = new ArrayList<Delete>();
     ResultScanner s = t.getScanner(new Scan());
@@ -455,7 +455,7 @@ public class HBaseTestingUtility {
     s.close();
     t.delete(deletes);
   }
-  
+
   /**
    * Starts a <code>MiniMRCluster</code> with a default number of 
    * <code>TaskTracker</code>'s.
@@ -523,7 +523,22 @@ public class HBaseTestingUtility {
    * @throws Exception
    */
   public void expireRegionServerSession(int index) throws Exception {
+    expireRegionServerSession(index, true);
+  }
+
+  /**
+   * Expire a region server's session
+   * @param index which RS
+   * @param hdfsShutdown True if we run the RS shutdown hdfs thread.  Set it to
+   * false if you don't want an RS to shutdown the FS on it way out.  You might
+   * not want it too if lots of RSs all running in the one JVM context.
+   * @throws Exception
+   */
+  public void expireRegionServerSession(final int index,
+    final boolean hdfsShutdown)
+  throws Exception {
     HRegionServer rs = hbaseCluster.getRegionServer(index);
+    if (!hdfsShutdown) rs.setHDFSShutdownThreadOnExit(null);
     expireSession(rs.getZooKeeperWrapper());
   }
 

Added: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestFullLogReconstruction.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestFullLogReconstruction.java?rev=927978&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestFullLogReconstruction.java
(added)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestFullLogReconstruction.java
Fri Mar 26 17:38:40 2010
@@ -0,0 +1,128 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestFullLogReconstruction {
+
+  private final static HBaseTestingUtility
+      TEST_UTIL = new HBaseTestingUtility();
+
+  private final static byte[] TABLE_NAME = Bytes.toBytes("tabletest");
+  private final static byte[] FAMILY = Bytes.toBytes("family");
+
+    /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    HBaseConfiguration c = TEST_UTIL.getConfiguration();
+    c.setBoolean("dfs.support.append", true);
+    // quicker heartbeat interval for faster DN death notification
+    c.setInt("heartbeat.recheck.interval", 5000);
+    c.setInt("dfs.heartbeat.interval", 1);
+    c.setInt("dfs.socket.timeout", 5000);
+    // faster failover with cluster.shutdown();fs.close() idiom
+    c.setInt("ipc.client.connect.max.retries", 1);
+    c.setInt("dfs.client.block.recovery.retries", 1);
+    c.setInt("hbase.regionserver.flushlogentries", 1);
+    TEST_UTIL.startMiniCluster(2);
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @Before
+  public void setUp() throws Exception {
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @After
+  public void tearDown() throws Exception {
+  }
+
+  /**
+   * Test the whole reconstruction loop. Build a table with regions aaa to zzz
+   * and load every one of them multiple times with the same date and do a flush
+   * at some point. Kill one of the region servers and scan the table. We should
+   * see all the rows.
+   * @throws Exception
+   */
+  @Test
+  public void testReconstruction() throws Exception {
+
+    TEST_UTIL.createTable(TABLE_NAME, FAMILY);
+
+    HTable table = new HTable(TABLE_NAME);
+
+    TEST_UTIL.createMultiRegions(table, Bytes.toBytes("family"));
+
+    // Load up the table with simple rows and count them
+    int initialCount = TEST_UTIL.loadTable(table, FAMILY);
+    Scan scan = new Scan();
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (@SuppressWarnings("unused") Result res : results) {
+      count++;
+    }
+    results.close();
+
+    assertEquals(initialCount, count);
+
+    for(int i = 0; i < 4; i++) {
+      TEST_UTIL.loadTable(table, FAMILY);
+    }
+
+    // Expire the first RS.  Don't have it run its shutdown hdfs thread.
+    TEST_UTIL.expireRegionServerSession(0, false);
+
+    scan = new Scan();
+    results = table.getScanner(scan);
+    int newCount = 0;
+    for (@SuppressWarnings("unused") Result res : results) {
+      newCount++;
+    }
+    results.close();
+    assertEquals(count, newCount);
+  }
+}



Mime
View raw message