hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r991397 [14/15] - in /hbase/trunk: ./ bin/ conf/ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/avro/ src/main/java/org/apache/hadoop/hbase/catalog/ src/main/java/org/apache/hadoop/hbase/client/ src/main/java/o...
Date Tue, 31 Aug 2010 23:51:50 GMT
Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java?rev=991397&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java Tue Aug 31 23:51:44 2010
@@ -0,0 +1,379 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestLoadBalancer {
+  private static final Log LOG = LogFactory.getLog(TestLoadBalancer.class);
+
+  private static LoadBalancer loadBalancer;
+
+  private static Random rand;
+
+  @BeforeClass
+  public static void beforeAllTests() throws Exception {
+    loadBalancer = new LoadBalancer();
+    rand = new Random();
+  }
+
+  // int[testnum][servernumber] -> numregions
+  int [][] clusterStateMocks = new int [][] {
+      // 1 node
+      new int [] { 0 },
+      new int [] { 1 },
+      new int [] { 10 },
+      // 2 node
+      new int [] { 0, 0 },
+      new int [] { 2, 0 },
+      new int [] { 2, 1 },
+      new int [] { 2, 2 },
+      new int [] { 2, 3 },
+      new int [] { 2, 4 },
+      new int [] { 1, 1 },
+      new int [] { 0, 1 },
+      new int [] { 10, 1 },
+      new int [] { 14, 1432 },
+      new int [] { 47, 53 },
+      // 3 node
+      new int [] { 0, 1, 2 },
+      new int [] { 1, 2, 3 },
+      new int [] { 0, 2, 2 },
+      new int [] { 0, 3, 0 },
+      new int [] { 0, 4, 0 },
+      new int [] { 20, 20, 0 },
+      // 4 node
+      new int [] { 0, 1, 2, 3 },
+      new int [] { 4, 0, 0, 0 },
+      new int [] { 5, 0, 0, 0 },
+      new int [] { 6, 6, 0, 0 },
+      new int [] { 6, 2, 0, 0 },
+      new int [] { 6, 1, 0, 0 },
+      new int [] { 6, 0, 0, 0 },
+      new int [] { 4, 4, 4, 7 },
+      new int [] { 4, 4, 4, 8 },
+      new int [] { 0, 0, 0, 7 },
+      // 5 node
+      new int [] { 1, 1, 1, 1, 4 },
+      // more nodes
+      new int [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 },
+      new int [] { 6, 6, 5, 6, 6, 6, 6, 6, 6, 1 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 },
+      new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 8 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 10 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 123 },
+      new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 155 },
+      new int [] { 0, 0, 144, 1, 1, 1, 1, 1123, 133, 138, 12, 1444 },
+      new int [] { 0, 0, 144, 1, 0, 4, 1, 1123, 133, 138, 12, 1444 }
+  };
+
+  int [][] regionsAndServersMocks = new int [][] {
+      // { num regions, num servers }
+      new int [] { 0, 0 },
+      new int [] { 0, 1 },
+      new int [] { 1, 1 },
+      new int [] { 2, 1 },
+      new int [] { 10, 1 },
+      new int [] { 1, 2 },
+      new int [] { 2, 2 },
+      new int [] { 3, 2 },
+      new int [] { 1, 3 },
+      new int [] { 2, 3 },
+      new int [] { 3, 3 },
+      new int [] { 25, 3 },
+      new int [] { 2, 10 },
+      new int [] { 2, 100 },
+      new int [] { 12, 10 },
+      new int [] { 12, 100 },
+  };
+
+  /**
+   * Test the load balancing algorithm.
+   *
+   * Invariant is that all servers should be hosting either
+   * floor(average) or ceiling(average)
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testBalanceCluster() throws Exception {
+
+    for(int [] mockCluster : clusterStateMocks) {
+      Map<HServerInfo,List<HRegionInfo>> servers = mockClusterServers(mockCluster);
+      LOG.info("Mock Cluster : " + printMock(servers) + " " + printStats(servers));
+      List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
+      List<HServerInfo> balancedCluster = reconcile(servers, plans);
+      LOG.info("Mock Balance : " + printMock(balancedCluster));
+      assertClusterAsBalanced(balancedCluster);
+      for(Map.Entry<HServerInfo, List<HRegionInfo>> entry : servers.entrySet()) {
+        returnRegions(entry.getValue());
+        returnServer(entry.getKey());
+      }
+    }
+
+  }
+
+  /**
+   * Invariant is that all servers have between floor(avg) and ceiling(avg)
+   * number of regions.
+   */
+  public void assertClusterAsBalanced(List<HServerInfo> servers) {
+    int numServers = servers.size();
+    int numRegions = 0;
+    int maxRegions = 0;
+    int minRegions = Integer.MAX_VALUE;
+    for(HServerInfo server : servers) {
+      int nr = server.getLoad().getNumberOfRegions();
+      if(nr > maxRegions) {
+        maxRegions = nr;
+      }
+      if(nr < minRegions) {
+        minRegions = nr;
+      }
+      numRegions += nr;
+    }
+    if(maxRegions - minRegions < 2) {
+      // less than 2 between max and min, can't balance
+      return;
+    }
+    int min = numRegions / numServers;
+    int max = numRegions % numServers == 0 ? min : min + 1;
+
+    for(HServerInfo server : servers) {
+      assertTrue(server.getLoad().getNumberOfRegions() <= max);
+      assertTrue(server.getLoad().getNumberOfRegions() >= min);
+    }
+  }
+
+  /**
+   * Tests immediate assignment.
+   *
+   * Invariant is that all regions have an assignment.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testImmediateAssignment() throws Exception {
+    for(int [] mock : regionsAndServersMocks) {
+      LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
+      List<HRegionInfo> regions = randomRegions(mock[0]);
+      List<HServerInfo> servers = randomServers(mock[1], 0);
+      Map<HRegionInfo,HServerInfo> assignments =
+        LoadBalancer.immediateAssignment(regions, servers);
+      assertImmediateAssignment(regions, servers, assignments);
+      returnRegions(regions);
+      returnServers(servers);
+    }
+  }
+
+  /**
+   * All regions have an assignment.
+   * @param regions
+   * @param servers
+   * @param assignments
+   */
+  private void assertImmediateAssignment(List<HRegionInfo> regions,
+      List<HServerInfo> servers, Map<HRegionInfo,HServerInfo> assignments) {
+    for(HRegionInfo region : regions) {
+      assertTrue(assignments.containsKey(region));
+    }
+  }
+
+  /**
+   * Tests the bulk assignment used during cluster startup.
+   *
+   * Round-robin.  Should yield a balanced cluster so same invariant as the load
+   * balancer holds, all servers holding either floor(avg) or ceiling(avg).
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testBulkAssignment() throws Exception {
+    for(int [] mock : regionsAndServersMocks) {
+      LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
+      List<HRegionInfo> regions = randomRegions(mock[0]);
+      List<HServerInfo> servers = randomServers(mock[1], 0);
+      Map<HServerInfo,List<HRegionInfo>> assignments =
+        LoadBalancer.bulkAssignment(regions, servers);
+      float average = (float)regions.size()/servers.size();
+      int min = (int)Math.floor(average);
+      int max = (int)Math.ceil(average);
+      if(assignments != null && !assignments.isEmpty()) {
+        for(List<HRegionInfo> regionList : assignments.values()) {
+          assertTrue(regionList.size() == min || regionList.size() == max);
+        }
+      }
+      returnRegions(regions);
+      returnServers(servers);
+    }
+  }
+
+  private String printStats(Map<HServerInfo, List<HRegionInfo>> servers) {
+    int numServers = servers.size();
+    int totalRegions = 0;
+    for(HServerInfo server : servers.keySet()) {
+      totalRegions += server.getLoad().getNumberOfRegions();
+    }
+    float average = (float)totalRegions / numServers;
+    int max = (int)Math.ceil(average);
+    int min = (int)Math.floor(average);
+    return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max + " min=" + min + "]";
+  }
+
+  private String printMock(Map<HServerInfo, List<HRegionInfo>> servers) {
+    return printMock(Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()])));
+  }
+
+  private String printMock(List<HServerInfo> balancedCluster) {
+    SortedSet<HServerInfo> sorted = new TreeSet<HServerInfo>(balancedCluster);
+    HServerInfo [] arr = sorted.toArray(new HServerInfo[sorted.size()]);
+    StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4);
+    sb.append("{ ");
+    for(int i=0;i<arr.length;i++) {
+      if(i != 0) {
+        sb.append(" , ");
+      }
+      sb.append(arr[i].getLoad().getNumberOfRegions());
+    }
+    sb.append(" }");
+    return sb.toString();
+  }
+
+  /**
+   * This assumes the RegionPlan HSI instances are the same ones in the map, so
+   * actually no need to even pass in the map, but I think it's clearer.
+   * @param servers
+   * @param plans
+   * @return
+   */
+  private List<HServerInfo> reconcile(
+      Map<HServerInfo, List<HRegionInfo>> servers, List<RegionPlan> plans) {
+    if(plans != null) {
+      for(RegionPlan plan : plans) {
+        plan.getSource().getLoad().setNumberOfRegions(
+            plan.getSource().getLoad().getNumberOfRegions() - 1);
+        plan.getDestination().getLoad().setNumberOfRegions(
+            plan.getDestination().getLoad().getNumberOfRegions() + 1);
+      }
+    }
+    return Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()]));
+  }
+
+  private Map<HServerInfo, List<HRegionInfo>> mockClusterServers(
+      int [] mockCluster) {
+    int numServers = mockCluster.length;
+    Map<HServerInfo,List<HRegionInfo>> servers =
+      new TreeMap<HServerInfo,List<HRegionInfo>>();
+    for(int i=0;i<numServers;i++) {
+      int numRegions = mockCluster[i];
+      HServerInfo server = randomServer(numRegions);
+      List<HRegionInfo> regions = randomRegions(numRegions);
+      servers.put(server, regions);
+    }
+    return servers;
+  }
+
+  private Queue<HRegionInfo> regionQueue = new LinkedList<HRegionInfo>();
+
+  private List<HRegionInfo> randomRegions(int numRegions) {
+    List<HRegionInfo> regions = new ArrayList<HRegionInfo>(numRegions);
+    byte [] start = new byte[16];
+    byte [] end = new byte[16];
+    rand.nextBytes(start);
+    rand.nextBytes(end);
+    for(int i=0;i<numRegions;i++) {
+      if(!regionQueue.isEmpty()) {
+        regions.add(regionQueue.poll());
+        continue;
+      }
+      Bytes.putInt(start, 0, numRegions << 1);
+      Bytes.putInt(end, 0, (numRegions << 1) + 1);
+      HRegionInfo hri = new HRegionInfo(
+          new HTableDescriptor(Bytes.toBytes("table")), start, end);
+      regions.add(hri);
+    }
+    return regions;
+  }
+
+  private void returnRegions(List<HRegionInfo> regions) {
+    regionQueue.addAll(regions);
+  }
+
+  private Queue<HServerInfo> serverQueue = new LinkedList<HServerInfo>();
+
+  private HServerInfo randomServer(int numRegions) {
+    if(!serverQueue.isEmpty()) {
+      HServerInfo server = this.serverQueue.poll();
+      server.getLoad().setNumberOfRegions(numRegions);
+      return server;
+    }
+    String host = RandomStringUtils.random(16);
+    int port = rand.nextInt(60000);
+    long startCode = rand.nextLong();
+    HServerInfo hsi =
+      new HServerInfo(new HServerAddress(host, port), startCode, port, host);
+    hsi.getLoad().setNumberOfRegions(numRegions);
+    return hsi;
+  }
+
+  private List<HServerInfo> randomServers(int numServers, int numRegionsPerServer) {
+    List<HServerInfo> servers = new ArrayList<HServerInfo>(numServers);
+    for(int i=0;i<numServers;i++) {
+      servers.add(randomServer(numRegionsPerServer));
+    }
+    return servers;
+  }
+
+  private void returnServer(HServerInfo server) {
+    serverQueue.add(server);
+  }
+
+  private void returnServers(List<HServerInfo> servers) {
+    serverQueue.addAll(servers);
+  }
+}

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java Tue Aug 31 23:51:44 2010
@@ -21,30 +21,27 @@ package org.apache.hadoop.hbase.master;
 
 import static org.junit.Assert.assertEquals;
 
+import java.net.URLEncoder;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.conf.Configuration;
-
-import java.net.URLEncoder;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 public class TestLogsCleaner {
 
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
-  private ReplicationZookeeperWrapper zkHelper;
+  private ReplicationZookeeper zkHelper;
 
   /**
    * @throws java.lang.Exception
@@ -68,9 +65,11 @@ public class TestLogsCleaner {
   @Before
   public void setUp() throws Exception {
     Configuration conf = TEST_UTIL.getConfiguration();
+    /* TODO REENABLE
     zkHelper = new ReplicationZookeeperWrapper(
         ZooKeeperWrapper.createInstance(conf, HRegionServer.class.getName()),
         conf, new AtomicBoolean(true), "test-cluster");
+        */
   }
 
   /**
@@ -83,13 +82,25 @@ public class TestLogsCleaner {
   @Test
   public void testLogCleaning() throws Exception{
     Configuration c = TEST_UTIL.getConfiguration();
-    Path oldLogDir = new Path(TEST_UTIL.getTestDir(),
+    Path oldLogDir = new Path(HBaseTestingUtility.getTestDir(),
         HConstants.HREGION_OLDLOGDIR_NAME);
     String fakeMachineName = URLEncoder.encode("regionserver:60020", "UTF8");
 
     FileSystem fs = FileSystem.get(c);
-    AtomicBoolean stop = new AtomicBoolean(false);
-    LogsCleaner cleaner = new LogsCleaner(1000, stop,c, fs, oldLogDir);
+    Stoppable stoppable = new Stoppable() {
+      private volatile boolean stopped = false;
+
+      @Override
+      public void stop(String why) {
+        this.stopped = true;
+      }
+
+      @Override
+      public boolean isStopped() {
+        return this.stopped;
+      }
+    };
+    LogCleaner cleaner  = new LogCleaner(1000, stoppable, c, fs, oldLogDir);
 
     // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
     long now = System.currentTimeMillis();
@@ -113,7 +124,7 @@ public class TestLogsCleaner {
       // (TimeToLiveLogCleaner) but would be rejected by the second
       // (ReplicationLogCleaner)
       if (i % (30/3) == 0) {
-        zkHelper.addLogToList(fileName.getName(), fakeMachineName);
+// REENABLE        zkHelper.addLogToList(fileName.getName(), fakeMachineName);
         System.out.println("Replication log file: " + fileName);
       }
     }
@@ -144,5 +155,4 @@ public class TestLogsCleaner {
       System.out.println("Keeped log files: " + file.getPath().getName());
     }
   }
-
-}
+}
\ No newline at end of file

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java Tue Aug 31 23:51:44 2010
@@ -23,16 +23,15 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.HMsg;
-import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventHandlerListener;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -51,7 +50,7 @@ import static org.junit.Assert.*;
 
 public class TestMaster {
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private static final Log LOG = LogFactory.getLog(TestMasterWithDisabling.class);
+  private static final Log LOG = LogFactory.getLog(TestMaster.class);
   private static final byte[] TABLENAME = Bytes.toBytes("TestMaster");
   private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
 
@@ -73,10 +72,12 @@ public class TestMaster {
     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
 
     TEST_UTIL.createTable(TABLENAME, FAMILYNAME);
-    TEST_UTIL.loadTable(new HTable(TABLENAME), FAMILYNAME);
+    TEST_UTIL.loadTable(new HTable(TEST_UTIL.getConfiguration(), TABLENAME),
+      FAMILYNAME);
 
     List<Pair<HRegionInfo, HServerAddress>> tableRegions =
-      m.getTableRegions(TABLENAME);
+      MetaReader.getTableRegionsAndLocations(m.getCatalogTracker(),
+          Bytes.toString(TABLENAME));
     LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
     assertEquals(1, tableRegions.size());
     assertArrayEquals(HConstants.EMPTY_START_ROW,
@@ -85,11 +86,12 @@ public class TestMaster {
         tableRegions.get(0).getFirst().getEndKey());
 
     // Now trigger a split and stop when the split is in progress
-    
+
     CountDownLatch aboutToOpen = new CountDownLatch(1);
     CountDownLatch proceed = new CountDownLatch(1);
     RegionOpenListener list = new RegionOpenListener(aboutToOpen, proceed);
-    HBaseEventHandler.registerListener(list);
+    cluster.getMaster().executorService.
+      registerListener(EventType.RS2ZK_REGION_OPENED, list);
 
     LOG.info("Splitting table");
     admin.split(TABLENAME);
@@ -97,7 +99,9 @@ public class TestMaster {
     aboutToOpen.await(60, TimeUnit.SECONDS);
     try {
       LOG.info("Making sure we can call getTableRegions while opening");
-      tableRegions = m.getTableRegions(TABLENAME);
+      tableRegions = MetaReader.getTableRegionsAndLocations(
+          m.getCatalogTracker(), Bytes.toString(TABLENAME));
+
       LOG.info("Regions: " + Joiner.on(',').join(tableRegions));
       // We have three regions because one is split-in-progress
       assertEquals(3, tableRegions.size());
@@ -105,14 +109,16 @@ public class TestMaster {
       Pair<HRegionInfo,HServerAddress> pair =
         m.getTableRegionForRow(TABLENAME, Bytes.toBytes("cde"));
       LOG.info("Result is: " + pair);
-      Pair<HRegionInfo, HServerAddress> tableRegionFromName = m.getTableRegionFromName(pair.getFirst().getRegionName());
+      Pair<HRegionInfo, HServerAddress> tableRegionFromName =
+        MetaReader.getRegion(m.getCatalogTracker(),
+            pair.getFirst().getRegionName());
       assertEquals(tableRegionFromName.getFirst(), pair.getFirst());
     } finally {
       proceed.countDown();
     }
   }
 
-  static class RegionOpenListener implements HBaseEventHandlerListener {
+  static class RegionOpenListener implements EventHandlerListener {
     CountDownLatch aboutToOpen, proceed;
 
     public RegionOpenListener(CountDownLatch aboutToOpen, CountDownLatch proceed)
@@ -122,8 +128,8 @@ public class TestMaster {
     }
 
     @Override
-    public void afterProcess(HBaseEventHandler event) {
-      if (event.getHBEvent() != HBaseEventType.RS2ZK_REGION_OPENED) {
+    public void afterProcess(EventHandler event) {
+      if (event.getEventType() != EventType.RS2ZK_REGION_OPENED) {
         return;
       }
       try {
@@ -136,8 +142,7 @@ public class TestMaster {
     }
 
     @Override
-    public void beforeProcess(HBaseEventHandler event) {
+    public void beforeProcess(EventHandler event) {
     }
   }
-
-}
+}
\ No newline at end of file

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java Tue Aug 31 23:51:44 2010
@@ -19,42 +19,25 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.net.BindException;
-import java.util.Collection;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArraySet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HMsg;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
-import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /**
@@ -104,6 +87,7 @@ public class TestMasterTransitions {
    * the requeuing,  send over a close of a region on 'otherServer' so it comes
    * into a master that has its meta region marked as offline.
    */
+  /*
   static class HBase2428Listener implements RegionServerOperationListener {
     // Map of what we've delayed so we don't do do repeated delays.
     private final Set<RegionServerOperation> postponed =
@@ -164,12 +148,13 @@ public class TestMasterTransitions {
       if (isWantedCloseOperation(op) != null) return;
       this.done = true;
     }
-
+*/
     /*
      * @param op
      * @return Null if not the wanted ProcessRegionClose, else <code>op</code>
      * cast as a ProcessRegionClose.
      */
+  /*
     private ProcessRegionClose isWantedCloseOperation(final RegionServerOperation op) {
       // Count every time we get a close operation.
       if (op instanceof ProcessRegionClose) {
@@ -198,14 +183,15 @@ public class TestMasterTransitions {
       return true;
     }
   }
-
+*/
   /**
    * In 2428, the meta region has just been set offline and then a close comes
    * in.
    * @see <a href="https://issues.apache.org/jira/browse/HBASE-2428">HBASE-2428</a> 
    */
-  @Test (timeout=300000) public void testRegionCloseWhenNoMetaHBase2428()
+  @Ignore @Test  (timeout=300000) public void testRegionCloseWhenNoMetaHBase2428()
   throws Exception {
+    /*
     LOG.info("Running testRegionCloseWhenNoMetaHBase2428");
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     final HMaster master = cluster.getMaster();
@@ -249,6 +235,7 @@ public class TestMasterTransitions {
       master.getRegionServerOperationQueue().
         unregisterRegionServerOperationListener(listener);
     }
+    */
   }
 
   /**
@@ -257,8 +244,9 @@ public class TestMasterTransitions {
    * If confusion between old and new, purportedly meta never comes back.  Test
    * that meta gets redeployed.
    */
-  @Test (timeout=300000) public void testAddingServerBeforeOldIsDead2413()
+  @Ignore @Test (timeout=300000) public void testAddingServerBeforeOldIsDead2413()
   throws IOException {
+    /*
     LOG.info("Running testAddingServerBeforeOldIsDead2413");
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     int count = count();
@@ -298,9 +286,9 @@ public class TestMasterTransitions {
     } finally {
       c.set(HConstants.REGIONSERVER_PORT, oldPort);
     }
+    */
   }
 
-
   /**
    * HBase2482 is about outstanding region openings.  If any are outstanding
    * when a regionserver goes down, then they'll never deploy.  They'll be
@@ -309,6 +297,7 @@ public class TestMasterTransitions {
    * then we kill it.  It also looks out for a close message on the victim
    * server because that signifies start of the fireworks.
    */
+  /*
   static class HBase2482Listener implements RegionServerOperationListener {
     private final HRegionServer victim;
     private boolean abortSent = false;
@@ -367,7 +356,7 @@ public class TestMasterTransitions {
       }
     }
   }
-
+*/
   /**
    * In 2482, a RS with an opening region on it dies.  The said region is then
    * stuck in the master's regions-in-transition and never leaves it.  This
@@ -382,8 +371,9 @@ public class TestMasterTransitions {
    * done.
    * @see <a href="https://issues.apache.org/jira/browse/HBASE-2482">HBASE-2482</a> 
    */
-  @Test (timeout=300000) public void testKillRSWithOpeningRegion2482()
+  @Ignore @Test (timeout=300000) public void testKillRSWithOpeningRegion2482()
   throws Exception {
+    /*
     LOG.info("Running testKillRSWithOpeningRegion2482");
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     if (cluster.getLiveRegionServerThreads().size() < 2) {
@@ -413,7 +403,7 @@ public class TestMasterTransitions {
       // After all closes, add blocking message before the region opens start to
       // come in.
       cluster.addMessageToSendRegionServer(hrs,
-        new HMsg(HMsg.Type.TESTING_MSG_BLOCK_RS));
+        new HMsg(HMsg.Type.TESTING_BLOCK_REGIONSERVER));
       // Wait till one of the above close messages has an effect before we start
       // wait on all regions back online.
       while (!listener.closed) Threads.sleep(100);
@@ -427,11 +417,13 @@ public class TestMasterTransitions {
       m.getRegionServerOperationQueue().
         unregisterRegionServerOperationListener(listener);
     }
+    */
   }
 
   /*
    * @return Count of all non-catalog regions on the designated server
    */
+/*
   private int closeAllNonCatalogRegions(final MiniHBaseCluster cluster,
     final MiniHBaseCluster.MiniHBaseClusterRegionServer hrs)
   throws IOException {
@@ -460,6 +452,7 @@ public class TestMasterTransitions {
    * @return Count of regions in meta table.
    * @throws IOException
    */
+  /*
   private static int countOfMetaRegions()
   throws IOException {
     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
@@ -477,7 +470,7 @@ public class TestMasterTransitions {
     s.close();
     return rows;
   }
-
+*/
   /*
    * Add to each of the regions in .META. a value.  Key is the startrow of the
    * region (except its 'aaa' for first region).  Actual value is the row name.

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java Tue Aug 31 23:51:44 2010
@@ -19,72 +19,112 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
-import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.junit.AfterClass;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class TestRestartCluster {
   private static final Log LOG = LogFactory.getLog(TestRestartCluster.class);
-  private static Configuration conf;
-  private static HBaseTestingUtility utility;
-  private static ZooKeeperWrapper zkWrapper;
+  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static ZooKeeperWatcher zooKeeper;
   private static final byte[] TABLENAME = Bytes.toBytes("master_transitions");
   private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a")};
-  
-  @BeforeClass public static void beforeAllTests() throws Exception {
-    conf = HBaseConfiguration.create();
-    utility = new HBaseTestingUtility(conf);
-  }
 
-  @AfterClass public static void afterAllTests() throws IOException {
-    utility.shutdownMiniCluster();
+
+  private static final byte [][] TABLES = new byte[][] {
+      Bytes.toBytes("restartTableOne"),
+      Bytes.toBytes("restartTableTwo"),
+      Bytes.toBytes("restartTableThree")
+  };
+  private static final byte [] FAMILY = Bytes.toBytes("family");
+
+  @Before public void setup() throws Exception {
   }
 
-  @Before public void setup() throws IOException {
+  @After public void teardown() throws IOException {
+    UTIL.shutdownMiniCluster();
   }
 
-  @Test (timeout=300000) public void testRestartClusterAfterKill()throws Exception {
-    utility.startMiniZKCluster();
-    zkWrapper = ZooKeeperWrapper.createInstance(conf, "cluster1");
+  @Test (timeout=300000) public void testRestartClusterAfterKill()
+  throws Exception {
+    UTIL.startMiniZKCluster();
+    zooKeeper = new ZooKeeperWatcher(UTIL.getConfiguration(), "cluster1", null);
 
     // create the unassigned region, throw up a region opened state for META
-    String unassignedZNode = zkWrapper.getRegionInTransitionZNode();
-    zkWrapper.createZNodeIfNotExists(unassignedZNode);
-    byte[] data = null;
-    HBaseEventType hbEventType = HBaseEventType.RS2ZK_REGION_OPENED;
-    try {
-      data = Writables.getBytes(new RegionTransitionEventData(hbEventType, HMaster.MASTER));
-    } catch (IOException e) {
-      LOG.error("Error creating event data for " + hbEventType, e);
-    }
-    zkWrapper.createOrUpdateUnassignedRegion(
-        HRegionInfo.ROOT_REGIONINFO.getEncodedName(), data);
-    zkWrapper.createOrUpdateUnassignedRegion(
-        HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), data);
-    LOG.debug("Created UNASSIGNED zNode for ROOT and META regions in state " + HBaseEventType.M2ZK_REGION_OFFLINE);
-    
+    String unassignedZNode = zooKeeper.assignmentZNode;
+    ZKUtil.createAndFailSilent(zooKeeper, unassignedZNode);
+
+    ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.ROOT_REGIONINFO,
+      HMaster.MASTER);
+
+    ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.FIRST_META_REGIONINFO,
+      HMaster.MASTER);
+
+    LOG.debug("Created UNASSIGNED zNode for ROOT and META regions in state " +
+        EventType.M2ZK_REGION_OFFLINE);
+
     // start the HB cluster
     LOG.info("Starting HBase cluster...");
-    utility.startMiniCluster(2);  
-    
-    utility.createTable(TABLENAME, FAMILIES);
+    UTIL.startMiniCluster(2);
+
+    UTIL.createTable(TABLENAME, FAMILIES);
     LOG.info("Created a table, waiting for table to be available...");
-    utility.waitTableAvailable(TABLENAME, 60*1000);
+    UTIL.waitTableAvailable(TABLENAME, 60*1000);
 
-    LOG.info("Master deleted unassgined region and started up successfully.");
+    LOG.info("Master deleted unassigned region and started up successfully.");
+  }
+
+  @Test (timeout=300000)
+  public void testClusterRestart() throws Exception {
+    UTIL.startMiniCluster(3);
+    LOG.info("\n\nCreating tables");
+    for(byte [] TABLE : TABLES) {
+      UTIL.createTable(TABLE, FAMILY);
+      UTIL.waitTableAvailable(TABLE, 30000);
+    }
+    List<HRegionInfo> allRegions =
+      MetaScanner.listAllRegions(UTIL.getConfiguration());
+    assertEquals(3, allRegions.size());
+
+    LOG.info("\n\nShutting down cluster");
+    UTIL.getHBaseCluster().shutdown();
+    UTIL.getHBaseCluster().join();
+
+    LOG.info("\n\nSleeping a bit");
+    Thread.sleep(2000);
+
+    LOG.info("\n\nStarting cluster the second time");
+    UTIL.restartHBaseCluster(3);
+
+    allRegions = MetaScanner.listAllRegions(UTIL.getConfiguration());
+    assertEquals(3, allRegions.size());
+
+    LOG.info("\n\nWaiting for tables to be available");
+    for(byte [] TABLE: TABLES) {
+      try {
+        UTIL.createTable(TABLE, FAMILY);
+        assertTrue("Able to create table that should already exist", false);
+      } catch(TableExistsException tee) {
+        LOG.info("Table already exists as expected");
+      }
+      UTIL.waitTableAvailable(TABLE, 30000);
+    }
   }
 }

Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java?rev=991397&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java Tue Aug 31 23:51:44 2010
@@ -0,0 +1,331 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.Writables;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test open and close of regions using zk.
+ */
+public class TestZKBasedOpenCloseRegion {
+  private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class);
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final String TABLENAME = "TestZKBasedOpenCloseRegion";
+  private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
+    Bytes.toBytes("b"), Bytes.toBytes("c")};
+
+  @BeforeClass public static void beforeAllTests() throws Exception {
+    Configuration c = TEST_UTIL.getConfiguration();
+    c.setBoolean("dfs.support.append", true);
+    c.setInt("hbase.regionserver.info.port", 0);
+    c.setInt("hbase.master.meta.thread.rescanfrequency", 5*1000);
+    TEST_UTIL.startMiniCluster(2);
+    TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
+    HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
+    int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
+    waitUntilAllRegionsAssigned(countOfRegions);
+    addToEachStartKey(countOfRegions);
+  }
+
+  @AfterClass public static void afterAllTests() throws IOException {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Before public void setup() throws IOException {
+    if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) {
+      // Need at least two servers.
+      LOG.info("Started new server=" +
+        TEST_UTIL.getHBaseCluster().startRegionServer());
+
+    }
+  }
+
+  /**
+   * Test we reopen a region once closed.
+   * @throws Exception
+   */
+  @Test (timeout=300000) public void testReOpenRegion()
+  throws Exception {
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    LOG.info("Number of region servers = " +
+      cluster.getLiveRegionServerThreads().size());
+
+    int rsIdx = 0;
+    HRegionServer regionServer =
+      TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
+    HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
+    LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
+
+    AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
+    AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
+
+    EventHandlerListener closeListener =
+      new ReopenEventListener(hri.getRegionNameAsString(),
+          closeEventProcessed, EventType.RS2ZK_REGION_CLOSED);
+    cluster.getMaster().executorService.
+      registerListener(EventType.RS2ZK_REGION_CLOSED, closeListener);
+
+    EventHandlerListener openListener =
+      new ReopenEventListener(hri.getRegionNameAsString(),
+          reopenEventProcessed, EventType.RS2ZK_REGION_OPENED);
+    cluster.getMaster().executorService.
+      registerListener(EventType.RS2ZK_REGION_OPENED, openListener);
+
+    LOG.info("Unassign " + hri.getRegionNameAsString());
+    cluster.getMaster().assignmentManager.unassign(hri);
+
+    while (!closeEventProcessed.get()) {
+      Threads.sleep(100);
+    }
+
+    while (!reopenEventProcessed.get()) {
+      Threads.sleep(100);
+    }
+
+    LOG.info("Done with testReOpenRegion");
+  }
+
+  private HRegionInfo getNonMetaRegion(final Collection<HRegionInfo> regions) {
+    HRegionInfo hri = null;
+    for (HRegionInfo i: regions) {
+      LOG.info(i.getRegionNameAsString());
+      if (!i.isMetaRegion()) {
+        hri = i;
+        break;
+      }
+    }
+    return hri;
+  }
+
+  public static class ReopenEventListener implements EventHandlerListener {
+    private static final Log LOG = LogFactory.getLog(ReopenEventListener.class);
+    String regionName;
+    AtomicBoolean eventProcessed;
+    EventType eventType;
+
+    public ReopenEventListener(String regionName,
+        AtomicBoolean eventProcessed, EventType eventType) {
+      this.regionName = regionName;
+      this.eventProcessed = eventProcessed;
+      this.eventType = eventType;
+    }
+
+    @Override
+    public void beforeProcess(EventHandler event) {
+      if(event.getEventType() == eventType) {
+        LOG.info("Received " + eventType + " and beginning to process it");
+      }
+    }
+
+    @Override
+    public void afterProcess(EventHandler event) {
+      LOG.info("afterProcess(" + event + ")");
+      if(event.getEventType() == eventType) {
+        LOG.info("Finished processing " + eventType);
+        String regionName = "";
+        if(eventType == EventType.RS2ZK_REGION_OPENED) {
+          TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
+          regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
+        } else if(eventType == EventType.RS2ZK_REGION_CLOSED) {
+          TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
+          regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
+        }
+        if(this.regionName.equals(regionName)) {
+          eventProcessed.set(true);
+        }
+        synchronized(eventProcessed) {
+          eventProcessed.notifyAll();
+        }
+      }
+    }
+  }
+
+  @Test (timeout=300000) public void testCloseRegion()
+  throws Exception {
+    LOG.info("Running testCloseRegion");
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size());
+
+    int rsIdx = 0;
+    HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
+    HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
+    LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
+
+    AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
+    EventHandlerListener listener =
+      new CloseRegionEventListener(hri.getRegionNameAsString(),
+          closeEventProcessed);
+    cluster.getMaster().executorService.registerListener(EventType.RS2ZK_REGION_CLOSED, listener);
+
+    cluster.getMaster().assignmentManager.unassign(hri);
+
+    while (!closeEventProcessed.get()) {
+      Threads.sleep(100);
+    }
+    LOG.info("Done with testCloseRegion");
+  }
+
+  public static class CloseRegionEventListener implements EventHandlerListener {
+    private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class);
+    String regionToClose;
+    AtomicBoolean closeEventProcessed;
+
+    public CloseRegionEventListener(String regionToClose,
+        AtomicBoolean closeEventProcessed) {
+      this.regionToClose = regionToClose;
+      this.closeEventProcessed = closeEventProcessed;
+    }
+
+    @Override
+    public void afterProcess(EventHandler event) {
+      LOG.info("afterProcess(" + event + ")");
+      if(event.getEventType() == EventType.RS2ZK_REGION_CLOSED) {
+        LOG.info("Finished processing CLOSE REGION");
+        TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
+        if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) {
+          LOG.info("Setting closeEventProcessed flag");
+          closeEventProcessed.set(true);
+        } else {
+          LOG.info("Region to close didn't match");
+        }
+      }
+    }
+
+    @Override
+    public void beforeProcess(EventHandler event) {
+      if(event.getEventType() == EventType.M2RS_CLOSE_REGION) {
+        LOG.info("Received CLOSE RPC and beginning to process it");
+      }
+    }
+  }
+
+  private static void waitUntilAllRegionsAssigned(final int countOfRegions)
+  throws IOException {
+    HTable meta = new HTable(TEST_UTIL.getConfiguration(),
+      HConstants.META_TABLE_NAME);
+    while (true) {
+      int rows = 0;
+      Scan scan = new Scan();
+      scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+      ResultScanner s = meta.getScanner(scan);
+      for (Result r = null; (r = s.next()) != null;) {
+        byte [] b =
+          r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+        if (b == null || b.length <= 0) {
+          break;
+        }
+        rows++;
+      }
+      s.close();
+      // If I get to here and all rows have a Server, then all have been assigned.
+      if (rows == countOfRegions) {
+        break;
+      }
+      LOG.info("Found=" + rows);
+      Threads.sleep(1000);
+    }
+  }
+
+  /*
+   * Add to each of the regions in .META. a value.  Key is the startrow of the
+   * region (except its 'aaa' for first region).  Actual value is the row name.
+   * @param expected
+   * @return
+   * @throws IOException
+   */
+  private static int addToEachStartKey(final int expected) throws IOException {
+    HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
+    HTable meta = new HTable(TEST_UTIL.getConfiguration(),
+        HConstants.META_TABLE_NAME);
+    int rows = 0;
+    Scan scan = new Scan();
+    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+    ResultScanner s = meta.getScanner(scan);
+    for (Result r = null; (r = s.next()) != null;) {
+      byte [] b =
+        r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+      if (b == null || b.length <= 0) {
+        break;
+      }
+      HRegionInfo hri = Writables.getHRegionInfo(b);
+      // If start key, add 'aaa'.
+      byte [] row = getStartKey(hri);
+      Put p = new Put(row);
+      p.add(getTestFamily(), getTestQualifier(), row);
+      t.put(p);
+      rows++;
+    }
+    s.close();
+    Assert.assertEquals(expected, rows);
+    return rows;
+  }
+
+  private static byte [] getStartKey(final HRegionInfo hri) {
+    return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
+        Bytes.toBytes("aaa"): hri.getStartKey();
+  }
+
+  private static byte [] getTestFamily() {
+    return FAMILIES[0];
+  }
+
+  private static byte [] getTestQualifier() {
+    return getTestFamily();
+  }
+
+  public static void main(String args[]) throws Exception {
+    TestZKBasedOpenCloseRegion.beforeAllTests();
+
+    TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion();
+    test.setup();
+    test.testCloseRegion();
+
+    TestZKBasedOpenCloseRegion.afterAllTests();
+  }
+}

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java Tue Aug 31 23:51:44 2010
@@ -130,7 +130,7 @@ public class DisabledTestRegionServerExi
     int server = -1;
     for (int i = 0; i < regionThreads.size() && server == -1; i++) {
       HRegionServer s = regionThreads.get(i).getRegionServer();
-      Collection<HRegion> regions = s.getOnlineRegions();
+      Collection<HRegion> regions = s.getOnlineRegionsLocalContext();
       for (HRegion r : regions) {
         if (Bytes.equals(r.getTableDesc().getName(),
             HConstants.META_TABLE_NAME)) {

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java Tue Aug 31 23:51:44 2010
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.client.Pu
 public class OOMERegionServer extends HRegionServer {
   private List<Put> retainer = new ArrayList<Put>();
 
-  public OOMERegionServer(HBaseConfiguration conf) throws IOException {
+  public OOMERegionServer(HBaseConfiguration conf) throws IOException, InterruptedException {
     super(conf);
   }
 

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java Tue Aug 31 23:51:44 2010
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.HTableDes
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -161,8 +160,9 @@ public class TestFSErrorsExposed {
           fam, 1, HColumnDescriptor.DEFAULT_COMPRESSION,
           false, false, HConstants.FOREVER, "NONE"));
       admin.createTable(desc);
-
-      HTable table = new HTable(tableName);
+      // Make it fail faster.
+      util.getConfiguration().setInt("hbase.client.retries.number", 1);
+      HTable table = new HTable(util.getConfiguration(), tableName);
 
       // Load some data
       util.loadTable(table, fam);

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Tue Aug 31 23:51:44 2010
@@ -1316,7 +1316,7 @@ public class TestHRegion extends HBaseTe
     // the prepare call -- we are not ready to split just now.  Just return.
     if (!st.prepare()) return null;
     try {
-      result = st.execute(null);
+      result = st.execute(null, null);
     } catch (IOException ioe) {
       try {
         LOG.info("Running rollback of failed split of " +

Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java?rev=991397&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java Tue Aug 31 23:51:44 2010
@@ -0,0 +1,116 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.Semaphore;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.MasterAddressTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestMasterAddressManager {
+  private static final Log LOG = LogFactory.getLog(TestMasterAddressManager.class);
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniZKCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+  }
+  /**
+   * Unit tests that uses ZooKeeper but does not use the master-side methods
+   * but rather acts directly on ZK.
+   * @throws Exception
+   */
+  @Test
+  public void testMasterAddressManagerFromZK() throws Exception {
+
+    ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+        "testMasterAddressManagerFromZK", null);
+    ZKUtil.createAndFailSilent(zk, zk.baseZNode);
+
+    // Should not have a master yet
+    MasterAddressTracker addressManager = new MasterAddressTracker(zk, null);
+    addressManager.start();
+    assertFalse(addressManager.hasMaster());
+    zk.registerListener(addressManager);
+
+    // Use a listener to capture when the node is actually created
+    NodeCreationListener listener = new NodeCreationListener(zk, zk.masterAddressZNode);
+    zk.registerListener(listener);
+
+    // Create the master node with a dummy address
+    String host = "hostname";
+    int port = 1234;
+    HServerAddress dummyAddress = new HServerAddress(host, port);
+    LOG.info("Creating master node");
+    ZKUtil.setAddressAndWatch(zk, zk.masterAddressZNode, dummyAddress);
+
+    // Wait for the node to be created
+    LOG.info("Waiting for master address manager to be notified");
+    listener.waitForCreation();
+    LOG.info("Master node created");
+    assertTrue(addressManager.hasMaster());
+    HServerAddress pulledAddress = addressManager.getMasterAddress();
+    assertTrue(pulledAddress.equals(dummyAddress));
+
+  }
+
+  public static class NodeCreationListener extends ZooKeeperListener {
+    private static final Log LOG = LogFactory.getLog(NodeCreationListener.class);
+
+    private Semaphore lock;
+    private String node;
+
+    public NodeCreationListener(ZooKeeperWatcher watcher, String node) {
+      super(watcher);
+      lock = new Semaphore(0);
+      this.node = node;
+    }
+
+    @Override
+    public void nodeCreated(String path) {
+      if(path.equals(node)) {
+        LOG.debug("nodeCreated(" + path + ")");
+        lock.release();
+      }
+    }
+
+    public void waitForCreation() throws InterruptedException {
+      lock.acquire();
+    }
+  }
+}

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java Tue Aug 31 23:51:44 2010
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.HConstant
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -44,6 +45,7 @@ import org.apache.hadoop.hbase.util.Pair
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 /**
  * Test the {@link SplitTransaction} class against an HRegion (as opposed to
@@ -67,8 +69,9 @@ public class TestSplitTransaction {
     this.fs.delete(this.testdir, true);
     this.wal = new HLog(fs, new Path(this.testdir, "logs"),
       new Path(this.testdir, "archive"),
-      TEST_UTIL.getConfiguration(), null);
+      TEST_UTIL.getConfiguration());
     this.parent = createRegion(this.testdir, this.wal);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
   }
 
   @After public void teardown() throws IOException {
@@ -128,7 +131,9 @@ public class TestSplitTransaction {
     SplitTransaction st = prepareGOOD_SPLIT_ROW();
 
     // Run the execute.  Look at what it returns.
-    PairOfSameType<HRegion> daughters = st.execute(null);
+    Server mockServer = Mockito.mock(Server.class);
+    when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
+    PairOfSameType<HRegion> daughters = st.execute(mockServer, null);
     // Do some assertions about execution.
     assertTrue(this.fs.exists(st.getSplitDir()));
     // Assert the parent region is closed.
@@ -150,7 +155,7 @@ public class TestSplitTransaction {
     int daughtersRowCount = 0;
     for (HRegion r: daughters) {
       // Open so can count its content.
-      HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(), this.testdir,
+      HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(),
         r.getLog(), r.getConf());
       try {
         int count = countRows(openRegion);
@@ -174,12 +179,14 @@ public class TestSplitTransaction {
     // Start transaction.
     SplitTransaction st = prepareGOOD_SPLIT_ROW();
     SplitTransaction spiedUponSt = spy(st);
-    when(spiedUponSt.createDaughterRegion(spiedUponSt.getSecondDaughter())).
+    when(spiedUponSt.createDaughterRegion(spiedUponSt.getSecondDaughter(), null)).
       thenThrow(new MockedFailedDaughterCreation());
     // Run the execute.  Look at what it returns.
     boolean expectedException = false;
+    Server mockServer = Mockito.mock(Server.class);
+    when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
     try {
-      spiedUponSt.execute(null);
+      spiedUponSt.execute(mockServer, null);
     } catch (MockedFailedDaughterCreation e) {
       expectedException = true;
     }
@@ -198,12 +205,12 @@ public class TestSplitTransaction {
 
     // Now retry the split but do not throw an exception this time.
     assertTrue(st.prepare());
-    PairOfSameType<HRegion> daughters = st.execute(null);
+    PairOfSameType<HRegion> daughters = st.execute(mockServer, null);
     // Count rows.
     int daughtersRowCount = 0;
     for (HRegion r: daughters) {
       // Open so can count its content.
-      HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(), this.testdir,
+      HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(),
         r.getLog(), r.getConf());
       try {
         int count = countRows(openRegion);
@@ -248,6 +255,6 @@ public class TestSplitTransaction {
     HColumnDescriptor hcd = new HColumnDescriptor(CF);
     htd.addFamily(hcd);
     HRegionInfo hri = new HRegionInfo(htd, STARTROW, ENDROW);
-    return HRegion.openHRegion(hri, testdir, wal, TEST_UTIL.getConfiguration());
+    return HRegion.openHRegion(hri, wal, TEST_UTIL.getConfiguration());
   }
 }
\ No newline at end of file

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java Tue Aug 31 23:51:44 2010
@@ -123,7 +123,7 @@ public class TestStore extends TestCase 
     HTableDescriptor htd = new HTableDescriptor(table);
     htd.addFamily(hcd);
     HRegionInfo info = new HRegionInfo(htd, null, null, false);
-    HLog hlog = new HLog(fs, logdir, oldLogDir, conf, null);
+    HLog hlog = new HLog(fs, logdir, oldLogDir, conf);
     HRegion region = new HRegion(basedir, hlog, fs, conf, info, null);
 
     store = new Store(basedir, region, hcd, fs, conf);

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java Tue Aug 31 23:51:44 2010
@@ -29,7 +29,7 @@ public class InstrumentedSequenceFileLog
   @Override
     public void append(HLog.Entry entry) throws IOException {
       super.append(entry);
-      if (activateFailure && Bytes.equals(entry.getKey().getRegionName(), "break".getBytes())) {
+      if (activateFailure && Bytes.equals(entry.getKey().getEncodedRegionName(), "break".getBytes())) {
         System.out.println(getClass().getName() + ": I will throw an exception now...");
         throw(new IOException("This exception is instrumented and should only be thrown for testing"));
       }

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java Tue Aug 31 23:51:44 2010
@@ -133,7 +133,7 @@ public class TestHLog  {
     final byte [] tableName = Bytes.toBytes(getName());
     final byte [] rowName = tableName;
     Path logdir = new Path(dir, HConstants.HREGION_LOGDIR_NAME);
-    HLog log = new HLog(fs, logdir, oldLogDir, conf, null);
+    HLog log = new HLog(fs, logdir, oldLogDir, conf);
     final int howmany = 3;
     HRegionInfo[] infos = new HRegionInfo[3];
     for(int i = 0; i < howmany; i++) {
@@ -192,7 +192,7 @@ public class TestHLog  {
     out.close();
     in.close();
     Path subdir = new Path(dir, "hlogdir");
-    HLog wal = new HLog(fs, subdir, oldLogDir, conf, null);
+    HLog wal = new HLog(fs, subdir, oldLogDir, conf);
     final int total = 20;
 
     HRegionInfo info = new HRegionInfo(new HTableDescriptor(bytes),
@@ -295,7 +295,7 @@ public class TestHLog  {
         HLog.Entry entry = new HLog.Entry();
         while((entry = reader.next(entry)) != null) {
           HLogKey key = entry.getKey();
-          String region = Bytes.toString(key.getRegionName());
+          String region = Bytes.toString(key.getEncodedRegionName());
           // Assert that all edits are for same region.
           if (previousRegion != null) {
             assertEquals(previousRegion, region);
@@ -325,7 +325,7 @@ public class TestHLog  {
         HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
     Path subdir = new Path(dir, "hlogdir");
     Path archdir = new Path(dir, "hlogdir_archive");
-    HLog wal = new HLog(fs, subdir, archdir, conf, null);
+    HLog wal = new HLog(fs, subdir, archdir, conf);
     final int total = 20;
 
     for (int i = 0; i < total; i++) {
@@ -429,7 +429,7 @@ public class TestHLog  {
     final byte [] tableName = Bytes.toBytes("tablename");
     final byte [] row = Bytes.toBytes("row");
     HLog.Reader reader = null;
-    HLog log = new HLog(fs, dir, oldLogDir, conf, null);
+    HLog log = new HLog(fs, dir, oldLogDir, conf);
     try {
       // Write columns named 1, 2, 3, etc. and then values of single byte
       // 1, 2, 3...
@@ -442,10 +442,9 @@ public class TestHLog  {
       }
       HRegionInfo info = new HRegionInfo(new HTableDescriptor(tableName),
         row,Bytes.toBytes(Bytes.toString(row) + "1"), false);
-      final byte [] regionName = info.getRegionName();
       log.append(info, tableName, cols, System.currentTimeMillis());
       long logSeqId = log.startCacheFlush();
-      log.completeCacheFlush(regionName, tableName, logSeqId, info.isMetaRegion());
+      log.completeCacheFlush(info.getEncodedNameAsBytes(), tableName, logSeqId, info.isMetaRegion());
       log.close();
       Path filename = log.computeFilename();
       log = null;
@@ -458,7 +457,7 @@ public class TestHLog  {
         if (entry == null) break;
         HLogKey key = entry.getKey();
         WALEdit val = entry.getEdit();
-        assertTrue(Bytes.equals(regionName, key.getRegionName()));
+        assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName()));
         assertTrue(Bytes.equals(tableName, key.getTablename()));
         KeyValue kv = val.getKeyValues().get(0);
         assertTrue(Bytes.equals(row, kv.getRow()));
@@ -470,7 +469,7 @@ public class TestHLog  {
         HLogKey key = entry.getKey();
         WALEdit val = entry.getEdit();
         // Assert only one more row... the meta flushed row.
-        assertTrue(Bytes.equals(regionName, key.getRegionName()));
+        assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName()));
         assertTrue(Bytes.equals(tableName, key.getTablename()));
         KeyValue kv = val.getKeyValues().get(0);
         assertTrue(Bytes.equals(HLog.METAROW, kv.getRow()));
@@ -498,7 +497,7 @@ public class TestHLog  {
     final byte [] tableName = Bytes.toBytes("tablename");
     final byte [] row = Bytes.toBytes("row");
     Reader reader = null;
-    HLog log = new HLog(fs, dir, oldLogDir, conf, null);
+    HLog log = new HLog(fs, dir, oldLogDir, conf);
     try {
       // Write columns named 1, 2, 3, etc. and then values of single byte
       // 1, 2, 3...
@@ -513,7 +512,7 @@ public class TestHLog  {
           HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
       log.append(hri, tableName, cols, System.currentTimeMillis());
       long logSeqId = log.startCacheFlush();
-      log.completeCacheFlush(hri.getRegionName(), tableName, logSeqId, false);
+      log.completeCacheFlush(hri.getEncodedNameAsBytes(), tableName, logSeqId, false);
       log.close();
       Path filename = log.computeFilename();
       log = null;
@@ -524,7 +523,7 @@ public class TestHLog  {
       int idx = 0;
       for (KeyValue val : entry.getEdit().getKeyValues()) {
         assertTrue(Bytes.equals(hri.getRegionName(),
-          entry.getKey().getRegionName()));
+          entry.getKey().getEncodedRegionName()));
         assertTrue(Bytes.equals(tableName, entry.getKey().getTablename()));
         assertTrue(Bytes.equals(row, val.getRow()));
         assertEquals((byte)(idx + '0'), val.getValue()[0]);
@@ -537,7 +536,7 @@ public class TestHLog  {
       assertEquals(1, entry.getEdit().size());
       for (KeyValue val : entry.getEdit().getKeyValues()) {
         assertTrue(Bytes.equals(hri.getRegionName(),
-          entry.getKey().getRegionName()));
+          entry.getKey().getEncodedRegionName()));
         assertTrue(Bytes.equals(tableName, entry.getKey().getTablename()));
         assertTrue(Bytes.equals(HLog.METAROW, val.getRow()));
         assertTrue(Bytes.equals(HLog.METAFAMILY, val.getFamily()));
@@ -564,9 +563,9 @@ public class TestHLog  {
     final int COL_COUNT = 10;
     final byte [] tableName = Bytes.toBytes("tablename");
     final byte [] row = Bytes.toBytes("row");
-    HLog log = new HLog(fs, dir, oldLogDir, conf, null);
-    DumbLogEntriesVisitor visitor = new DumbLogEntriesVisitor();
-    log.addLogEntryVisitor(visitor);
+    HLog log = new HLog(fs, dir, oldLogDir, conf);
+    DumbWALObserver visitor = new DumbWALObserver();
+    log.registerWALActionsListener(visitor);
     long timestamp = System.currentTimeMillis();
     HRegionInfo hri = new HRegionInfo(new HTableDescriptor(tableName),
         HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
@@ -578,7 +577,7 @@ public class TestHLog  {
       log.append(hri, tableName, cols, System.currentTimeMillis());
     }
     assertEquals(COL_COUNT, visitor.increments);
-    log.removeLogEntryVisitor(visitor);
+    log.unregisterWALActionsListener(visitor);
     WALEdit cols = new WALEdit();
     cols.add(new KeyValue(row, Bytes.toBytes("column"),
         Bytes.toBytes(Integer.toString(11)),
@@ -587,8 +586,7 @@ public class TestHLog  {
     assertEquals(COL_COUNT, visitor.increments);
   }
 
-  static class DumbLogEntriesVisitor implements LogEntryVisitor {
-
+  static class DumbWALObserver implements WALObserver {
     int increments = 0;
 
     @Override
@@ -596,5 +594,17 @@ public class TestHLog  {
                                          WALEdit logEdit) {
       increments++;
     }
+
+    @Override
+    public void logRolled(Path newFile) {
+      // TODO Auto-generated method stub
+      
+    }
+
+    @Override
+    public void logRollRequested() {
+      // TODO Auto-generated method stub
+      
+    }
   }
 }

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java Tue Aug 31 23:51:44 2010
@@ -154,6 +154,8 @@ public class TestLogRolling  {
   private void startAndWriteData() throws IOException {
     // When the META table can be opened, the region servers are running
     new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
+    this.server = cluster.getRegionServerThreads().get(0).getRegionServer();
+    this.log = server.getWAL();
 
     // Create the test table and open it
     HTableDescriptor desc = new HTableDescriptor(tableName);
@@ -162,7 +164,7 @@ public class TestLogRolling  {
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
 
     server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName));
-    this.log = server.getLog();
+    this.log = server.getWAL();
     for (int i = 1; i <= 256; i++) {    // 256 writes should cause 8 log rolls
       Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", i)));
       put.add(HConstants.CATALOG_FAMILY, null, value);
@@ -192,7 +194,7 @@ public class TestLogRolling  {
       // flush all regions
 
       List<HRegion> regions =
-        new ArrayList<HRegion>(server.getOnlineRegions());
+        new ArrayList<HRegion>(server.getOnlineRegionsLocalContext());
       for (HRegion r: regions) {
         r.flushcache();
       }
@@ -226,7 +228,6 @@ public class TestLogRolling  {
   /**
    * Give me the HDFS pipeline for this log file
    */
-  @SuppressWarnings("null")
   DatanodeInfo[] getPipeline(HLog log) throws IllegalArgumentException,
       IllegalAccessException, InvocationTargetException {
     OutputStream stm = log.getOutputStream();
@@ -258,10 +259,15 @@ public class TestLogRolling  {
   public void testLogRollOnDatanodeDeath() throws IOException,
       InterruptedException, IllegalArgumentException, IllegalAccessException,
       InvocationTargetException {
-    assertTrue("This test requires HLog file replication.", fs
-        .getDefaultReplication() > 1);
+    assertTrue("This test requires HLog file replication.",
+      fs.getDefaultReplication() > 1);
+    LOG.info("Replication=" + fs.getDefaultReplication());
     // When the META table can be opened, the region servers are running
     new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
+
+    this.server = cluster.getRegionServer(0);
+    this.log = server.getWAL();
+    
     // Create the test table and open it
     String tableName = getName();
     HTableDescriptor desc = new HTableDescriptor(tableName);
@@ -275,7 +281,7 @@ public class TestLogRolling  {
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
 
     server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName));
-    this.log = server.getLog();
+    this.log = server.getWAL();
 
     assertTrue("Need HDFS-826 for this test", log.canGetCurReplicas());
     // don't run this test without append support (HDFS-200 & HDFS-142)
@@ -297,8 +303,7 @@ public class TestLogRolling  {
     assertTrue("Log should have a timestamp older than now",
         curTime > oldFilenum && oldFilenum != -1);
 
-    assertTrue("The log shouldn't have rolled yet", oldFilenum == log
-        .getFilenum());
+    assertTrue("The log shouldn't have rolled yet", oldFilenum == log.getFilenum());
     DatanodeInfo[] pipeline = getPipeline(log);
     assertTrue(pipeline.length == fs.getDefaultReplication());
 

Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java?rev=991397&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java Tue Aug 31 23:51:44 2010
@@ -0,0 +1,135 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * Test that the actions are called while playing with an HLog
+ */
+public class TestWALObserver {
+  protected static final Log LOG = LogFactory.getLog(TestWALObserver.class);
+
+  private final static HBaseTestingUtility TEST_UTIL =
+      new HBaseTestingUtility();
+
+  private final static byte[] SOME_BYTES =  Bytes.toBytes("t");
+  private static FileSystem fs;
+  private static Path oldLogDir;
+  private static Path logDir;
+  private static Configuration conf;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    conf = TEST_UTIL.getConfiguration();
+    conf.setInt("hbase.regionserver.maxlogs", 5);
+    fs = FileSystem.get(conf);
+    oldLogDir = new Path(HBaseTestingUtility.getTestDir(),
+        HConstants.HREGION_OLDLOGDIR_NAME);
+    logDir = new Path(HBaseTestingUtility.getTestDir(),
+        HConstants.HREGION_LOGDIR_NAME);
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    fs.delete(logDir, true);
+    fs.delete(oldLogDir, true);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    setUp();
+  }
+
+  /**
+   * Add a bunch of dummy data and roll the logs every two insert. We
+   * should end up with 10 rolled files (plus the roll called in
+   * the constructor). Also test adding a listener while it's running.
+   */
+  @Test
+  public void testActionListener() throws Exception {
+    DummyWALObserver observer = new DummyWALObserver();
+    List<WALObserver> list = new ArrayList<WALObserver>();
+    list.add(observer);
+    DummyWALObserver laterobserver = new DummyWALObserver();
+    HLog hlog = new HLog(fs, logDir, oldLogDir, conf, list, null);
+    HRegionInfo hri = new HRegionInfo(new HTableDescriptor(SOME_BYTES),
+        SOME_BYTES, SOME_BYTES, false);
+
+    for (int i = 0; i < 20; i++) {
+      byte[] b = Bytes.toBytes(i+"");
+      KeyValue kv = new KeyValue(b,b,b);
+      WALEdit edit = new WALEdit();
+      edit.add(kv);
+      HLogKey key = new HLogKey(b,b, 0, 0);
+      hlog.append(hri, key, edit);
+      if (i == 10) {
+        hlog.registerWALActionsListener(laterobserver);
+      }
+      if (i % 2 == 0) {
+        hlog.rollWriter();
+      }
+    }
+    assertEquals(11, observer.logRollCounter);
+    assertEquals(5, laterobserver.logRollCounter);
+  }
+
+  /**
+   * Just counts when methods are called
+   */
+  static class DummyWALObserver implements WALObserver {
+    public int logRollCounter = 0;
+
+    @Override
+    public void logRolled(Path newFile) {
+      logRollCounter++;
+    }
+
+    @Override
+    public void logRollRequested() {
+      // Not interested
+    }
+
+    @Override
+    public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
+        WALEdit logEdit) {
+      // Not interested
+      
+    }
+  }
+}
\ No newline at end of file

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java Tue Aug 31 23:51:44 2010
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.client.Ge
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.regionserver.FlushRequester;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -183,7 +184,7 @@ public class TestWALReplay {
     Path basedir = new Path(this.hbaseRootDir, tableNameStr);
     deleteDir(basedir);
     HLog wal = createWAL(this.conf);
-    HRegion region = HRegion.openHRegion(hri, basedir, wal, this.conf);
+    HRegion region = HRegion.openHRegion(hri, wal, this.conf);
     Path f =  new Path(basedir, "hfile");
     HFile.Writer writer = new HFile.Writer(this.fs, f);
     byte [] family = hri.getTableDesc().getFamilies().iterator().next().getName();
@@ -327,7 +328,7 @@ public class TestWALReplay {
     HLog wal = createWAL(this.conf);
     final byte[] tableName = Bytes.toBytes(tableNameStr);
     final byte[] rowName = tableName;
-    final byte[] regionName = hri.getRegionName();
+    final byte[] regionName = hri.getEncodedNameAsBytes();
 
     // Add 1k to each family.
     final int countPerFamily = 1000;
@@ -358,7 +359,6 @@ public class TestWALReplay {
     // Set down maximum recovery so we dfsclient doesn't linger retrying something
     // long gone.
     HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
-
     // Make a new conf and a new fs for the splitter to run on so we can take
     // over old wal.
     Configuration newConf = HBaseTestingUtility.setDifferentUser(this.conf,
@@ -396,6 +396,23 @@ public class TestWALReplay {
     }
   }
 
+  // Flusher used in this test.  Keep count of how often we are called and
+  // actually run the flush inside here.
+  class TestFlusher implements FlushRequester {
+    private int count = 0;
+    private HRegion r;
+
+    @Override
+    public void requestFlush(HRegion region) {
+      count++;
+      try {
+        r.flushcache();
+      } catch (IOException e) {
+        throw new RuntimeException("Exception flushing", e);
+      }
+    }
+  }
+
   private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
       final byte [] rowName, final byte [] family, 
       final int count, EnvironmentEdge ee, final HLog wal)
@@ -464,7 +481,7 @@ public class TestWALReplay {
    * @throws IOException
    */
   private HLog createWAL(final Configuration c) throws IOException {
-    HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c, null);
+    HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c);
     // Set down maximum recovery so we dfsclient doesn't linger retrying something
     // long gone.
     HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java?rev=991397&r1=991396&r2=991397&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java Tue Aug 31 23:51:44 2010
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.replicat
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface;
 import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager;
 
@@ -39,7 +40,7 @@ public class ReplicationSourceDummy impl
 
   @Override
   public void init(Configuration conf, FileSystem fs,
-                   ReplicationSourceManager manager, AtomicBoolean stopper,
+                   ReplicationSourceManager manager, Stoppable stopper,
                    AtomicBoolean replicating, String peerClusterId)
       throws IOException {
     this.manager = manager;



Mime
View raw message