Return-Path: Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: (qmail 40807 invoked from network); 31 Aug 2010 23:53:44 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 31 Aug 2010 23:53:44 -0000 Received: (qmail 16486 invoked by uid 500); 31 Aug 2010 23:53:44 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 16461 invoked by uid 500); 31 Aug 2010 23:53:44 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 16454 invoked by uid 99); 31 Aug 2010 23:53:44 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Aug 2010 23:53:44 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Aug 2010 23:53:41 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 77D492388BA6; Tue, 31 Aug 2010 23:51:53 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r991397 [14/15] - in /hbase/trunk: ./ bin/ conf/ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/avro/ src/main/java/org/apache/hadoop/hbase/catalog/ src/main/java/org/apache/hadoop/hbase/client/ src/main/java/o... Date: Tue, 31 Aug 2010 23:51:50 -0000 To: commits@hbase.apache.org From: stack@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100831235153.77D492388BA6@eris.apache.org> Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java?rev=991397&view=auto ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java (added) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java Tue Aug 31 23:51:44 2010 @@ -0,0 +1,379 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Random; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HServerInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestLoadBalancer { + private static final Log LOG = LogFactory.getLog(TestLoadBalancer.class); + + private static LoadBalancer loadBalancer; + + private static Random rand; + + @BeforeClass + public static void beforeAllTests() throws Exception { + loadBalancer = new LoadBalancer(); + rand = new Random(); + } + + // int[testnum][servernumber] -> numregions + int [][] clusterStateMocks = new int [][] { + // 1 node + new int [] { 0 }, + new int [] { 1 }, + new int [] { 10 }, + // 2 node + new int [] { 0, 0 }, + new int [] { 2, 0 }, + new int [] { 2, 1 }, + new int [] { 2, 2 }, + new int [] { 2, 3 }, + new int [] { 2, 4 }, + new int [] { 1, 1 }, + new int [] { 0, 1 }, + new int [] { 10, 1 }, + new int [] { 14, 1432 }, + new int [] { 47, 53 }, + // 3 node + new int [] { 0, 1, 2 }, + new int [] { 1, 2, 3 }, + new int [] { 0, 2, 2 }, + new int [] { 0, 3, 0 }, + new int [] { 0, 4, 0 }, + new int [] { 20, 20, 0 }, + // 4 node + new int [] { 0, 1, 2, 3 }, + new int [] { 4, 0, 0, 0 }, + new int [] { 5, 0, 0, 0 }, + new int [] { 6, 6, 0, 0 }, + new int [] { 6, 2, 0, 0 }, + new int [] { 6, 1, 0, 0 }, + new int [] { 6, 0, 0, 0 }, + new int [] { 4, 4, 4, 7 }, + new int [] { 4, 4, 4, 8 }, + new int [] { 0, 0, 0, 7 }, + // 5 node + new int [] { 1, 1, 1, 1, 4 }, + // more nodes + new int [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 }, + new int [] { 6, 6, 5, 6, 6, 6, 6, 6, 6, 1 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 8 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 10 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 123 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 155 }, + new int [] { 0, 0, 144, 1, 1, 1, 1, 1123, 133, 138, 12, 1444 }, + new int [] { 0, 0, 144, 1, 0, 4, 1, 1123, 133, 138, 12, 1444 } + }; + + int [][] regionsAndServersMocks = new int [][] { + // { num regions, num servers } + new int [] { 0, 0 }, + new int [] { 0, 1 }, + new int [] { 1, 1 }, + new int [] { 2, 1 }, + new int [] { 10, 1 }, + new int [] { 1, 2 }, + new int [] { 2, 2 }, + new int [] { 3, 2 }, + new int [] { 1, 3 }, + new int [] { 2, 3 }, + new int [] { 3, 3 }, + new int [] { 25, 3 }, + new int [] { 2, 10 }, + new int [] { 2, 100 }, + new int [] { 12, 10 }, + new int [] { 12, 100 }, + }; + + /** + * Test the load balancing algorithm. + * + * Invariant is that all servers should be hosting either + * floor(average) or ceiling(average) + * + * @throws Exception + */ + @Test + public void testBalanceCluster() throws Exception { + + for(int [] mockCluster : clusterStateMocks) { + Map> servers = mockClusterServers(mockCluster); + LOG.info("Mock Cluster : " + printMock(servers) + " " + printStats(servers)); + List plans = loadBalancer.balanceCluster(servers); + List balancedCluster = reconcile(servers, plans); + LOG.info("Mock Balance : " + printMock(balancedCluster)); + assertClusterAsBalanced(balancedCluster); + for(Map.Entry> entry : servers.entrySet()) { + returnRegions(entry.getValue()); + returnServer(entry.getKey()); + } + } + + } + + /** + * Invariant is that all servers have between floor(avg) and ceiling(avg) + * number of regions. + */ + public void assertClusterAsBalanced(List servers) { + int numServers = servers.size(); + int numRegions = 0; + int maxRegions = 0; + int minRegions = Integer.MAX_VALUE; + for(HServerInfo server : servers) { + int nr = server.getLoad().getNumberOfRegions(); + if(nr > maxRegions) { + maxRegions = nr; + } + if(nr < minRegions) { + minRegions = nr; + } + numRegions += nr; + } + if(maxRegions - minRegions < 2) { + // less than 2 between max and min, can't balance + return; + } + int min = numRegions / numServers; + int max = numRegions % numServers == 0 ? min : min + 1; + + for(HServerInfo server : servers) { + assertTrue(server.getLoad().getNumberOfRegions() <= max); + assertTrue(server.getLoad().getNumberOfRegions() >= min); + } + } + + /** + * Tests immediate assignment. + * + * Invariant is that all regions have an assignment. + * + * @throws Exception + */ + @Test + public void testImmediateAssignment() throws Exception { + for(int [] mock : regionsAndServersMocks) { + LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers"); + List regions = randomRegions(mock[0]); + List servers = randomServers(mock[1], 0); + Map assignments = + LoadBalancer.immediateAssignment(regions, servers); + assertImmediateAssignment(regions, servers, assignments); + returnRegions(regions); + returnServers(servers); + } + } + + /** + * All regions have an assignment. + * @param regions + * @param servers + * @param assignments + */ + private void assertImmediateAssignment(List regions, + List servers, Map assignments) { + for(HRegionInfo region : regions) { + assertTrue(assignments.containsKey(region)); + } + } + + /** + * Tests the bulk assignment used during cluster startup. + * + * Round-robin. Should yield a balanced cluster so same invariant as the load + * balancer holds, all servers holding either floor(avg) or ceiling(avg). + * + * @throws Exception + */ + @Test + public void testBulkAssignment() throws Exception { + for(int [] mock : regionsAndServersMocks) { + LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers"); + List regions = randomRegions(mock[0]); + List servers = randomServers(mock[1], 0); + Map> assignments = + LoadBalancer.bulkAssignment(regions, servers); + float average = (float)regions.size()/servers.size(); + int min = (int)Math.floor(average); + int max = (int)Math.ceil(average); + if(assignments != null && !assignments.isEmpty()) { + for(List regionList : assignments.values()) { + assertTrue(regionList.size() == min || regionList.size() == max); + } + } + returnRegions(regions); + returnServers(servers); + } + } + + private String printStats(Map> servers) { + int numServers = servers.size(); + int totalRegions = 0; + for(HServerInfo server : servers.keySet()) { + totalRegions += server.getLoad().getNumberOfRegions(); + } + float average = (float)totalRegions / numServers; + int max = (int)Math.ceil(average); + int min = (int)Math.floor(average); + return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max + " min=" + min + "]"; + } + + private String printMock(Map> servers) { + return printMock(Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()]))); + } + + private String printMock(List balancedCluster) { + SortedSet sorted = new TreeSet(balancedCluster); + HServerInfo [] arr = sorted.toArray(new HServerInfo[sorted.size()]); + StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4); + sb.append("{ "); + for(int i=0;i reconcile( + Map> servers, List plans) { + if(plans != null) { + for(RegionPlan plan : plans) { + plan.getSource().getLoad().setNumberOfRegions( + plan.getSource().getLoad().getNumberOfRegions() - 1); + plan.getDestination().getLoad().setNumberOfRegions( + plan.getDestination().getLoad().getNumberOfRegions() + 1); + } + } + return Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()])); + } + + private Map> mockClusterServers( + int [] mockCluster) { + int numServers = mockCluster.length; + Map> servers = + new TreeMap>(); + for(int i=0;i regions = randomRegions(numRegions); + servers.put(server, regions); + } + return servers; + } + + private Queue regionQueue = new LinkedList(); + + private List randomRegions(int numRegions) { + List regions = new ArrayList(numRegions); + byte [] start = new byte[16]; + byte [] end = new byte[16]; + rand.nextBytes(start); + rand.nextBytes(end); + for(int i=0;i regions) { + regionQueue.addAll(regions); + } + + private Queue serverQueue = new LinkedList(); + + private HServerInfo randomServer(int numRegions) { + if(!serverQueue.isEmpty()) { + HServerInfo server = this.serverQueue.poll(); + server.getLoad().setNumberOfRegions(numRegions); + return server; + } + String host = RandomStringUtils.random(16); + int port = rand.nextInt(60000); + long startCode = rand.nextLong(); + HServerInfo hsi = + new HServerInfo(new HServerAddress(host, port), startCode, port, host); + hsi.getLoad().setNumberOfRegions(numRegions); + return hsi; + } + + private List randomServers(int numServers, int numRegionsPerServer) { + List servers = new ArrayList(numServers); + for(int i=0;i servers) { + serverQueue.addAll(servers); + } +} Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java Tue Aug 31 23:51:44 2010 @@ -21,30 +21,27 @@ package org.apache.hadoop.hbase.master; import static org.junit.Assert.assertEquals; +import java.net.URLEncoder; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.replication.ReplicationZookeeper; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.conf.Configuration; - -import java.net.URLEncoder; -import java.util.concurrent.atomic.AtomicBoolean; - public class TestLogsCleaner { private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private ReplicationZookeeperWrapper zkHelper; + private ReplicationZookeeper zkHelper; /** * @throws java.lang.Exception @@ -68,9 +65,11 @@ public class TestLogsCleaner { @Before public void setUp() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); + /* TODO REENABLE zkHelper = new ReplicationZookeeperWrapper( ZooKeeperWrapper.createInstance(conf, HRegionServer.class.getName()), conf, new AtomicBoolean(true), "test-cluster"); + */ } /** @@ -83,13 +82,25 @@ public class TestLogsCleaner { @Test public void testLogCleaning() throws Exception{ Configuration c = TEST_UTIL.getConfiguration(); - Path oldLogDir = new Path(TEST_UTIL.getTestDir(), + Path oldLogDir = new Path(HBaseTestingUtility.getTestDir(), HConstants.HREGION_OLDLOGDIR_NAME); String fakeMachineName = URLEncoder.encode("regionserver:60020", "UTF8"); FileSystem fs = FileSystem.get(c); - AtomicBoolean stop = new AtomicBoolean(false); - LogsCleaner cleaner = new LogsCleaner(1000, stop,c, fs, oldLogDir); + Stoppable stoppable = new Stoppable() { + private volatile boolean stopped = false; + + @Override + public void stop(String why) { + this.stopped = true; + } + + @Override + public boolean isStopped() { + return this.stopped; + } + }; + LogCleaner cleaner = new LogCleaner(1000, stoppable, c, fs, oldLogDir); // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files long now = System.currentTimeMillis(); @@ -113,7 +124,7 @@ public class TestLogsCleaner { // (TimeToLiveLogCleaner) but would be rejected by the second // (ReplicationLogCleaner) if (i % (30/3) == 0) { - zkHelper.addLogToList(fileName.getName(), fakeMachineName); +// REENABLE zkHelper.addLogToList(fileName.getName(), fakeMachineName); System.out.println("Replication log file: " + fileName); } } @@ -144,5 +155,4 @@ public class TestLogsCleaner { System.out.println("Keeped log files: " + file.getPath().getName()); } } - -} +} \ No newline at end of file Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java Tue Aug 31 23:51:44 2010 @@ -23,16 +23,15 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.HMsg; -import org.apache.hadoop.hbase.HServerInfo; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.executor.HBaseEventHandler; -import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventHandlerListener; -import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType; +import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -51,7 +50,7 @@ import static org.junit.Assert.*; public class TestMaster { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static final Log LOG = LogFactory.getLog(TestMasterWithDisabling.class); + private static final Log LOG = LogFactory.getLog(TestMaster.class); private static final byte[] TABLENAME = Bytes.toBytes("TestMaster"); private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); @@ -73,10 +72,12 @@ public class TestMaster { HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); TEST_UTIL.createTable(TABLENAME, FAMILYNAME); - TEST_UTIL.loadTable(new HTable(TABLENAME), FAMILYNAME); + TEST_UTIL.loadTable(new HTable(TEST_UTIL.getConfiguration(), TABLENAME), + FAMILYNAME); List> tableRegions = - m.getTableRegions(TABLENAME); + MetaReader.getTableRegionsAndLocations(m.getCatalogTracker(), + Bytes.toString(TABLENAME)); LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions)); assertEquals(1, tableRegions.size()); assertArrayEquals(HConstants.EMPTY_START_ROW, @@ -85,11 +86,12 @@ public class TestMaster { tableRegions.get(0).getFirst().getEndKey()); // Now trigger a split and stop when the split is in progress - + CountDownLatch aboutToOpen = new CountDownLatch(1); CountDownLatch proceed = new CountDownLatch(1); RegionOpenListener list = new RegionOpenListener(aboutToOpen, proceed); - HBaseEventHandler.registerListener(list); + cluster.getMaster().executorService. + registerListener(EventType.RS2ZK_REGION_OPENED, list); LOG.info("Splitting table"); admin.split(TABLENAME); @@ -97,7 +99,9 @@ public class TestMaster { aboutToOpen.await(60, TimeUnit.SECONDS); try { LOG.info("Making sure we can call getTableRegions while opening"); - tableRegions = m.getTableRegions(TABLENAME); + tableRegions = MetaReader.getTableRegionsAndLocations( + m.getCatalogTracker(), Bytes.toString(TABLENAME)); + LOG.info("Regions: " + Joiner.on(',').join(tableRegions)); // We have three regions because one is split-in-progress assertEquals(3, tableRegions.size()); @@ -105,14 +109,16 @@ public class TestMaster { Pair pair = m.getTableRegionForRow(TABLENAME, Bytes.toBytes("cde")); LOG.info("Result is: " + pair); - Pair tableRegionFromName = m.getTableRegionFromName(pair.getFirst().getRegionName()); + Pair tableRegionFromName = + MetaReader.getRegion(m.getCatalogTracker(), + pair.getFirst().getRegionName()); assertEquals(tableRegionFromName.getFirst(), pair.getFirst()); } finally { proceed.countDown(); } } - static class RegionOpenListener implements HBaseEventHandlerListener { + static class RegionOpenListener implements EventHandlerListener { CountDownLatch aboutToOpen, proceed; public RegionOpenListener(CountDownLatch aboutToOpen, CountDownLatch proceed) @@ -122,8 +128,8 @@ public class TestMaster { } @Override - public void afterProcess(HBaseEventHandler event) { - if (event.getHBEvent() != HBaseEventType.RS2ZK_REGION_OPENED) { + public void afterProcess(EventHandler event) { + if (event.getEventType() != EventType.RS2ZK_REGION_OPENED) { return; } try { @@ -136,8 +142,7 @@ public class TestMaster { } @Override - public void beforeProcess(HBaseEventHandler event) { + public void beforeProcess(EventHandler event) { } } - -} +} \ No newline at end of file Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java Tue Aug 31 23:51:44 2010 @@ -19,42 +19,25 @@ */ package org.apache.hadoop.hbase.master; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.net.BindException; -import java.util.Collection; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HMsg; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HServerAddress; -import org.apache.hadoop.hbase.HServerInfo; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer; -import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Writables; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; /** @@ -104,6 +87,7 @@ public class TestMasterTransitions { * the requeuing, send over a close of a region on 'otherServer' so it comes * into a master that has its meta region marked as offline. */ + /* static class HBase2428Listener implements RegionServerOperationListener { // Map of what we've delayed so we don't do do repeated delays. private final Set postponed = @@ -164,12 +148,13 @@ public class TestMasterTransitions { if (isWantedCloseOperation(op) != null) return; this.done = true; } - +*/ /* * @param op * @return Null if not the wanted ProcessRegionClose, else op * cast as a ProcessRegionClose. */ + /* private ProcessRegionClose isWantedCloseOperation(final RegionServerOperation op) { // Count every time we get a close operation. if (op instanceof ProcessRegionClose) { @@ -198,14 +183,15 @@ public class TestMasterTransitions { return true; } } - +*/ /** * In 2428, the meta region has just been set offline and then a close comes * in. * @see HBASE-2428 */ - @Test (timeout=300000) public void testRegionCloseWhenNoMetaHBase2428() + @Ignore @Test (timeout=300000) public void testRegionCloseWhenNoMetaHBase2428() throws Exception { + /* LOG.info("Running testRegionCloseWhenNoMetaHBase2428"); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); final HMaster master = cluster.getMaster(); @@ -249,6 +235,7 @@ public class TestMasterTransitions { master.getRegionServerOperationQueue(). unregisterRegionServerOperationListener(listener); } + */ } /** @@ -257,8 +244,9 @@ public class TestMasterTransitions { * If confusion between old and new, purportedly meta never comes back. Test * that meta gets redeployed. */ - @Test (timeout=300000) public void testAddingServerBeforeOldIsDead2413() + @Ignore @Test (timeout=300000) public void testAddingServerBeforeOldIsDead2413() throws IOException { + /* LOG.info("Running testAddingServerBeforeOldIsDead2413"); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); int count = count(); @@ -298,9 +286,9 @@ public class TestMasterTransitions { } finally { c.set(HConstants.REGIONSERVER_PORT, oldPort); } + */ } - /** * HBase2482 is about outstanding region openings. If any are outstanding * when a regionserver goes down, then they'll never deploy. They'll be @@ -309,6 +297,7 @@ public class TestMasterTransitions { * then we kill it. It also looks out for a close message on the victim * server because that signifies start of the fireworks. */ + /* static class HBase2482Listener implements RegionServerOperationListener { private final HRegionServer victim; private boolean abortSent = false; @@ -367,7 +356,7 @@ public class TestMasterTransitions { } } } - +*/ /** * In 2482, a RS with an opening region on it dies. The said region is then * stuck in the master's regions-in-transition and never leaves it. This @@ -382,8 +371,9 @@ public class TestMasterTransitions { * done. * @see HBASE-2482 */ - @Test (timeout=300000) public void testKillRSWithOpeningRegion2482() + @Ignore @Test (timeout=300000) public void testKillRSWithOpeningRegion2482() throws Exception { + /* LOG.info("Running testKillRSWithOpeningRegion2482"); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); if (cluster.getLiveRegionServerThreads().size() < 2) { @@ -413,7 +403,7 @@ public class TestMasterTransitions { // After all closes, add blocking message before the region opens start to // come in. cluster.addMessageToSendRegionServer(hrs, - new HMsg(HMsg.Type.TESTING_MSG_BLOCK_RS)); + new HMsg(HMsg.Type.TESTING_BLOCK_REGIONSERVER)); // Wait till one of the above close messages has an effect before we start // wait on all regions back online. while (!listener.closed) Threads.sleep(100); @@ -427,11 +417,13 @@ public class TestMasterTransitions { m.getRegionServerOperationQueue(). unregisterRegionServerOperationListener(listener); } + */ } /* * @return Count of all non-catalog regions on the designated server */ +/* private int closeAllNonCatalogRegions(final MiniHBaseCluster cluster, final MiniHBaseCluster.MiniHBaseClusterRegionServer hrs) throws IOException { @@ -460,6 +452,7 @@ public class TestMasterTransitions { * @return Count of regions in meta table. * @throws IOException */ + /* private static int countOfMetaRegions() throws IOException { HTable meta = new HTable(TEST_UTIL.getConfiguration(), @@ -477,7 +470,7 @@ public class TestMasterTransitions { s.close(); return rows; } - +*/ /* * Add to each of the regions in .META. a value. Key is the startrow of the * region (except its 'aaa' for first region). Actual value is the row name. Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java Tue Aug 31 23:51:44 2010 @@ -19,72 +19,112 @@ */ package org.apache.hadoop.hbase.master; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.io.IOException; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.executor.RegionTransitionEventData; -import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Writables; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper; -import org.junit.AfterClass; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; public class TestRestartCluster { private static final Log LOG = LogFactory.getLog(TestRestartCluster.class); - private static Configuration conf; - private static HBaseTestingUtility utility; - private static ZooKeeperWrapper zkWrapper; + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static ZooKeeperWatcher zooKeeper; private static final byte[] TABLENAME = Bytes.toBytes("master_transitions"); private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a")}; - - @BeforeClass public static void beforeAllTests() throws Exception { - conf = HBaseConfiguration.create(); - utility = new HBaseTestingUtility(conf); - } - @AfterClass public static void afterAllTests() throws IOException { - utility.shutdownMiniCluster(); + + private static final byte [][] TABLES = new byte[][] { + Bytes.toBytes("restartTableOne"), + Bytes.toBytes("restartTableTwo"), + Bytes.toBytes("restartTableThree") + }; + private static final byte [] FAMILY = Bytes.toBytes("family"); + + @Before public void setup() throws Exception { } - @Before public void setup() throws IOException { + @After public void teardown() throws IOException { + UTIL.shutdownMiniCluster(); } - @Test (timeout=300000) public void testRestartClusterAfterKill()throws Exception { - utility.startMiniZKCluster(); - zkWrapper = ZooKeeperWrapper.createInstance(conf, "cluster1"); + @Test (timeout=300000) public void testRestartClusterAfterKill() + throws Exception { + UTIL.startMiniZKCluster(); + zooKeeper = new ZooKeeperWatcher(UTIL.getConfiguration(), "cluster1", null); // create the unassigned region, throw up a region opened state for META - String unassignedZNode = zkWrapper.getRegionInTransitionZNode(); - zkWrapper.createZNodeIfNotExists(unassignedZNode); - byte[] data = null; - HBaseEventType hbEventType = HBaseEventType.RS2ZK_REGION_OPENED; - try { - data = Writables.getBytes(new RegionTransitionEventData(hbEventType, HMaster.MASTER)); - } catch (IOException e) { - LOG.error("Error creating event data for " + hbEventType, e); - } - zkWrapper.createOrUpdateUnassignedRegion( - HRegionInfo.ROOT_REGIONINFO.getEncodedName(), data); - zkWrapper.createOrUpdateUnassignedRegion( - HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), data); - LOG.debug("Created UNASSIGNED zNode for ROOT and META regions in state " + HBaseEventType.M2ZK_REGION_OFFLINE); - + String unassignedZNode = zooKeeper.assignmentZNode; + ZKUtil.createAndFailSilent(zooKeeper, unassignedZNode); + + ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.ROOT_REGIONINFO, + HMaster.MASTER); + + ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.FIRST_META_REGIONINFO, + HMaster.MASTER); + + LOG.debug("Created UNASSIGNED zNode for ROOT and META regions in state " + + EventType.M2ZK_REGION_OFFLINE); + // start the HB cluster LOG.info("Starting HBase cluster..."); - utility.startMiniCluster(2); - - utility.createTable(TABLENAME, FAMILIES); + UTIL.startMiniCluster(2); + + UTIL.createTable(TABLENAME, FAMILIES); LOG.info("Created a table, waiting for table to be available..."); - utility.waitTableAvailable(TABLENAME, 60*1000); + UTIL.waitTableAvailable(TABLENAME, 60*1000); - LOG.info("Master deleted unassgined region and started up successfully."); + LOG.info("Master deleted unassigned region and started up successfully."); + } + + @Test (timeout=300000) + public void testClusterRestart() throws Exception { + UTIL.startMiniCluster(3); + LOG.info("\n\nCreating tables"); + for(byte [] TABLE : TABLES) { + UTIL.createTable(TABLE, FAMILY); + UTIL.waitTableAvailable(TABLE, 30000); + } + List allRegions = + MetaScanner.listAllRegions(UTIL.getConfiguration()); + assertEquals(3, allRegions.size()); + + LOG.info("\n\nShutting down cluster"); + UTIL.getHBaseCluster().shutdown(); + UTIL.getHBaseCluster().join(); + + LOG.info("\n\nSleeping a bit"); + Thread.sleep(2000); + + LOG.info("\n\nStarting cluster the second time"); + UTIL.restartHBaseCluster(3); + + allRegions = MetaScanner.listAllRegions(UTIL.getConfiguration()); + assertEquals(3, allRegions.size()); + + LOG.info("\n\nWaiting for tables to be available"); + for(byte [] TABLE: TABLES) { + try { + UTIL.createTable(TABLE, FAMILY); + assertTrue("Able to create table that should already exist", false); + } catch(TableExistsException tee) { + LOG.info("Table already exists as expected"); + } + UTIL.waitTableAvailable(TABLE, 30000); + } } } Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java?rev=991397&view=auto ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java (added) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java Tue Aug 31 23:51:44 2010 @@ -0,0 +1,331 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + + +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.Writables; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test open and close of regions using zk. + */ +public class TestZKBasedOpenCloseRegion { + private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String TABLENAME = "TestZKBasedOpenCloseRegion"; + private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"), + Bytes.toBytes("b"), Bytes.toBytes("c")}; + + @BeforeClass public static void beforeAllTests() throws Exception { + Configuration c = TEST_UTIL.getConfiguration(); + c.setBoolean("dfs.support.append", true); + c.setInt("hbase.regionserver.info.port", 0); + c.setInt("hbase.master.meta.thread.rescanfrequency", 5*1000); + TEST_UTIL.startMiniCluster(2); + TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES); + HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily()); + waitUntilAllRegionsAssigned(countOfRegions); + addToEachStartKey(countOfRegions); + } + + @AfterClass public static void afterAllTests() throws IOException { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before public void setup() throws IOException { + if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) { + // Need at least two servers. + LOG.info("Started new server=" + + TEST_UTIL.getHBaseCluster().startRegionServer()); + + } + } + + /** + * Test we reopen a region once closed. + * @throws Exception + */ + @Test (timeout=300000) public void testReOpenRegion() + throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + LOG.info("Number of region servers = " + + cluster.getLiveRegionServerThreads().size()); + + int rsIdx = 0; + HRegionServer regionServer = + TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx); + HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions()); + LOG.debug("Asking RS to close region " + hri.getRegionNameAsString()); + + AtomicBoolean closeEventProcessed = new AtomicBoolean(false); + AtomicBoolean reopenEventProcessed = new AtomicBoolean(false); + + EventHandlerListener closeListener = + new ReopenEventListener(hri.getRegionNameAsString(), + closeEventProcessed, EventType.RS2ZK_REGION_CLOSED); + cluster.getMaster().executorService. + registerListener(EventType.RS2ZK_REGION_CLOSED, closeListener); + + EventHandlerListener openListener = + new ReopenEventListener(hri.getRegionNameAsString(), + reopenEventProcessed, EventType.RS2ZK_REGION_OPENED); + cluster.getMaster().executorService. + registerListener(EventType.RS2ZK_REGION_OPENED, openListener); + + LOG.info("Unassign " + hri.getRegionNameAsString()); + cluster.getMaster().assignmentManager.unassign(hri); + + while (!closeEventProcessed.get()) { + Threads.sleep(100); + } + + while (!reopenEventProcessed.get()) { + Threads.sleep(100); + } + + LOG.info("Done with testReOpenRegion"); + } + + private HRegionInfo getNonMetaRegion(final Collection regions) { + HRegionInfo hri = null; + for (HRegionInfo i: regions) { + LOG.info(i.getRegionNameAsString()); + if (!i.isMetaRegion()) { + hri = i; + break; + } + } + return hri; + } + + public static class ReopenEventListener implements EventHandlerListener { + private static final Log LOG = LogFactory.getLog(ReopenEventListener.class); + String regionName; + AtomicBoolean eventProcessed; + EventType eventType; + + public ReopenEventListener(String regionName, + AtomicBoolean eventProcessed, EventType eventType) { + this.regionName = regionName; + this.eventProcessed = eventProcessed; + this.eventType = eventType; + } + + @Override + public void beforeProcess(EventHandler event) { + if(event.getEventType() == eventType) { + LOG.info("Received " + eventType + " and beginning to process it"); + } + } + + @Override + public void afterProcess(EventHandler event) { + LOG.info("afterProcess(" + event + ")"); + if(event.getEventType() == eventType) { + LOG.info("Finished processing " + eventType); + String regionName = ""; + if(eventType == EventType.RS2ZK_REGION_OPENED) { + TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event; + regionName = hriCarrier.getHRegionInfo().getRegionNameAsString(); + } else if(eventType == EventType.RS2ZK_REGION_CLOSED) { + TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event; + regionName = hriCarrier.getHRegionInfo().getRegionNameAsString(); + } + if(this.regionName.equals(regionName)) { + eventProcessed.set(true); + } + synchronized(eventProcessed) { + eventProcessed.notifyAll(); + } + } + } + } + + @Test (timeout=300000) public void testCloseRegion() + throws Exception { + LOG.info("Running testCloseRegion"); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size()); + + int rsIdx = 0; + HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx); + HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions()); + LOG.debug("Asking RS to close region " + hri.getRegionNameAsString()); + + AtomicBoolean closeEventProcessed = new AtomicBoolean(false); + EventHandlerListener listener = + new CloseRegionEventListener(hri.getRegionNameAsString(), + closeEventProcessed); + cluster.getMaster().executorService.registerListener(EventType.RS2ZK_REGION_CLOSED, listener); + + cluster.getMaster().assignmentManager.unassign(hri); + + while (!closeEventProcessed.get()) { + Threads.sleep(100); + } + LOG.info("Done with testCloseRegion"); + } + + public static class CloseRegionEventListener implements EventHandlerListener { + private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class); + String regionToClose; + AtomicBoolean closeEventProcessed; + + public CloseRegionEventListener(String regionToClose, + AtomicBoolean closeEventProcessed) { + this.regionToClose = regionToClose; + this.closeEventProcessed = closeEventProcessed; + } + + @Override + public void afterProcess(EventHandler event) { + LOG.info("afterProcess(" + event + ")"); + if(event.getEventType() == EventType.RS2ZK_REGION_CLOSED) { + LOG.info("Finished processing CLOSE REGION"); + TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event; + if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) { + LOG.info("Setting closeEventProcessed flag"); + closeEventProcessed.set(true); + } else { + LOG.info("Region to close didn't match"); + } + } + } + + @Override + public void beforeProcess(EventHandler event) { + if(event.getEventType() == EventType.M2RS_CLOSE_REGION) { + LOG.info("Received CLOSE RPC and beginning to process it"); + } + } + } + + private static void waitUntilAllRegionsAssigned(final int countOfRegions) + throws IOException { + HTable meta = new HTable(TEST_UTIL.getConfiguration(), + HConstants.META_TABLE_NAME); + while (true) { + int rows = 0; + Scan scan = new Scan(); + scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + ResultScanner s = meta.getScanner(scan); + for (Result r = null; (r = s.next()) != null;) { + byte [] b = + r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + if (b == null || b.length <= 0) { + break; + } + rows++; + } + s.close(); + // If I get to here and all rows have a Server, then all have been assigned. + if (rows == countOfRegions) { + break; + } + LOG.info("Found=" + rows); + Threads.sleep(1000); + } + } + + /* + * Add to each of the regions in .META. a value. Key is the startrow of the + * region (except its 'aaa' for first region). Actual value is the row name. + * @param expected + * @return + * @throws IOException + */ + private static int addToEachStartKey(final int expected) throws IOException { + HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + HTable meta = new HTable(TEST_UTIL.getConfiguration(), + HConstants.META_TABLE_NAME); + int rows = 0; + Scan scan = new Scan(); + scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + ResultScanner s = meta.getScanner(scan); + for (Result r = null; (r = s.next()) != null;) { + byte [] b = + r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + if (b == null || b.length <= 0) { + break; + } + HRegionInfo hri = Writables.getHRegionInfo(b); + // If start key, add 'aaa'. + byte [] row = getStartKey(hri); + Put p = new Put(row); + p.add(getTestFamily(), getTestQualifier(), row); + t.put(p); + rows++; + } + s.close(); + Assert.assertEquals(expected, rows); + return rows; + } + + private static byte [] getStartKey(final HRegionInfo hri) { + return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())? + Bytes.toBytes("aaa"): hri.getStartKey(); + } + + private static byte [] getTestFamily() { + return FAMILIES[0]; + } + + private static byte [] getTestQualifier() { + return getTestFamily(); + } + + public static void main(String args[]) throws Exception { + TestZKBasedOpenCloseRegion.beforeAllTests(); + + TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion(); + test.setup(); + test.testCloseRegion(); + + TestZKBasedOpenCloseRegion.afterAllTests(); + } +} Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/DisabledTestRegionServerExit.java Tue Aug 31 23:51:44 2010 @@ -130,7 +130,7 @@ public class DisabledTestRegionServerExi int server = -1; for (int i = 0; i < regionThreads.size() && server == -1; i++) { HRegionServer s = regionThreads.get(i).getRegionServer(); - Collection regions = s.getOnlineRegions(); + Collection regions = s.getOnlineRegionsLocalContext(); for (HRegion r : regions) { if (Bytes.equals(r.getTableDesc().getName(), HConstants.META_TABLE_NAME)) { Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java Tue Aug 31 23:51:44 2010 @@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.client.Pu public class OOMERegionServer extends HRegionServer { private List retainer = new ArrayList(); - public OOMERegionServer(HBaseConfiguration conf) throws IOException { + public OOMERegionServer(HBaseConfiguration conf) throws IOException, InterruptedException { super(conf); } Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java Tue Aug 31 23:51:44 2010 @@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.HTableDes import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; import org.apache.hadoop.hbase.util.Bytes; @@ -161,8 +160,9 @@ public class TestFSErrorsExposed { fam, 1, HColumnDescriptor.DEFAULT_COMPRESSION, false, false, HConstants.FOREVER, "NONE")); admin.createTable(desc); - - HTable table = new HTable(tableName); + // Make it fail faster. + util.getConfiguration().setInt("hbase.client.retries.number", 1); + HTable table = new HTable(util.getConfiguration(), tableName); // Load some data util.loadTable(table, fam); Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Tue Aug 31 23:51:44 2010 @@ -1316,7 +1316,7 @@ public class TestHRegion extends HBaseTe // the prepare call -- we are not ready to split just now. Just return. if (!st.prepare()) return null; try { - result = st.execute(null); + result = st.execute(null, null); } catch (IOException ioe) { try { LOG.info("Running rollback of failed split of " + Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java?rev=991397&view=auto ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java (added) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java Tue Aug 31 23:51:44 2010 @@ -0,0 +1,116 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.Semaphore; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.MasterAddressTracker; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestMasterAddressManager { + private static final Log LOG = LogFactory.getLog(TestMasterAddressManager.class); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + /** + * Unit tests that uses ZooKeeper but does not use the master-side methods + * but rather acts directly on ZK. + * @throws Exception + */ + @Test + public void testMasterAddressManagerFromZK() throws Exception { + + ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "testMasterAddressManagerFromZK", null); + ZKUtil.createAndFailSilent(zk, zk.baseZNode); + + // Should not have a master yet + MasterAddressTracker addressManager = new MasterAddressTracker(zk, null); + addressManager.start(); + assertFalse(addressManager.hasMaster()); + zk.registerListener(addressManager); + + // Use a listener to capture when the node is actually created + NodeCreationListener listener = new NodeCreationListener(zk, zk.masterAddressZNode); + zk.registerListener(listener); + + // Create the master node with a dummy address + String host = "hostname"; + int port = 1234; + HServerAddress dummyAddress = new HServerAddress(host, port); + LOG.info("Creating master node"); + ZKUtil.setAddressAndWatch(zk, zk.masterAddressZNode, dummyAddress); + + // Wait for the node to be created + LOG.info("Waiting for master address manager to be notified"); + listener.waitForCreation(); + LOG.info("Master node created"); + assertTrue(addressManager.hasMaster()); + HServerAddress pulledAddress = addressManager.getMasterAddress(); + assertTrue(pulledAddress.equals(dummyAddress)); + + } + + public static class NodeCreationListener extends ZooKeeperListener { + private static final Log LOG = LogFactory.getLog(NodeCreationListener.class); + + private Semaphore lock; + private String node; + + public NodeCreationListener(ZooKeeperWatcher watcher, String node) { + super(watcher); + lock = new Semaphore(0); + this.node = node; + } + + @Override + public void nodeCreated(String path) { + if(path.equals(node)) { + LOG.debug("nodeCreated(" + path + ")"); + lock.release(); + } + } + + public void waitForCreation() throws InterruptedException { + lock.acquire(); + } + } +} Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java Tue Aug 31 23:51:44 2010 @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.HConstant import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.util.Bytes; @@ -44,6 +45,7 @@ import org.apache.hadoop.hbase.util.Pair import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; /** * Test the {@link SplitTransaction} class against an HRegion (as opposed to @@ -67,8 +69,9 @@ public class TestSplitTransaction { this.fs.delete(this.testdir, true); this.wal = new HLog(fs, new Path(this.testdir, "logs"), new Path(this.testdir, "archive"), - TEST_UTIL.getConfiguration(), null); + TEST_UTIL.getConfiguration()); this.parent = createRegion(this.testdir, this.wal); + TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true); } @After public void teardown() throws IOException { @@ -128,7 +131,9 @@ public class TestSplitTransaction { SplitTransaction st = prepareGOOD_SPLIT_ROW(); // Run the execute. Look at what it returns. - PairOfSameType daughters = st.execute(null); + Server mockServer = Mockito.mock(Server.class); + when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); + PairOfSameType daughters = st.execute(mockServer, null); // Do some assertions about execution. assertTrue(this.fs.exists(st.getSplitDir())); // Assert the parent region is closed. @@ -150,7 +155,7 @@ public class TestSplitTransaction { int daughtersRowCount = 0; for (HRegion r: daughters) { // Open so can count its content. - HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(), this.testdir, + HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(), r.getLog(), r.getConf()); try { int count = countRows(openRegion); @@ -174,12 +179,14 @@ public class TestSplitTransaction { // Start transaction. SplitTransaction st = prepareGOOD_SPLIT_ROW(); SplitTransaction spiedUponSt = spy(st); - when(spiedUponSt.createDaughterRegion(spiedUponSt.getSecondDaughter())). + when(spiedUponSt.createDaughterRegion(spiedUponSt.getSecondDaughter(), null)). thenThrow(new MockedFailedDaughterCreation()); // Run the execute. Look at what it returns. boolean expectedException = false; + Server mockServer = Mockito.mock(Server.class); + when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); try { - spiedUponSt.execute(null); + spiedUponSt.execute(mockServer, null); } catch (MockedFailedDaughterCreation e) { expectedException = true; } @@ -198,12 +205,12 @@ public class TestSplitTransaction { // Now retry the split but do not throw an exception this time. assertTrue(st.prepare()); - PairOfSameType daughters = st.execute(null); + PairOfSameType daughters = st.execute(mockServer, null); // Count rows. int daughtersRowCount = 0; for (HRegion r: daughters) { // Open so can count its content. - HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(), this.testdir, + HRegion openRegion = HRegion.openHRegion(r.getRegionInfo(), r.getLog(), r.getConf()); try { int count = countRows(openRegion); @@ -248,6 +255,6 @@ public class TestSplitTransaction { HColumnDescriptor hcd = new HColumnDescriptor(CF); htd.addFamily(hcd); HRegionInfo hri = new HRegionInfo(htd, STARTROW, ENDROW); - return HRegion.openHRegion(hri, testdir, wal, TEST_UTIL.getConfiguration()); + return HRegion.openHRegion(hri, wal, TEST_UTIL.getConfiguration()); } } \ No newline at end of file Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java Tue Aug 31 23:51:44 2010 @@ -123,7 +123,7 @@ public class TestStore extends TestCase HTableDescriptor htd = new HTableDescriptor(table); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd, null, null, false); - HLog hlog = new HLog(fs, logdir, oldLogDir, conf, null); + HLog hlog = new HLog(fs, logdir, oldLogDir, conf); HRegion region = new HRegion(basedir, hlog, fs, conf, info, null); store = new Store(basedir, region, hcd, fs, conf); Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java Tue Aug 31 23:51:44 2010 @@ -29,7 +29,7 @@ public class InstrumentedSequenceFileLog @Override public void append(HLog.Entry entry) throws IOException { super.append(entry); - if (activateFailure && Bytes.equals(entry.getKey().getRegionName(), "break".getBytes())) { + if (activateFailure && Bytes.equals(entry.getKey().getEncodedRegionName(), "break".getBytes())) { System.out.println(getClass().getName() + ": I will throw an exception now..."); throw(new IOException("This exception is instrumented and should only be thrown for testing")); } Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java Tue Aug 31 23:51:44 2010 @@ -133,7 +133,7 @@ public class TestHLog { final byte [] tableName = Bytes.toBytes(getName()); final byte [] rowName = tableName; Path logdir = new Path(dir, HConstants.HREGION_LOGDIR_NAME); - HLog log = new HLog(fs, logdir, oldLogDir, conf, null); + HLog log = new HLog(fs, logdir, oldLogDir, conf); final int howmany = 3; HRegionInfo[] infos = new HRegionInfo[3]; for(int i = 0; i < howmany; i++) { @@ -192,7 +192,7 @@ public class TestHLog { out.close(); in.close(); Path subdir = new Path(dir, "hlogdir"); - HLog wal = new HLog(fs, subdir, oldLogDir, conf, null); + HLog wal = new HLog(fs, subdir, oldLogDir, conf); final int total = 20; HRegionInfo info = new HRegionInfo(new HTableDescriptor(bytes), @@ -295,7 +295,7 @@ public class TestHLog { HLog.Entry entry = new HLog.Entry(); while((entry = reader.next(entry)) != null) { HLogKey key = entry.getKey(); - String region = Bytes.toString(key.getRegionName()); + String region = Bytes.toString(key.getEncodedRegionName()); // Assert that all edits are for same region. if (previousRegion != null) { assertEquals(previousRegion, region); @@ -325,7 +325,7 @@ public class TestHLog { HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false); Path subdir = new Path(dir, "hlogdir"); Path archdir = new Path(dir, "hlogdir_archive"); - HLog wal = new HLog(fs, subdir, archdir, conf, null); + HLog wal = new HLog(fs, subdir, archdir, conf); final int total = 20; for (int i = 0; i < total; i++) { @@ -429,7 +429,7 @@ public class TestHLog { final byte [] tableName = Bytes.toBytes("tablename"); final byte [] row = Bytes.toBytes("row"); HLog.Reader reader = null; - HLog log = new HLog(fs, dir, oldLogDir, conf, null); + HLog log = new HLog(fs, dir, oldLogDir, conf); try { // Write columns named 1, 2, 3, etc. and then values of single byte // 1, 2, 3... @@ -442,10 +442,9 @@ public class TestHLog { } HRegionInfo info = new HRegionInfo(new HTableDescriptor(tableName), row,Bytes.toBytes(Bytes.toString(row) + "1"), false); - final byte [] regionName = info.getRegionName(); log.append(info, tableName, cols, System.currentTimeMillis()); long logSeqId = log.startCacheFlush(); - log.completeCacheFlush(regionName, tableName, logSeqId, info.isMetaRegion()); + log.completeCacheFlush(info.getEncodedNameAsBytes(), tableName, logSeqId, info.isMetaRegion()); log.close(); Path filename = log.computeFilename(); log = null; @@ -458,7 +457,7 @@ public class TestHLog { if (entry == null) break; HLogKey key = entry.getKey(); WALEdit val = entry.getEdit(); - assertTrue(Bytes.equals(regionName, key.getRegionName())); + assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName())); assertTrue(Bytes.equals(tableName, key.getTablename())); KeyValue kv = val.getKeyValues().get(0); assertTrue(Bytes.equals(row, kv.getRow())); @@ -470,7 +469,7 @@ public class TestHLog { HLogKey key = entry.getKey(); WALEdit val = entry.getEdit(); // Assert only one more row... the meta flushed row. - assertTrue(Bytes.equals(regionName, key.getRegionName())); + assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName())); assertTrue(Bytes.equals(tableName, key.getTablename())); KeyValue kv = val.getKeyValues().get(0); assertTrue(Bytes.equals(HLog.METAROW, kv.getRow())); @@ -498,7 +497,7 @@ public class TestHLog { final byte [] tableName = Bytes.toBytes("tablename"); final byte [] row = Bytes.toBytes("row"); Reader reader = null; - HLog log = new HLog(fs, dir, oldLogDir, conf, null); + HLog log = new HLog(fs, dir, oldLogDir, conf); try { // Write columns named 1, 2, 3, etc. and then values of single byte // 1, 2, 3... @@ -513,7 +512,7 @@ public class TestHLog { HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); log.append(hri, tableName, cols, System.currentTimeMillis()); long logSeqId = log.startCacheFlush(); - log.completeCacheFlush(hri.getRegionName(), tableName, logSeqId, false); + log.completeCacheFlush(hri.getEncodedNameAsBytes(), tableName, logSeqId, false); log.close(); Path filename = log.computeFilename(); log = null; @@ -524,7 +523,7 @@ public class TestHLog { int idx = 0; for (KeyValue val : entry.getEdit().getKeyValues()) { assertTrue(Bytes.equals(hri.getRegionName(), - entry.getKey().getRegionName())); + entry.getKey().getEncodedRegionName())); assertTrue(Bytes.equals(tableName, entry.getKey().getTablename())); assertTrue(Bytes.equals(row, val.getRow())); assertEquals((byte)(idx + '0'), val.getValue()[0]); @@ -537,7 +536,7 @@ public class TestHLog { assertEquals(1, entry.getEdit().size()); for (KeyValue val : entry.getEdit().getKeyValues()) { assertTrue(Bytes.equals(hri.getRegionName(), - entry.getKey().getRegionName())); + entry.getKey().getEncodedRegionName())); assertTrue(Bytes.equals(tableName, entry.getKey().getTablename())); assertTrue(Bytes.equals(HLog.METAROW, val.getRow())); assertTrue(Bytes.equals(HLog.METAFAMILY, val.getFamily())); @@ -564,9 +563,9 @@ public class TestHLog { final int COL_COUNT = 10; final byte [] tableName = Bytes.toBytes("tablename"); final byte [] row = Bytes.toBytes("row"); - HLog log = new HLog(fs, dir, oldLogDir, conf, null); - DumbLogEntriesVisitor visitor = new DumbLogEntriesVisitor(); - log.addLogEntryVisitor(visitor); + HLog log = new HLog(fs, dir, oldLogDir, conf); + DumbWALObserver visitor = new DumbWALObserver(); + log.registerWALActionsListener(visitor); long timestamp = System.currentTimeMillis(); HRegionInfo hri = new HRegionInfo(new HTableDescriptor(tableName), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); @@ -578,7 +577,7 @@ public class TestHLog { log.append(hri, tableName, cols, System.currentTimeMillis()); } assertEquals(COL_COUNT, visitor.increments); - log.removeLogEntryVisitor(visitor); + log.unregisterWALActionsListener(visitor); WALEdit cols = new WALEdit(); cols.add(new KeyValue(row, Bytes.toBytes("column"), Bytes.toBytes(Integer.toString(11)), @@ -587,8 +586,7 @@ public class TestHLog { assertEquals(COL_COUNT, visitor.increments); } - static class DumbLogEntriesVisitor implements LogEntryVisitor { - + static class DumbWALObserver implements WALObserver { int increments = 0; @Override @@ -596,5 +594,17 @@ public class TestHLog { WALEdit logEdit) { increments++; } + + @Override + public void logRolled(Path newFile) { + // TODO Auto-generated method stub + + } + + @Override + public void logRollRequested() { + // TODO Auto-generated method stub + + } } } Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java Tue Aug 31 23:51:44 2010 @@ -154,6 +154,8 @@ public class TestLogRolling { private void startAndWriteData() throws IOException { // When the META table can be opened, the region servers are running new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME); + this.server = cluster.getRegionServerThreads().get(0).getRegionServer(); + this.log = server.getWAL(); // Create the test table and open it HTableDescriptor desc = new HTableDescriptor(tableName); @@ -162,7 +164,7 @@ public class TestLogRolling { HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName)); - this.log = server.getLog(); + this.log = server.getWAL(); for (int i = 1; i <= 256; i++) { // 256 writes should cause 8 log rolls Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", i))); put.add(HConstants.CATALOG_FAMILY, null, value); @@ -192,7 +194,7 @@ public class TestLogRolling { // flush all regions List regions = - new ArrayList(server.getOnlineRegions()); + new ArrayList(server.getOnlineRegionsLocalContext()); for (HRegion r: regions) { r.flushcache(); } @@ -226,7 +228,6 @@ public class TestLogRolling { /** * Give me the HDFS pipeline for this log file */ - @SuppressWarnings("null") DatanodeInfo[] getPipeline(HLog log) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { OutputStream stm = log.getOutputStream(); @@ -258,10 +259,15 @@ public class TestLogRolling { public void testLogRollOnDatanodeDeath() throws IOException, InterruptedException, IllegalArgumentException, IllegalAccessException, InvocationTargetException { - assertTrue("This test requires HLog file replication.", fs - .getDefaultReplication() > 1); + assertTrue("This test requires HLog file replication.", + fs.getDefaultReplication() > 1); + LOG.info("Replication=" + fs.getDefaultReplication()); // When the META table can be opened, the region servers are running new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME); + + this.server = cluster.getRegionServer(0); + this.log = server.getWAL(); + // Create the test table and open it String tableName = getName(); HTableDescriptor desc = new HTableDescriptor(tableName); @@ -275,7 +281,7 @@ public class TestLogRolling { HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName)); - this.log = server.getLog(); + this.log = server.getWAL(); assertTrue("Need HDFS-826 for this test", log.canGetCurReplicas()); // don't run this test without append support (HDFS-200 & HDFS-142) @@ -297,8 +303,7 @@ public class TestLogRolling { assertTrue("Log should have a timestamp older than now", curTime > oldFilenum && oldFilenum != -1); - assertTrue("The log shouldn't have rolled yet", oldFilenum == log - .getFilenum()); + assertTrue("The log shouldn't have rolled yet", oldFilenum == log.getFilenum()); DatanodeInfo[] pipeline = getPipeline(log); assertTrue(pipeline.length == fs.getDefaultReplication()); Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java?rev=991397&view=auto ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java (added) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALObserver.java Tue Aug 31 23:51:44 2010 @@ -0,0 +1,135 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Test that the actions are called while playing with an HLog + */ +public class TestWALObserver { + protected static final Log LOG = LogFactory.getLog(TestWALObserver.class); + + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private final static byte[] SOME_BYTES = Bytes.toBytes("t"); + private static FileSystem fs; + private static Path oldLogDir; + private static Path logDir; + private static Configuration conf; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.maxlogs", 5); + fs = FileSystem.get(conf); + oldLogDir = new Path(HBaseTestingUtility.getTestDir(), + HConstants.HREGION_OLDLOGDIR_NAME); + logDir = new Path(HBaseTestingUtility.getTestDir(), + HConstants.HREGION_LOGDIR_NAME); + } + + @Before + public void setUp() throws Exception { + fs.delete(logDir, true); + fs.delete(oldLogDir, true); + } + + @After + public void tearDown() throws Exception { + setUp(); + } + + /** + * Add a bunch of dummy data and roll the logs every two insert. We + * should end up with 10 rolled files (plus the roll called in + * the constructor). Also test adding a listener while it's running. + */ + @Test + public void testActionListener() throws Exception { + DummyWALObserver observer = new DummyWALObserver(); + List list = new ArrayList(); + list.add(observer); + DummyWALObserver laterobserver = new DummyWALObserver(); + HLog hlog = new HLog(fs, logDir, oldLogDir, conf, list, null); + HRegionInfo hri = new HRegionInfo(new HTableDescriptor(SOME_BYTES), + SOME_BYTES, SOME_BYTES, false); + + for (int i = 0; i < 20; i++) { + byte[] b = Bytes.toBytes(i+""); + KeyValue kv = new KeyValue(b,b,b); + WALEdit edit = new WALEdit(); + edit.add(kv); + HLogKey key = new HLogKey(b,b, 0, 0); + hlog.append(hri, key, edit); + if (i == 10) { + hlog.registerWALActionsListener(laterobserver); + } + if (i % 2 == 0) { + hlog.rollWriter(); + } + } + assertEquals(11, observer.logRollCounter); + assertEquals(5, laterobserver.logRollCounter); + } + + /** + * Just counts when methods are called + */ + static class DummyWALObserver implements WALObserver { + public int logRollCounter = 0; + + @Override + public void logRolled(Path newFile) { + logRollCounter++; + } + + @Override + public void logRollRequested() { + // Not interested + } + + @Override + public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, + WALEdit logEdit) { + // Not interested + + } + } +} \ No newline at end of file Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java Tue Aug 31 23:51:44 2010 @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.client.Ge import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.regionserver.FlushRequester; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.util.Bytes; @@ -183,7 +184,7 @@ public class TestWALReplay { Path basedir = new Path(this.hbaseRootDir, tableNameStr); deleteDir(basedir); HLog wal = createWAL(this.conf); - HRegion region = HRegion.openHRegion(hri, basedir, wal, this.conf); + HRegion region = HRegion.openHRegion(hri, wal, this.conf); Path f = new Path(basedir, "hfile"); HFile.Writer writer = new HFile.Writer(this.fs, f); byte [] family = hri.getTableDesc().getFamilies().iterator().next().getName(); @@ -327,7 +328,7 @@ public class TestWALReplay { HLog wal = createWAL(this.conf); final byte[] tableName = Bytes.toBytes(tableNameStr); final byte[] rowName = tableName; - final byte[] regionName = hri.getRegionName(); + final byte[] regionName = hri.getEncodedNameAsBytes(); // Add 1k to each family. final int countPerFamily = 1000; @@ -358,7 +359,6 @@ public class TestWALReplay { // Set down maximum recovery so we dfsclient doesn't linger retrying something // long gone. HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1); - // Make a new conf and a new fs for the splitter to run on so we can take // over old wal. Configuration newConf = HBaseTestingUtility.setDifferentUser(this.conf, @@ -396,6 +396,23 @@ public class TestWALReplay { } } + // Flusher used in this test. Keep count of how often we are called and + // actually run the flush inside here. + class TestFlusher implements FlushRequester { + private int count = 0; + private HRegion r; + + @Override + public void requestFlush(HRegion region) { + count++; + try { + r.flushcache(); + } catch (IOException e) { + throw new RuntimeException("Exception flushing", e); + } + } + } + private void addWALEdits (final byte [] tableName, final HRegionInfo hri, final byte [] rowName, final byte [] family, final int count, EnvironmentEdge ee, final HLog wal) @@ -464,7 +481,7 @@ public class TestWALReplay { * @throws IOException */ private HLog createWAL(final Configuration c) throws IOException { - HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c, null); + HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c); // Set down maximum recovery so we dfsclient doesn't linger retrying something // long gone. HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1); Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java?rev=991397&r1=991396&r2=991397&view=diff ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java (original) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java Tue Aug 31 23:51:44 2010 @@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.replicat import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface; import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager; @@ -39,7 +40,7 @@ public class ReplicationSourceDummy impl @Override public void init(Configuration conf, FileSystem fs, - ReplicationSourceManager manager, AtomicBoolean stopper, + ReplicationSourceManager manager, Stoppable stopper, AtomicBoolean replicating, String peerClusterId) throws IOException { this.manager = manager;