Return-Path: Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: (qmail 10867 invoked from network); 22 Jun 2010 18:25:05 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 22 Jun 2010 18:25:05 -0000 Received: (qmail 23131 invoked by uid 500); 22 Jun 2010 18:25:05 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 23055 invoked by uid 500); 22 Jun 2010 18:25:04 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 23048 invoked by uid 99); 22 Jun 2010 18:25:04 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 22 Jun 2010 18:25:04 +0000 X-ASF-Spam-Status: No, hits=-1405.3 required=10.0 tests=ALL_TRUSTED,AWL X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 22 Jun 2010 18:25:02 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id BFED32388980; Tue, 22 Jun 2010 18:24:12 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r956968 - in /hbase/trunk: ./ src/main/java/org/apache/hadoop/hbase/executor/ src/main/java/org/apache/hadoop/hbase/master/ src/main/java/org/apache/hadoop/hbase/master/handler/ src/test/java/org/apache/hadoop/hbase/ Date: Tue, 22 Jun 2010 18:24:12 -0000 To: commits@hbase.apache.org From: jdcryans@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100622182412.BFED32388980@eris.apache.org> Author: jdcryans Date: Tue Jun 22 18:24:12 2010 New Revision: 956968 URL: http://svn.apache.org/viewvc?rev=956968&view=rev Log: HBASE-2741 HBaseExecutorService needs to be multi-cluster friendly (Karthik Ranganathan via JD) Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestMultiClusters.java Modified: hbase/trunk/CHANGES.txt hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java Modified: hbase/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=956968&r1=956967&r2=956968&view=diff ============================================================================== --- hbase/trunk/CHANGES.txt (original) +++ hbase/trunk/CHANGES.txt Tue Jun 22 18:24:12 2010 @@ -404,6 +404,8 @@ Release 0.21.0 - Unreleased the first row in a table. HBASE-1025 Reconstruction log playback has no bounds on memory used HBASE-2757 Fix flaky TestFromClientSide test by forcing region assignment + HBASE-2741 HBaseExecutorService needs to be multi-cluster friendly + (Karthik Ranganathan via JD) IMPROVEMENTS HBASE-1760 Cleanup TODOs in HTable Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java?rev=956968&r1=956967&r2=956968&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java Tue Jun 22 18:24:12 2010 @@ -60,18 +60,7 @@ public abstract class HBaseEventHandler // listeners that are called before and after an event is processed protected static List eventHandlerListeners = Collections.synchronizedList(new ArrayList()); - // static instances needed by the handlers - protected static ServerManager serverManager; - - /** - * Note that this has to be called first BEFORE the subclass constructors. - * - * TODO: take out after refactor - */ - public static void init(ServerManager serverManager) { - HBaseEventHandler.serverManager = serverManager; - } - + /** * This interface provides hooks to listen to various events received by the * queue. A class implementing this can listen to the updates by calling @@ -124,7 +113,7 @@ public abstract class HBaseEventHandler case RS2ZK_REGION_OPENING: case RS2ZK_REGION_OPENED: - executorServiceType = HBaseExecutorServiceType.MASTER_CLOSEREGION; + executorServiceType = HBaseExecutorServiceType.MASTER_OPENREGION; break; case M2ZK_REGION_OFFLINE: @@ -212,7 +201,11 @@ public abstract class HBaseEventHandler } // call the main process function - process(); + try { + process(); + } catch(Throwable t) { + LOG.error("Caught throwable while processing event " + eventType, t); + } // fire all afterProcess listeners for(HBaseEventHandlerListener listener : eventHandlerListeners) { Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=956968&r1=956967&r2=956968&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Tue Jun 22 18:24:12 2010 @@ -222,13 +222,11 @@ public class HMaster extends Thread impl // Start the unassigned watcher - which will create the unassgined region // in ZK. This is needed before RegionManager() constructor tries to assign // the root region. - ZKUnassignedWatcher.start(this.conf); - // init the various event handlers - HBaseEventHandler.init(serverManager); + ZKUnassignedWatcher.start(this.conf, serverManager, address.toString()); // start the "close region" executor service - HBaseEventType.RS2ZK_REGION_CLOSED.startMasterExecutorService(MASTER); + HBaseEventType.RS2ZK_REGION_CLOSED.startMasterExecutorService(address.toString()); // start the "open region" executor service - HBaseEventType.RS2ZK_REGION_OPENED.startMasterExecutorService(MASTER); + HBaseEventType.RS2ZK_REGION_OPENED.startMasterExecutorService(address.toString()); // start the region manager Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java?rev=956968&r1=956967&r2=956968&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java Tue Jun 22 18:24:12 2010 @@ -41,14 +41,20 @@ import org.apache.zookeeper.Watcher.Even public class ZKUnassignedWatcher implements Watcher { private static final Log LOG = LogFactory.getLog(ZKUnassignedWatcher.class); - private ZooKeeperWrapper zkWrapper = null; - - public static void start(Configuration conf) throws IOException { - new ZKUnassignedWatcher(conf); + private ZooKeeperWrapper zkWrapper; + String serverName; + ServerManager serverManager; + + public static void start(Configuration conf, ServerManager serverManager, + String serverName) throws IOException { + new ZKUnassignedWatcher(conf, serverManager, serverName); LOG.debug("Started ZKUnassigned watcher"); } - public ZKUnassignedWatcher(Configuration conf) throws IOException { + public ZKUnassignedWatcher(Configuration conf, ServerManager serverManager, + String serverName) throws IOException { + this.serverName = serverName; + this.serverManager = serverManager; zkWrapper = ZooKeeperWrapper.getInstance(conf, HMaster.class.getName()); // If the UNASSIGNED ZNode does not exist, create it. @@ -80,7 +86,8 @@ public class ZKUnassignedWatcher impleme // check if the path is for the UNASSIGNED directory we care about if(event.getPath() == null || - !event.getPath().startsWith(zkWrapper.getZNodePathForHBase(zkWrapper.getRegionInTransitionZNode()))) { + !event.getPath().startsWith(zkWrapper.getZNodePathForHBase( + zkWrapper.getRegionInTransitionZNode()))) { return; } @@ -107,10 +114,12 @@ public class ZKUnassignedWatcher impleme * If there were some nodes created then watch those nodes */ else if(type.equals(EventType.NodeChildrenChanged)) { - List newZNodes = zkWrapper.watchAndGetNewChildren(event.getPath()); + List newZNodes = + zkWrapper.watchAndGetNewChildren(event.getPath()); for(ZNodePathAndData zNodePathAndData : newZNodes) { LOG.debug("Handling updates for znode: " + zNodePathAndData.getzNodePath()); - handleRegionStateInZK(zNodePathAndData.getzNodePath(), zNodePathAndData.getData()); + handleRegionStateInZK(zNodePathAndData.getzNodePath(), + zNodePathAndData.getData()); } } } @@ -139,17 +148,18 @@ public class ZKUnassignedWatcher impleme return; } String rgnInTransitNode = zkWrapper.getRegionInTransitionZNode(); - String region = zNodePath.substring(zNodePath.indexOf(rgnInTransitNode) + rgnInTransitNode.length() + 1); + String region = zNodePath.substring( + zNodePath.indexOf(rgnInTransitNode) + rgnInTransitNode.length() + 1); HBaseEventType rsEvent = HBaseEventType.fromByte(data[0]); // if the node was CLOSED then handle it if(rsEvent == HBaseEventType.RS2ZK_REGION_CLOSED) { - new MasterCloseRegionHandler(rsEvent, region, data).submit(); + new MasterCloseRegionHandler(rsEvent, serverManager, serverName, region, data).submit(); } // if the region was OPENED then handle that else if(rsEvent == HBaseEventType.RS2ZK_REGION_OPENED || rsEvent == HBaseEventType.RS2ZK_REGION_OPENING) { - new MasterOpenRegionHandler(rsEvent, region, data).submit(); + new MasterOpenRegionHandler(rsEvent, serverManager, serverName, region, data).submit(); } } } Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java?rev=956968&r1=956967&r2=956968&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java Tue Jun 22 18:24:12 2010 @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.hbase.executor.RegionTransitionEventData; import org.apache.hadoop.hbase.executor.HBaseEventHandler; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.util.Writables; /** @@ -41,11 +42,17 @@ public class MasterCloseRegionHandler ex private String regionName; protected byte[] serializedData; RegionTransitionEventData hbEventData; + ServerManager serverManager; - public MasterCloseRegionHandler(HBaseEventType eventType, String regionName, byte[] serializedData) { - super(false, HMaster.MASTER, eventType); + public MasterCloseRegionHandler(HBaseEventType eventType, + ServerManager serverManager, + String serverName, + String regionName, + byte[] serializedData) { + super(false, serverName, eventType); this.regionName = regionName; this.serializedData = serializedData; + this.serverManager = serverManager; } /** Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java?rev=956968&r1=956967&r2=956968&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java Tue Jun 22 18:24:12 2010 @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HServerIn import org.apache.hadoop.hbase.executor.RegionTransitionEventData; import org.apache.hadoop.hbase.executor.HBaseEventHandler; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.util.Writables; /** @@ -46,11 +47,17 @@ public class MasterOpenRegionHandler ext protected byte[] serializedData; private String regionName; private RegionTransitionEventData hbEventData; + ServerManager serverManager; - public MasterOpenRegionHandler(HBaseEventType eventType, String regionName, byte[] serData) { - super(false, HMaster.MASTER, eventType); + public MasterOpenRegionHandler(HBaseEventType eventType, + ServerManager serverManager, + String serverName, + String regionName, + byte[] serData) { + super(false, serverName, eventType); this.regionName = regionName; this.serializedData = serData; + this.serverManager = serverManager; } /** Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestMultiClusters.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestMultiClusters.java?rev=956968&view=auto ============================================================================== --- hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestMultiClusters.java (added) +++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestMultiClusters.java Tue Jun 22 18:24:12 2010 @@ -0,0 +1,83 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Class that tests the multi-cluster in 1 JVM case, useful + * only for "unit'ish tests". + */ +public class TestMultiClusters { + + private static final byte[] TABLE_NAME = Bytes.toBytes("test"); + private static final byte[] FAM_NAME = Bytes.toBytes("fam"); + private static final byte[] ROW = Bytes.toBytes("row"); + private static final byte[] QUAL_NAME = Bytes.toBytes("qual"); + private static final byte[] VALUE = Bytes.toBytes("value"); + + /** + * Basic sanity test that spins up 2 HDFS and HBase clusters that share the + * same ZK ensemble. We then create the same table in both and make sure that + * what we insert in one place doesn't end up in the other. + * @throws Exception + */ + @Test (timeout=100000) + public void twoClusters() throws Exception{ + Configuration conf1 = HBaseConfiguration.create(); + // Different path for different clusters + conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + HBaseTestingUtility utility1 = new HBaseTestingUtility(conf1); + utility1.startMiniZKCluster(); + + Configuration conf2 = HBaseConfiguration.create(); + conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); + HBaseTestingUtility utility2 = new HBaseTestingUtility(conf2); + // They share the same ensemble, but homed differently + utility2.setZkCluster(utility1.getZkCluster()); + + utility1.startMiniCluster(); + utility2.startMiniCluster(); + + HTable table1 = utility1.createTable(TABLE_NAME, FAM_NAME); + HTable table2 = utility2.createTable(TABLE_NAME, FAM_NAME); + + Put put = new Put(ROW); + put.add(FAM_NAME, QUAL_NAME, VALUE); + table1.put(put); + + Get get = new Get(ROW); + get.addColumn(FAM_NAME, QUAL_NAME); + Result res = table1.get(get); + assertEquals(1, res.size()); + + res = table2.get(get); + assertEquals(0, res.size()); + } + +}