pair: getTableRegions(tableName)) {
- if (pair.getSecond() == null) continue; // undeployed
+ if (pair.getSecond() == null) {
+ continue; // undeployed
+ }
this.regionManager.startAction(pair.getFirst().getRegionName(),
pair.getFirst(), pair.getSecond(), op);
}
@@ -869,7 +933,9 @@ public class HMaster extends Thread impl
// an open or whatever.
this.regionManager.clearFromInTransition(regionname);
// If hostnameAndPort is still null, then none, exit.
- if (hostnameAndPort == null) break;
+ if (hostnameAndPort == null) {
+ break;
+ }
long startCode =
Bytes.toLong(rr.getValue(HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER));
@@ -905,8 +971,8 @@ public class HMaster extends Thread impl
*/
@Override
public void process(WatchedEvent event) {
- LOG.debug("Event " + event.getType() +
- " with state " + event.getState() +
+ LOG.debug("Event " + event.getType() +
+ " with state " + event.getState() +
" with path " + event.getPath());
// Master should kill itself if its session expired or if its
// znode was deleted manually (usually for testing purposes)
@@ -925,15 +991,13 @@ public class HMaster extends Thread impl
zooKeeperWrapper =
new ZooKeeperWatcher(conf, HMaster.class.getName(), this);
zooKeeperWrapper.registerListener(this);
- this.zkMasterAddressWatcher.setZookeeper(zooKeeperWrapper);
- if(!this.zkMasterAddressWatcher.
- writeAddressToZooKeeper(this.address,false)) {
- throw new Exception("Another Master is currently active");
- }
+ activeMasterManager = new ActiveMasterManager(zooKeeperWrapper,
+ this.address, this);
+ activeMasterManager.blockUntilBecomingActiveMaster();
- // we are a failed over master, reset the fact that we started the
+ // we are a failed over master, reset the fact that we started the
// cluster
- resetClusterStartup();
+ setClusterStartup(false);
// Verify the cluster to see if anything happened while we were away
joinCluster();
} catch (Exception e) {
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java?rev=963935&r1=963934&r2=963935&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java Wed Jul 14 03:41:11 2010
@@ -25,10 +25,10 @@ import org.apache.hadoop.hbase.ServerSta
import org.apache.hadoop.hbase.client.ServerConnection;
/**
- * These are the set of functions implemented by the HMaster and accessed by
+ * These are the set of functions implemented by the HMaster and accessed by
* the other packages in the master.
- *
- * TODO: this list has to be cleaned up, this is a re-factor only change that
+ *
+ * TODO: this list has to be cleaned up, this is a re-factor only change that
* preserves the functions in the interface.
*/
public interface MasterStatus extends ServerStatus {
@@ -42,26 +42,32 @@ public interface MasterStatus extends Se
* Return the region manager for region related info
*/
public RegionManager getRegionManager();
-
+
/**
* Return the file system manager for dealing with FS related stuff
*/
public FileSystemManager getFileSystemManager();
/**
- * Is this the master that is starting the cluster up? If true, yes.
+ * Is this the master that is starting the cluster up? If true, yes.
* Otherwise this is a failed over master.
*/
public boolean isClusterStartup();
/**
+ * Set whether this is a cluster starting up.
+ * @param isClusterStartup whether this is a cluster startup or failover
+ */
+ public void setClusterStartup(boolean isClusterStartup);
+
+ /**
* Return the server RPC connection
*/
public ServerConnection getServerConnection();
-
- // TODO: the semantics of the following methods should be defined. Once that
+
+ // TODO: the semantics of the following methods should be defined. Once that
// is clear, most of these should move to server status
-
+
// start shutting down the server
public void startShutdown();
// is a shutdown requested
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=963935&r1=963934&r2=963935&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java Wed Jul 14 03:41:11 2010
@@ -154,8 +154,6 @@ public class RegionManager {
HConstants.DEFAULT_ZOOKEEPER_RETRIES);
zooKeeperPause = conf.getInt(HConstants.ZOOKEEPER_PAUSE,
HConstants.DEFAULT_ZOOKEEPER_PAUSE);
-
- reassignRootRegion();
}
void start() {
@@ -174,7 +172,7 @@ public class RegionManager {
}
}
- void reassignRootRegion() {
+ public void reassignRootRegion() {
unsetRootRegion();
if (!masterStatus.getShutdownRequested().get()) {
synchronized (regionsInTransition) {
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java?rev=963935&r1=963934&r2=963935&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ZKMasterAddressWatcher.java Wed Jul 14 03:41:11 2010
@@ -1,129 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.EventType;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-
-/**
- * ZooKeeper watcher for the master address. Also watches the cluster state
- * flag so will shutdown this master if cluster has been shutdown.
- * Used by the Master. Waits on the master address ZNode delete event. When
- * multiple masters are brought up, they race to become master by writing their
- * address to ZooKeeper. Whoever wins becomes the master, and the rest wait for
- * that ephemeral node in ZooKeeper to evaporate (meaning the master went down),
- * at which point they try to write their own address to become the new master.
- */
-class ZKMasterAddressWatcher implements Watcher {
- private static final Log LOG = LogFactory.getLog(ZKMasterAddressWatcher.class);
-
- private ZooKeeperWrapper zookeeper;
- private final AtomicBoolean requestShutdown;
-
- /**
- * Create this watcher using passed ZooKeeperWrapper instance.
- * @param zk ZooKeeper
- * @param flag Flag to set to request shutdown.
- */
- ZKMasterAddressWatcher(final ZooKeeperWrapper zk, final AtomicBoolean flag) {
- this.requestShutdown = flag;
- this.zookeeper = zk;
- }
-
- /**
- * @see org.apache.zookeeper.Watcher#process(org.apache.zookeeper.WatchedEvent)
- */
- @Override
- public synchronized void process (WatchedEvent event) {
- EventType type = event.getType();
- LOG.debug(("Got event " + type + " with path " + event.getPath()));
- if (type.equals(EventType.NodeDeleted)) {
- if (event.getPath().equals(this.zookeeper.clusterStateZNode)) {
- LOG.info("Cluster shutdown while waiting, shutting down" +
- " this master.");
- this.requestShutdown.set(true);
- } else {
- LOG.debug("Master address ZNode deleted, notifying waiting masters");
- notifyAll();
- }
- } else if(type.equals(EventType.NodeCreated) &&
- event.getPath().equals(this.zookeeper.clusterStateZNode)) {
- LOG.debug("Resetting watch on cluster state node.");
- this.zookeeper.setClusterStateWatch();
- }
- }
-
- /**
- * Wait for master address to be available. This sets a watch in ZooKeeper and
- * blocks until the master address ZNode gets deleted.
- */
- public synchronized void waitForMasterAddressAvailability() {
- while (zookeeper.readMasterAddress(this) != null) {
- try {
- LOG.debug("Waiting for master address ZNode to be deleted " +
- "(Also watching cluster state node)");
- this.zookeeper.setClusterStateWatch();
- wait();
- } catch (InterruptedException e) {
- }
- }
- }
-
- /**
- * Write address to zookeeper. Parks here until we successfully write our
- * address (or until cluster shutdown).
- * @param address Address whose format is HServerAddress.toString
- */
- boolean writeAddressToZooKeeper(
- final HServerAddress address, boolean retry) {
- do {
- waitForMasterAddressAvailability();
- // Check if we need to shutdown instead of taking control
- if (this.requestShutdown.get()) {
- LOG.debug("Won't start Master because cluster is shuting down");
- return false;
- }
- if(this.zookeeper.writeMasterAddress(address)) {
- this.zookeeper.setClusterState(true);
- this.zookeeper.setClusterStateWatch();
- // Watch our own node
- this.zookeeper.readMasterAddress(this);
- return true;
- }
- } while(retry);
- return false;
- }
-
- /**
- * Reset the ZK in case a new connection is required
- * @param zookeeper new instance
- */
- public void setZookeeper(ZooKeeperWrapper zookeeper) {
- this.zookeeper = zookeeper;
- }
-}
\ No newline at end of file
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=963935&r1=963934&r2=963935&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Wed Jul 14 03:41:11 2010
@@ -327,7 +327,7 @@ public class HRegionServer implements HR
zooKeeper = new ZooKeeperWatcher(conf, serverInfo.getServerName(), this);
// create the master address manager, register with zk, and start it
- masterAddressManager = new MasterAddressManager(zooKeeper);
+ masterAddressManager = new MasterAddressManager(zooKeeper, this);
zooKeeper.registerListener(masterAddressManager);
masterAddressManager.monitorMaster();
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java?rev=963935&r1=963934&r2=963935&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java Wed Jul 14 03:41:11 2010
@@ -22,21 +22,23 @@ package org.apache.hadoop.hbase.regionse
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.ServerStatus;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
/**
* Manages the location of the current active Master for this RegionServer.
- *
- * Listens for ZooKeeper events related to the master address. The node /master
- * will contain the address of the current master. This listener is interested
+ *
+ * Listens for ZooKeeper events related to the master address. The node /master
+ * will contain the address of the current master. This listener is interested
* in NodeDeleted and NodeCreated events on /master.
- *
+ *
* This class is thread-safe and takes care of re-setting all watchers to
* ensure it always knows the up-to-date master. To kick it off, instantiate
* the class and run the {@link #monitorMaster()} method.
- *
+ *
* You can get the current master via {@link #getMasterAddress()} or the
* blocking method {@link #waitMasterAddress()}.
*/
@@ -45,25 +47,29 @@ public class MasterAddressManager extend
// Address of the current primary master, null if no primary master
private HServerAddress masterAddress;
-
+
+ // Status and controller for the regionserver
+ private ServerStatus status;
+
/**
* Construct a master address listener with the specified zookeeper reference.
- *
+ *
* This constructor does not trigger any actions, you must call methods
* explicitly. Normally you will just want to execute {@link #monitorMaster()}
- * and you will ensure to
- *
+ * and you will ensure to
+ *
* @param watcher zk reference and watcher
*/
- public MasterAddressManager(ZooKeeperWatcher watcher) {
+ public MasterAddressManager(ZooKeeperWatcher watcher, ServerStatus status) {
super(watcher);
- masterAddress = null;
+ this.status = status;
+ this.masterAddress = null;
}
-
+
/**
* Get the address of the current master if one is available. Returns null
* if no current master.
- *
+ *
* Use {@link #waitMasterAddress} if you want to block until the master is
* available.
* @return server address of current active master, or null if none available
@@ -71,7 +77,7 @@ public class MasterAddressManager extend
public synchronized HServerAddress getMasterAddress() {
return masterAddress;
}
-
+
/**
* Check if there is a master available.
* @return true if there is a master set, false if not.
@@ -79,14 +85,14 @@ public class MasterAddressManager extend
public synchronized boolean hasMaster() {
return masterAddress != null;
}
-
+
/**
* Get the address of the current master. If no master is available, method
* will block until one is available, the thread is interrupted, or timeout
* has passed.
- *
+ *
* TODO: Make this work, currently unused, kept with existing retry semantics.
- *
+ *
* @return server address of current active master, null if timed out
* @throws InterruptedException if the thread is interrupted while waiting
*/
@@ -94,22 +100,28 @@ public class MasterAddressManager extend
throws InterruptedException {
return masterAddress;
}
-
+
/**
* Setup to watch for the primary master of the cluster.
- *
+ *
* If the master is already available in ZooKeeper, this method will ensure
* it gets set and that any further changes are also watched for.
- *
+ *
* If no master is available, this method ensures we become aware of it and
* will take care of setting it.
*/
public void monitorMaster() {
- if(ZKUtil.watchAndCheckExists(watcher, watcher.masterAddressZNode)) {
- handleNewMaster();
+ try {
+ if(ZKUtil.watchAndCheckExists(watcher, watcher.masterAddressZNode)) {
+ handleNewMaster();
+ }
+ } catch(KeeperException ke) {
+ // If we have a ZK exception trying to find the master we must abort
+ LOG.fatal("Unexpected ZooKeeper exception", ke);
+ status.abortServer();
}
}
-
+
@Override
public void nodeCreated(String path) {
LOG.info("nodeCreated(" + path + ")");
@@ -118,7 +130,7 @@ public class MasterAddressManager extend
}
monitorMaster();
}
-
+
@Override
public void nodeDeleted(String path) {
if(path.equals(watcher.masterAddressZNode)) {
@@ -126,7 +138,7 @@ public class MasterAddressManager extend
}
monitorMaster();
}
-
+
/**
* Set the master address to the specified address. This operation is
* idempotent, a master will only be set if there is currently no master set.
@@ -137,7 +149,7 @@ public class MasterAddressManager extend
masterAddress = address;
}
}
-
+
/**
* Unsets the master address. Used when the master goes offline so none is
* available.
@@ -148,34 +160,40 @@ public class MasterAddressManager extend
masterAddress = null;
}
}
-
+
/**
* Handle a new master being set.
- *
+ *
* This method should be called to check if there is a new master. If there
* is already a master set, this method returns immediately. If none is set,
* this will attempt to grab the master location from ZooKeeper and will set
* it.
- *
- * This method uses an atomic operation to ensure a new master is only set
+ *
+ * This method uses an atomic operation to ensure a new master is only set
* once.
*/
private void handleNewMaster() {
if(hasMaster()) {
return;
}
- HServerAddress address =
- ZKUtil.getDataAsAddress(watcher, watcher.masterAddressZNode);
+ HServerAddress address = null;
+ try {
+ address = ZKUtil.getDataAsAddress(watcher, watcher.masterAddressZNode);
+ } catch (KeeperException ke) {
+ // If we have a ZK exception trying to find the master we must abort
+ LOG.fatal("Unexpected ZooKeeper exception", ke);
+ status.abortServer();
+ }
if(address != null) {
setMasterAddress(address);
}
}
-
+
/**
* Handle a master failure.
- *
+ *
* Triggered when a master node is deleted.
- *
+ *
* TODO: Other ways we figure master is "dead"? What do we do if set in ZK
* but we can't communicate with TCP?
*/
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java?rev=963935&r1=963934&r2=963935&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java Wed Jul 14 03:41:11 2010
@@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.zookeeper;
import java.io.IOException;
+import java.util.List;
import java.util.Properties;
import org.apache.commons.logging.Log;
@@ -27,28 +28,34 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat;
/**
* Internal HBase utility class for ZooKeeper.
- *
- * Contains only static methods and constants.
+ *
+ *
Contains only static methods and constants.
+ *
+ *
Methods all throw {@link KeeperException} if there is an unexpected
+ * zookeeper exception, so callers of these methods must handle appropriately.
+ * If ZK is required for the operation, the server will need to be aborted.
*/
public class ZKUtil {
private static final Log LOG = LogFactory.getLog(ZKUtil.class);
// TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
private static final char ZNODE_PATH_SEPARATOR = '/';
-
+
/**
* Creates a new connection to ZooKeeper, pulling settings and quorum config
* from the specified configuration object using methods from {@link ZKConfig}.
- *
+ *
* Sets the connection status monitoring watcher to the specified watcher.
- *
+ *
* @param conf configuration to pull quorum and other settings from
* @param watcher watcher to monitor connection changes
* @return connection to zookeeper
@@ -66,12 +73,16 @@ public class ZKUtil {
return new ZooKeeper(quorum, timeout, watcher);
}
+ //
+ // Helper methods
+ //
+
/**
* Join the prefix znode name with the suffix znode name to generate a proper
* full znode name.
- *
+ *
* Assumes prefix does not end with slash and suffix does not begin with it.
- *
+ *
* @param prefix beginning of znode name
* @param suffix ending of znode name
* @return result of properly joining prefix with suffix
@@ -80,16 +91,22 @@ public class ZKUtil {
return prefix + ZNODE_PATH_SEPARATOR + suffix;
}
+ //
+ // Existence checks and watches
+ //
+
/**
* Watch the specified znode for delete/create/change events. The watcher is
* set whether or not the node exists. If the node already exists, the method
* returns true. If the node does not exist, the method returns false.
- *
+ *
* @param zkw zk reference
* @param znode path of node to watch
* @return true if znode exists, false if does not exist or error
+ * @throws KeeperException if unexpected zookeeper exception
*/
- public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode) {
+ public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
+ throws KeeperException {
try {
Stat s = zkw.getZooKeeper().exists(znode, zkw);
zkw.debug("Set watcher on existing znode (" + znode + ")");
@@ -104,21 +121,103 @@ public class ZKUtil {
return false;
}
}
-
+
+ //
+ // Znode listings
+ //
+
+ /**
+ * Lists the children znodes of the specified znode. Also sets a watch on
+ * the specified znode which will capture a NodeDeleted event on the specified
+ * znode as well as NodeChildrenChanged if any children of the specified znode
+ * are created or deleted.
+ *
+ * Returns null if the specified node does not exist. Otherwise returns a
+ * list of children of the specified node. If the node exists but it has no
+ * children, an empty list will be returned.
+ *
+ * @param zkw zk reference
+ * @param znode path of node to list and watch children of
+ * @returns list of children of the specified node, an empty list if the node
+ * exists but has no children, and null if the node does not exist
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static List listChildrenAndWatchForNewChildren(
+ ZooKeeperWatcher zkw, String znode)
+ throws KeeperException {
+ try {
+ List children = zkw.getZooKeeper().getChildren(znode, zkw);
+ return children;
+ } catch(KeeperException.NoNodeException ke) {
+ zkw.debug("Unable to list children of znode (" + znode + ") " +
+ "because node does not exist (not an error)");
+ return null;
+ } catch (KeeperException e) {
+ zkw.warn("Unable to list children of znode (" + znode + ")", e);
+ zkw.keeperException(e);
+ return null;
+ } catch (InterruptedException e) {
+ zkw.warn("Unable to list children of znode (" + znode + ")", e);
+ zkw.interruptedException(e);
+ return null;
+ }
+ }
+
+ /**
+ * Checks if the specified znode has any children. Sets no watches.
+ *
+ * Returns true if the node exists and has children. Returns false if the
+ * node does not exist or if the node does not have any children.
+ *
+ * Used during master initialization to determine if the master is a
+ * failed-over-to master or the first master during initial cluster startup.
+ * If the directory for regionserver ephemeral nodes is empty then this is
+ * a cluster startup, if not then it is not cluster startup.
+ *
+ * @param zkw zk reference
+ * @param znode path of node to check for children of
+ * @return true if node has children, false if not or node does not exist
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
+ throws KeeperException {
+ try {
+ return !zkw.getZooKeeper().getChildren(znode, null).isEmpty();
+ } catch(KeeperException.NoNodeException ke) {
+ zkw.debug("Unable to list children of znode (" + znode + ") " +
+ "because node does not exist (not an error)");
+ return false;
+ } catch (KeeperException e) {
+ zkw.warn("Unable to list children of znode (" + znode + ")", e);
+ zkw.keeperException(e);
+ return false;
+ } catch (InterruptedException e) {
+ zkw.warn("Unable to list children of znode (" + znode + ")", e);
+ zkw.interruptedException(e);
+ return false;
+ }
+ }
+
+ //
+ // Data retrieval
+ //
+
/**
* Get the data at the specified znode and set a watch.
- *
+ *
* Returns the data and sets a watch if the node exists. Returns null and no
* watch is set if the node does not exist or there is an exception.
- *
+ *
* @param zkw zk reference
* @param znode path of node
* @return data of the specified znode, or null
+ * @throws KeeperException if unexpected zookeeper exception
*/
- public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode) {
+ public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
+ throws KeeperException {
try {
byte [] data = zkw.getZooKeeper().getData(znode, zkw, null);
- zkw.debug("Retrieved " + data.length + " bytes of data from znode (" +
+ zkw.debug("Retrieved " + data.length + " bytes of data from znode (" +
znode + ") and set a watcher");
return data;
} catch (KeeperException.NoNodeException e) {
@@ -135,21 +234,23 @@ public class ZKUtil {
return null;
}
}
-
+
/**
* Get the data at the specified znode, deserialize it as an HServerAddress,
* and set a watch.
- *
+ *
* Returns the data as a server address and sets a watch if the node exists.
* Returns null and no watch is set if the node does not exist or there is an
* exception.
- *
+ *
* @param zkw zk reference
* @param znode path of node
* @return data of the specified node as a server address, or null
+ * @throws KeeperException if unexpected zookeeper exception
*/
- public static HServerAddress getDataAsAddress(ZooKeeperWatcher zkw,
- String znode) {
+ public static HServerAddress getDataAsAddress(ZooKeeperWatcher zkw,
+ String znode)
+ throws KeeperException {
byte [] data = getDataAndWatch(zkw, znode);
if(data == null) {
return null;
@@ -158,4 +259,86 @@ public class ZKUtil {
zkw.debug("Read server address from znode (" + znode + "): " + addrString);
return new HServerAddress(addrString);
}
+
+ /**
+ * Set the specified znode to be an ephemeral node carrying the specified
+ * server address. Used by masters for their ephemeral node and regionservers
+ * for their ephemeral node.
+ *
+ * If the node is created successfully, a watcher is also set on the node.
+ *
+ * If the node is not created successfully because it already exists, this
+ * method will also set a watcher on the node.
+ *
+ * If there is another problem, a KeeperException will be thrown.
+ *
+ * @param zkw zk reference
+ * @param znode path of node
+ * @param address server address
+ * @return true if address set, false if not, watch set in both cases
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static boolean setAddressAndWatch(ZooKeeperWatcher zkw,
+ String znode, HServerAddress address)
+ throws KeeperException {
+ return createEphemeralNodeAndWatch(zkw, znode,
+ Bytes.toBytes(address.toString()));
+ }
+ /**
+ *
+ * Set the specified znode to be an ephemeral node carrying the specified
+ * data.
+ *
+ * If the node is created successfully, a watcher is also set on the node.
+ *
+ * If the node is not created successfully because it already exists, this
+ * method will also set a watcher on the node.
+ *
+ * If there is another problem, a KeeperException will be thrown.
+ *
+ * @param zkw zk reference
+ * @param znode path of node
+ * @param data data of node
+ * @return true if node created, false if not, watch set in both cases
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
+ String znode, byte [] data)
+ throws KeeperException {
+ try {
+ zkw.getZooKeeper().create(znode, data, Ids.OPEN_ACL_UNSAFE,
+ CreateMode.EPHEMERAL);
+ } catch (KeeperException.NodeExistsException nee) {
+ if(!watchAndCheckExists(zkw, znode)) {
+ // It did exist but now it doesn't, try again
+ return createEphemeralNodeAndWatch(zkw, znode, data);
+ }
+ return false;
+ } catch (InterruptedException e) {
+ LOG.info("Interrupted", e);
+ }
+ return true;
+ }
+
+ /**
+ * Creates the specified node, if the node does not exist. Does not set a
+ * watch and fails silently if the node already exists.
+ *
+ * The node created is persistent and open access.
+ *
+ * @param zkw zk reference
+ * @param znode path of node
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static void createIfNotExists(ZooKeeperWatcher zkw,
+ String znode)
+ throws KeeperException {
+ try {
+ zkw.getZooKeeper().create(znode, new byte[0], Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ } catch(KeeperException.NodeExistsException nee) {
+ } catch(InterruptedException ie) {
+ zkw.interruptedException(ie);
+ }
+ }
}
\ No newline at end of file
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java?rev=963935&r1=963934&r2=963935&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java Wed Jul 14 03:41:11 2010
@@ -88,6 +88,12 @@ public class ZooKeeperWatcher extends Zo
this.server = server;
info("Connected to ZooKeeper");
setNodeNames(conf);
+ try {
+ ZKUtil.createIfNotExists(this, baseZNode);
+ } catch (KeeperException e) {
+ error("Unexpected KeeperException creating base node", e);
+ throw new IOException(e);
+ }
}
/**
@@ -198,9 +204,10 @@ public class ZooKeeperWatcher extends Zo
// Abort the server if Disconnected or Expired
// TODO: Ã
ny reason to handle these two differently?
case Disconnected:
+ info("Received Disconnected from ZooKeeper, ignoring");
+ break;
case Expired:
- error("Received Disconnected/Expired [" + event.getState() + "] " +
- "from ZooKeeper, aborting server");
+ error("Received Expired from ZooKeeper, aborting server");
if(server != null) {
server.abortServer();
}
@@ -213,13 +220,15 @@ public class ZooKeeperWatcher extends Zo
*
* This may be temporary but for now this gives one place to deal with these.
*
- * TODO: Currently this method aborts the server.
+ * TODO: Currently this method rethrows the exception to let the caller handle
*
* @param ke
+ * @throws KeeperException
*/
- public void keeperException(KeeperException ke) {
- error("Received unexpected KeeperException, aborting server", ke);
- server.abortServer();
+ public void keeperException(KeeperException ke)
+ throws KeeperException {
+ error("Received unexpected KeeperException, re-throwing exception", ke);
+ throw ke;
}
/**
@@ -229,6 +238,7 @@ public class ZooKeeperWatcher extends Zo
*
* TODO: Currently, this method does nothing.
* Is this ever expected to happen? Do we abort or can we let it run?
+ * Maybe this should be logged as WARN? It shouldn't happen?
*
* @param ie
*/
Added: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java?rev=963935&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java (added)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java Wed Jul 14 03:41:11 2010
@@ -0,0 +1,269 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.client.ServerConnection;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestActiveMasterManager {
+ private static final Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
+
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.startMiniZKCluster();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniZKCluster();
+ }
+ /**
+ * Unit tests that uses ZooKeeper but does not use the master-side methods
+ * but rather acts directly on ZK.
+ * @throws Exception
+ */
+ @Test
+ public void testActiveMasterManagerFromZK() throws Exception {
+
+ ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+ "testActiveMasterManagerFromZK", null);
+ zk.createZNodeIfNotExists(zk.baseZNode);
+ try {
+ zk.deleteZNode(zk.masterAddressZNode);
+ } catch(KeeperException.NoNodeException nne) {}
+
+ // Create the master node with a dummy address
+ HServerAddress firstMasterAddress = new HServerAddress("firstMaster", 1234);
+ HServerAddress secondMasterAddress = new HServerAddress("secondMaster", 1234);
+
+ // Should not have a master yet
+ DummyMasterStatus ms1 = new DummyMasterStatus();
+ ActiveMasterManager activeMasterManager = new ActiveMasterManager(zk,
+ firstMasterAddress, ms1);
+ zk.registerListener(activeMasterManager);
+ assertFalse(activeMasterManager.clusterHasActiveMaster.get());
+
+ // First test becoming the active master uninterrupted
+ activeMasterManager.blockUntilBecomingActiveMaster();
+ assertTrue(activeMasterManager.clusterHasActiveMaster.get());
+ assertMaster(zk, firstMasterAddress);
+
+ // New manager will now try to become the active master in another thread
+ WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
+ zk.registerListener(t.manager);
+ t.start();
+ // Wait for this guy to figure out there is another active master
+ // Wait for 1 second at most
+ int sleeps = 0;
+ while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
+ Thread.sleep(10);
+ sleeps++;
+ }
+
+ // Both should see that there is an active master
+ assertTrue(activeMasterManager.clusterHasActiveMaster.get());
+ assertTrue(t.manager.clusterHasActiveMaster.get());
+ // But secondary one should not be the active master
+ assertFalse(t.isActiveMaster);
+
+ // Close the first server and delete it's master node
+ ms1.setClosed();
+
+ // Use a listener to capture when the node is actually deleted
+ NodeDeletionListener listener = new NodeDeletionListener(zk, zk.masterAddressZNode);
+ zk.registerListener(listener);
+
+ LOG.info("Deleting master node");
+ zk.deleteZNode(zk.masterAddressZNode);
+
+ // Wait for the node to be deleted
+ LOG.info("Waiting for active master manager to be notified");
+ listener.waitForDeletion();
+ LOG.info("Master node deleted");
+
+ // Now we expect the secondary manager to have and be the active master
+ // Wait for 1 second at most
+ sleeps = 0;
+ while(!t.isActiveMaster && sleeps < 100) {
+ Thread.sleep(10);
+ sleeps++;
+ }
+ LOG.debug("Slept " + sleeps + " times");
+
+ assertTrue(t.manager.clusterHasActiveMaster.get());
+ assertTrue(t.isActiveMaster);
+ }
+
+ /**
+ * Assert there is an active master and that it has the specified address.
+ * @param zk
+ * @param thisMasterAddress
+ * @throws KeeperException
+ */
+ private void assertMaster(ZooKeeperWatcher zk,
+ HServerAddress expectedAddress) throws KeeperException {
+ HServerAddress readAddress = ZKUtil.getDataAsAddress(zk, zk.masterAddressZNode);
+ assertNotNull(readAddress);
+ assertTrue(expectedAddress.equals(readAddress));
+ }
+
+ public static class WaitToBeMasterThread extends Thread {
+
+ ActiveMasterManager manager;
+ boolean isActiveMaster;
+
+ public WaitToBeMasterThread(ZooKeeperWatcher zk,
+ HServerAddress address) {
+ this.manager = new ActiveMasterManager(zk, address,
+ new DummyMasterStatus());
+ isActiveMaster = false;
+ }
+
+ @Override
+ public void run() {
+ manager.blockUntilBecomingActiveMaster();
+ LOG.info("Second master has become the active master!");
+ isActiveMaster = true;
+ }
+ }
+
+ public static class NodeDeletionListener extends ZooKeeperListener {
+ private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
+
+ private Semaphore lock;
+ private String node;
+
+ public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
+ super(watcher);
+ lock = new Semaphore(0);
+ this.node = node;
+ }
+
+ @Override
+ public void nodeDeleted(String path) {
+ if(path.equals(node)) {
+ LOG.debug("nodeDeleted(" + path + ")");
+ lock.release();
+ }
+ }
+
+ public void waitForDeletion() throws InterruptedException {
+ lock.acquire();
+ }
+ }
+
+ public static class DummyMasterStatus implements MasterStatus {
+
+ private AtomicBoolean closed = new AtomicBoolean(false);
+
+ @Override
+ public AtomicBoolean getClosed() {
+ return closed;
+ }
+
+ @Override
+ public FileSystemManager getFileSystemManager() {
+ return null;
+ }
+
+ @Override
+ public RegionManager getRegionManager() {
+ return null;
+ }
+
+ @Override
+ public ServerConnection getServerConnection() {
+ return null;
+ }
+
+ @Override
+ public ServerManager getServerManager() {
+ return null;
+ }
+
+ @Override
+ public AtomicBoolean getShutdownRequested() {
+ return null;
+ }
+
+ @Override
+ public boolean isClosed() {
+ return closed.get();
+ }
+
+ @Override
+ public boolean isClusterStartup() {
+ return false;
+ }
+
+ @Override
+ public void setClosed() {
+ closed.set(true);
+ }
+
+ @Override
+ public void setClusterStartup(boolean isClusterStartup) {}
+
+ @Override
+ public void shutdown() {}
+
+ @Override
+ public void startShutdown() {}
+
+ @Override
+ public void abortServer() {}
+
+ @Override
+ public Configuration getConfiguration() {
+ return null;
+ }
+
+ @Override
+ public HServerAddress getHServerAddress() {
+ return null;
+ }
+
+ @Override
+ public ZooKeeperWatcher getZooKeeper() {
+ return null;
+ }
+
+ }
+}
Modified: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java?rev=963935&r1=963934&r2=963935&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java Wed Jul 14 03:41:11 2010
@@ -40,12 +40,12 @@ public class TestMasterAddressManager {
private static final Log LOG = LogFactory.getLog(TestMasterAddressManager.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniZKCluster();
}
-
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniZKCluster();
@@ -57,29 +57,29 @@ public class TestMasterAddressManager {
*/
@Test
public void testMasterAddressManagerFromZK() throws Exception {
-
- ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
+
+ ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
"testMasterAddressManagerFromZK", null);
zk.createZNodeIfNotExists(zk.baseZNode);
-
+
// Should not have a master yet
- MasterAddressManager addressManager = new MasterAddressManager(zk);
+ MasterAddressManager addressManager = new MasterAddressManager(zk, null);
addressManager.monitorMaster();
assertFalse(addressManager.hasMaster());
zk.registerListener(addressManager);
-
+
// Use a listener to capture when the node is actually created
NodeCreationListener listener = new NodeCreationListener(zk, zk.masterAddressZNode);
zk.registerListener(listener);
-
+
// Create the master node with a dummy address
String host = "hostname";
int port = 1234;
HServerAddress dummyAddress = new HServerAddress(host, port);
LOG.info("Creating master node");
- zk.createZNodeIfNotExists(zk.masterAddressZNode,
+ zk.createZNodeIfNotExists(zk.masterAddressZNode,
Bytes.toBytes(dummyAddress.toString()), CreateMode.EPHEMERAL, false);
-
+
// Wait for the node to be created
LOG.info("Waiting for master address manager to be notified");
listener.waitForCreation();
@@ -87,21 +87,21 @@ public class TestMasterAddressManager {
assertTrue(addressManager.hasMaster());
HServerAddress pulledAddress = addressManager.getMasterAddress();
assertTrue(pulledAddress.equals(dummyAddress));
-
+
}
-
+
public static class NodeCreationListener extends ZooKeeperListener {
private static final Log LOG = LogFactory.getLog(NodeCreationListener.class);
-
+
private Semaphore lock;
private String node;
-
+
public NodeCreationListener(ZooKeeperWatcher watcher, String node) {
super(watcher);
lock = new Semaphore(0);
this.node = node;
}
-
+
@Override
public void nodeCreated(String path) {
if(path.equals(node)) {
@@ -109,7 +109,7 @@ public class TestMasterAddressManager {
lock.release();
}
}
-
+
public void waitForCreation() throws InterruptedException {
lock.acquire();
}