hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [01/13] hbase git commit: HBASE-19114 Split out o.a.h.h.zookeeper from hbase-server and hbase-client
Date Fri, 17 Nov 2017 21:20:33 GMT
Repository: hbase
Updated Branches:
  refs/heads/master ca74ec774 -> 330b0d05b


http://git-wip-us.apache.org/repos/asf/hbase/blob/330b0d05/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKWatcher.java
b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKWatcher.java
new file mode 100644
index 0000000..d0b0081
--- /dev/null
+++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKWatcher.java
@@ -0,0 +1,634 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.AuthUtil;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hadoop.hbase.security.Superusers;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooDefs.Perms;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ * Acts as the single ZooKeeper Watcher.  One instance of this is instantiated
+ * for each Master, RegionServer, and client process.
+ *
+ * <p>This is the only class that implements {@link Watcher}.  Other internal
+ * classes which need to be notified of ZooKeeper events must register with
+ * the local instance of this watcher via {@link #registerListener}.
+ *
+ * <p>This class also holds and manages the connection to ZooKeeper.  Code to
+ * deal with connection related events and exceptions are handled here.
+ */
+@InterfaceAudience.Private
+public class ZKWatcher implements Watcher, Abortable, Closeable {
+  private static final Log LOG = LogFactory.getLog(ZKWatcher.class);
+
+  // Identifier for this watcher (for logging only).  It is made of the prefix
+  // passed on construction and the zookeeper sessionid.
+  private String prefix;
+  private String identifier;
+
+  // zookeeper quorum
+  private String quorum;
+
+  // zookeeper connection
+  private final RecoverableZooKeeper recoverableZooKeeper;
+
+  // abortable in case of zk failure
+  protected Abortable abortable;
+  // Used if abortable is null
+  private boolean aborted = false;
+
+  public final ZNodePaths znodePaths;
+
+  // listeners to be notified
+  private final List<ZKListener> listeners = new CopyOnWriteArrayList<>();
+
+  // Used by ZKUtil:waitForZKConnectionIfAuthenticating to wait for SASL
+  // negotiation to complete
+  public CountDownLatch saslLatch = new CountDownLatch(1);
+
+
+
+  private final Configuration conf;
+
+  /* A pattern that matches a Kerberos name, borrowed from Hadoop's KerberosName */
+  private static final Pattern NAME_PATTERN = Pattern.compile("([^/@]*)(/([^/@]*))?@([^/@]*)");
+
+  /**
+   * Instantiate a ZooKeeper connection and watcher.
+   * @param identifier string that is passed to RecoverableZookeeper to be used as
+   * identifier for this instance. Use null for default.
+   * @throws IOException
+   * @throws ZooKeeperConnectionException
+   */
+  public ZKWatcher(Configuration conf, String identifier,
+                   Abortable abortable) throws ZooKeeperConnectionException, IOException
{
+    this(conf, identifier, abortable, false);
+  }
+
+  /**
+   * Instantiate a ZooKeeper connection and watcher.
+   * @param conf
+   * @param identifier string that is passed to RecoverableZookeeper to be used as identifier
for
+   *          this instance. Use null for default.
+   * @param abortable Can be null if there is on error there is no host to abort: e.g. client
+   *          context.
+   * @param canCreateBaseZNode
+   * @throws IOException
+   * @throws ZooKeeperConnectionException
+   */
+  public ZKWatcher(Configuration conf, String identifier,
+                   Abortable abortable, boolean canCreateBaseZNode)
+  throws IOException, ZooKeeperConnectionException {
+    this.conf = conf;
+    this.quorum = ZKConfig.getZKQuorumServersString(conf);
+    this.prefix = identifier;
+    // Identifier will get the sessionid appended later below down when we
+    // handle the syncconnect event.
+    this.identifier = identifier + "0x0";
+    this.abortable = abortable;
+    this.znodePaths = new ZNodePaths(conf);
+    PendingWatcher pendingWatcher = new PendingWatcher();
+    this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, pendingWatcher, identifier);
+    pendingWatcher.prepare(this);
+    if (canCreateBaseZNode) {
+      try {
+        createBaseZNodes();
+      } catch (ZooKeeperConnectionException zce) {
+        try {
+          this.recoverableZooKeeper.close();
+        } catch (InterruptedException ie) {
+          LOG.debug("Encountered InterruptedException when closing " + this.recoverableZooKeeper);
+          Thread.currentThread().interrupt();
+        }
+        throw zce;
+      }
+    }
+  }
+
+  private void createBaseZNodes() throws ZooKeeperConnectionException {
+    try {
+      // Create all the necessary "directories" of znodes
+      ZKUtil.createWithParents(this, znodePaths.baseZNode);
+      ZKUtil.createAndFailSilent(this, znodePaths.rsZNode);
+      ZKUtil.createAndFailSilent(this, znodePaths.drainingZNode);
+      ZKUtil.createAndFailSilent(this, znodePaths.tableZNode);
+      ZKUtil.createAndFailSilent(this, znodePaths.splitLogZNode);
+      ZKUtil.createAndFailSilent(this, znodePaths.backupMasterAddressesZNode);
+      ZKUtil.createAndFailSilent(this, znodePaths.tableLockZNode);
+      ZKUtil.createAndFailSilent(this, znodePaths.masterMaintZNode);
+    } catch (KeeperException e) {
+      throw new ZooKeeperConnectionException(
+          prefix("Unexpected KeeperException creating base node"), e);
+    }
+  }
+
+  /** Returns whether the znode is supposed to be readable by the client
+   * and DOES NOT contain sensitive information (world readable).*/
+  public boolean isClientReadable(String node) {
+    // Developer notice: These znodes are world readable. DO NOT add more znodes here UNLESS
+    // all clients need to access this data to work. Using zk for sharing data to clients
(other
+    // than service lookup case is not a recommended design pattern.
+    return
+        node.equals(znodePaths.baseZNode) ||
+        znodePaths.isAnyMetaReplicaZNode(node) ||
+        node.equals(znodePaths.masterAddressZNode) ||
+        node.equals(znodePaths.clusterIdZNode)||
+        node.equals(znodePaths.rsZNode) ||
+        // /hbase/table and /hbase/table/foo is allowed, /hbase/table-lock is not
+        node.equals(znodePaths.tableZNode) ||
+        node.startsWith(znodePaths.tableZNode + "/");
+  }
+
+  /**
+   * On master start, we check the znode ACLs under the root directory and set the ACLs properly
+   * if needed. If the cluster goes from an unsecure setup to a secure setup, this step is
needed
+   * so that the existing znodes created with open permissions are now changed with restrictive
+   * perms.
+   */
+  public void checkAndSetZNodeAcls() {
+    if (!ZKUtil.isSecureZooKeeper(getConfiguration())) {
+      LOG.info("not a secure deployment, proceeding");
+      return;
+    }
+
+    // Check the base znodes permission first. Only do the recursion if base znode's perms
are not
+    // correct.
+    try {
+      List<ACL> actualAcls = recoverableZooKeeper.getAcl(znodePaths.baseZNode, new
Stat());
+
+      if (!isBaseZnodeAclSetup(actualAcls)) {
+        LOG.info("setting znode ACLs");
+        setZnodeAclsRecursive(znodePaths.baseZNode);
+      }
+    } catch(KeeperException.NoNodeException nne) {
+      return;
+    } catch(InterruptedException ie) {
+      interruptedExceptionNoThrow(ie, false);
+    } catch (IOException|KeeperException e) {
+      LOG.warn("Received exception while checking and setting zookeeper ACLs", e);
+    }
+  }
+
+  /**
+   * Set the znode perms recursively. This will do post-order recursion, so that baseZnode
ACLs
+   * will be set last in case the master fails in between.
+   * @param znode
+   */
+  private void setZnodeAclsRecursive(String znode) throws KeeperException, InterruptedException
{
+    List<String> children = recoverableZooKeeper.getChildren(znode, false);
+
+    for (String child : children) {
+      setZnodeAclsRecursive(ZNodePaths.joinZNode(znode, child));
+    }
+    List<ACL> acls = ZKUtil.createACL(this, znode, true);
+    LOG.info("Setting ACLs for znode:" + znode + " , acl:" + acls);
+    recoverableZooKeeper.setAcl(znode, acls, -1);
+  }
+
+  /**
+   * Checks whether the ACLs returned from the base znode (/hbase) is set for secure setup.
+   * @param acls acls from zookeeper
+   * @return whether ACLs are set for the base znode
+   * @throws IOException
+   */
+  private boolean isBaseZnodeAclSetup(List<ACL> acls) throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Checking znode ACLs");
+    }
+    String[] superUsers = conf.getStrings(Superusers.SUPERUSER_CONF_KEY);
+    // Check whether ACL set for all superusers
+    if (superUsers != null && !checkACLForSuperUsers(superUsers, acls)) {
+      return false;
+    }
+
+    // this assumes that current authenticated user is the same as zookeeper client user
+    // configured via JAAS
+    String hbaseUser = UserGroupInformation.getCurrentUser().getShortUserName();
+
+    if (acls.isEmpty()) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("ACL is empty");
+      }
+      return false;
+    }
+
+    for (ACL acl : acls) {
+      int perms = acl.getPerms();
+      Id id = acl.getId();
+      // We should only set at most 3 possible ACLs for 3 Ids. One for everyone, one for
superuser
+      // and one for the hbase user
+      if (Ids.ANYONE_ID_UNSAFE.equals(id)) {
+        if (perms != Perms.READ) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug(String.format("permissions for '%s' are not correct: have 0x%x, want
0x%x",
+              id, perms, Perms.READ));
+          }
+          return false;
+        }
+      } else if (superUsers != null && isSuperUserId(superUsers, id)) {
+        if (perms != Perms.ALL) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug(String.format("permissions for '%s' are not correct: have 0x%x, want
0x%x",
+              id, perms, Perms.ALL));
+          }
+          return false;
+        }
+      } else if ("sasl".equals(id.getScheme())) {
+        String name = id.getId();
+        // If ZooKeeper recorded the Kerberos full name in the ACL, use only the shortname
+        Matcher match = NAME_PATTERN.matcher(name);
+        if (match.matches()) {
+          name = match.group(1);
+        }
+        if (name.equals(hbaseUser)) {
+          if (perms != Perms.ALL) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug(String.format("permissions for '%s' are not correct: have 0x%x, want
0x%x",
+                id, perms, Perms.ALL));
+            }
+            return false;
+          }
+        } else {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Unexpected shortname in SASL ACL: " + id);
+          }
+          return false;
+        }
+      } else {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("unexpected ACL id '" + id + "'");
+        }
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /*
+   * Validate whether ACL set for all superusers.
+   */
+  private boolean checkACLForSuperUsers(String[] superUsers, List<ACL> acls) {
+    for (String user : superUsers) {
+      boolean hasAccess = false;
+      // TODO: Validate super group members also when ZK supports setting node ACL for groups.
+      if (!AuthUtil.isGroupPrincipal(user)) {
+        for (ACL acl : acls) {
+          if (user.equals(acl.getId().getId())) {
+            if (acl.getPerms() == Perms.ALL) {
+              hasAccess = true;
+            } else {
+              if (LOG.isDebugEnabled()) {
+                LOG.debug(String.format(
+                  "superuser '%s' does not have correct permissions: have 0x%x, want 0x%x",
+                  acl.getId().getId(), acl.getPerms(), Perms.ALL));
+              }
+            }
+            break;
+          }
+        }
+        if (!hasAccess) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+  /*
+   * Validate whether ACL ID is superuser.
+   */
+  public static boolean isSuperUserId(String[] superUsers, Id id) {
+    for (String user : superUsers) {
+      // TODO: Validate super group members also when ZK supports setting node ACL for groups.
+      if (!AuthUtil.isGroupPrincipal(user) && new Id("sasl", user).equals(id)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return this.identifier + ", quorum=" + quorum + ", baseZNode=" + znodePaths.baseZNode;
+  }
+
+  /**
+   * Adds this instance's identifier as a prefix to the passed <code>str</code>
+   * @param str String to amend.
+   * @return A new string with this instance's identifier as prefix: e.g.
+   * if passed 'hello world', the returned string could be
+   */
+  public String prefix(final String str) {
+    return this.toString() + " " + str;
+  }
+
+  /**
+   * Get the znodes corresponding to the meta replicas from ZK
+   * @return list of znodes
+   * @throws KeeperException
+   */
+  public List<String> getMetaReplicaNodes() throws KeeperException {
+    List<String> childrenOfBaseNode = ZKUtil.listChildrenNoWatch(this, znodePaths.baseZNode);
+    List<String> metaReplicaNodes = new ArrayList<>(2);
+    if (childrenOfBaseNode != null) {
+      String pattern = conf.get("zookeeper.znode.metaserver","meta-region-server");
+      for (String child : childrenOfBaseNode) {
+        if (child.startsWith(pattern)) metaReplicaNodes.add(child);
+      }
+    }
+    return metaReplicaNodes;
+  }
+
+  /**
+   * Register the specified listener to receive ZooKeeper events.
+   * @param listener
+   */
+  public void registerListener(ZKListener listener) {
+    listeners.add(listener);
+  }
+
+  /**
+   * Register the specified listener to receive ZooKeeper events and add it as
+   * the first in the list of current listeners.
+   * @param listener
+   */
+  public void registerListenerFirst(ZKListener listener) {
+    listeners.add(0, listener);
+  }
+
+  public void unregisterListener(ZKListener listener) {
+    listeners.remove(listener);
+  }
+
+  /**
+   * Clean all existing listeners
+   */
+  public void unregisterAllListeners() {
+    listeners.clear();
+  }
+
+  /**
+   * Get a copy of current registered listeners
+   */
+  public List<ZKListener> getListeners() {
+    return new ArrayList<>(listeners);
+  }
+
+  /**
+   * @return The number of currently registered listeners
+   */
+  public int getNumberOfListeners() {
+    return listeners.size();
+  }
+
+  /**
+   * Get the connection to ZooKeeper.
+   * @return connection reference to zookeeper
+   */
+  public RecoverableZooKeeper getRecoverableZooKeeper() {
+    return recoverableZooKeeper;
+  }
+
+  public void reconnectAfterExpiration() throws IOException, KeeperException, InterruptedException
{
+    recoverableZooKeeper.reconnectAfterExpiration();
+  }
+
+  /**
+   * Get the quorum address of this instance.
+   * @return quorum string of this zookeeper connection instance
+   */
+  public String getQuorum() {
+    return quorum;
+  }
+
+  /**
+   * Get the znodePaths.
+   * <p>
+   * Mainly used for mocking as mockito can not mock a field access.
+   */
+  public ZNodePaths getZNodePaths() {
+    return znodePaths;
+  }
+
+  /**
+   * Method called from ZooKeeper for events and connection status.
+   * <p>
+   * Valid events are passed along to listeners.  Connection status changes
+   * are dealt with locally.
+   */
+  @Override
+  public void process(WatchedEvent event) {
+    LOG.debug(prefix("Received ZooKeeper Event, " +
+        "type=" + event.getType() + ", " +
+        "state=" + event.getState() + ", " +
+        "path=" + event.getPath()));
+
+    switch(event.getType()) {
+
+      // If event type is NONE, this is a connection status change
+      case None: {
+        connectionEvent(event);
+        break;
+      }
+
+      // Otherwise pass along to the listeners
+
+      case NodeCreated: {
+        for(ZKListener listener : listeners) {
+          listener.nodeCreated(event.getPath());
+        }
+        break;
+      }
+
+      case NodeDeleted: {
+        for(ZKListener listener : listeners) {
+          listener.nodeDeleted(event.getPath());
+        }
+        break;
+      }
+
+      case NodeDataChanged: {
+        for(ZKListener listener : listeners) {
+          listener.nodeDataChanged(event.getPath());
+        }
+        break;
+      }
+
+      case NodeChildrenChanged: {
+        for(ZKListener listener : listeners) {
+          listener.nodeChildrenChanged(event.getPath());
+        }
+        break;
+      }
+    }
+  }
+
+  // Connection management
+
+  /**
+   * Called when there is a connection-related event via the Watcher callback.
+   * <p>
+   * If Disconnected or Expired, this should shutdown the cluster. But, since
+   * we send a KeeperException.SessionExpiredException along with the abort
+   * call, it's possible for the Abortable to catch it and try to create a new
+   * session with ZooKeeper. This is what the client does in HCM.
+   * <p>
+   * @param event
+   */
+  private void connectionEvent(WatchedEvent event) {
+    switch(event.getState()) {
+      case SyncConnected:
+        this.identifier = this.prefix + "-0x" +
+          Long.toHexString(this.recoverableZooKeeper.getSessionId());
+        // Update our identifier.  Otherwise ignore.
+        LOG.debug(this.identifier + " connected");
+        break;
+
+      // Abort the server if Disconnected or Expired
+      case Disconnected:
+        LOG.debug(prefix("Received Disconnected from ZooKeeper, ignoring"));
+        break;
+
+      case Expired:
+        String msg = prefix(this.identifier + " received expired from " +
+          "ZooKeeper, aborting");
+        // TODO: One thought is to add call to ZKListener so say,
+        // ZKNodeTracker can zero out its data values.
+        if (this.abortable != null) {
+          this.abortable.abort(msg, new KeeperException.SessionExpiredException());
+        }
+        break;
+
+      case ConnectedReadOnly:
+      case SaslAuthenticated:
+      case AuthFailed:
+        break;
+
+      default:
+        throw new IllegalStateException("Received event is not valid: " + event.getState());
+    }
+  }
+
+  /**
+   * Forces a synchronization of this ZooKeeper client connection.
+   * <p>
+   * Executing this method before running other methods will ensure that the
+   * subsequent operations are up-to-date and consistent as of the time that
+   * the sync is complete.
+   * <p>
+   * This is used for compareAndSwap type operations where we need to read the
+   * data of an existing node and delete or transition that node, utilizing the
+   * previously read version and data.  We want to ensure that the version read
+   * is up-to-date from when we begin the operation.
+   */
+  public void sync(String path) throws KeeperException {
+    this.recoverableZooKeeper.sync(path, null, null);
+  }
+
+  /**
+   * Handles KeeperExceptions in client calls.
+   * <p>
+   * This may be temporary but for now this gives one place to deal with these.
+   * <p>
+   * TODO: Currently this method rethrows the exception to let the caller handle
+   * <p>
+   * @param ke
+   * @throws KeeperException
+   */
+  public void keeperException(KeeperException ke)
+  throws KeeperException {
+    LOG.error(prefix("Received unexpected KeeperException, re-throwing exception"), ke);
+    throw ke;
+  }
+
+  /**
+   * Handles InterruptedExceptions in client calls.
+   * @param ie the InterruptedException instance thrown
+   * @throws KeeperException the exception to throw, transformed from the InterruptedException
+   */
+  public void interruptedException(InterruptedException ie) throws KeeperException {
+    interruptedExceptionNoThrow(ie, true);
+    // Throw a system error exception to let upper level handle it
+    throw new KeeperException.SystemErrorException();
+  }
+
+  /**
+   * Log the InterruptedException and interrupt current thread
+   * @param ie The IterruptedException to log
+   * @param throwLater Whether we will throw the exception latter
+   */
+  public void interruptedExceptionNoThrow(InterruptedException ie, boolean throwLater) {
+    LOG.debug(prefix("Received InterruptedException, will interrupt current thread"
+        + (throwLater ? " and rethrow a SystemErrorException" : "")),
+      ie);
+    // At least preserve interrupt.
+    Thread.currentThread().interrupt();
+  }
+
+  /**
+   * Close the connection to ZooKeeper.
+   *
+   */
+  @Override
+  public void close() {
+    try {
+      recoverableZooKeeper.close();
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  public Configuration getConfiguration() {
+    return conf;
+  }
+
+  @Override
+  public void abort(String why, Throwable e) {
+    if (this.abortable != null) this.abortable.abort(why, e);
+    else this.aborted = true;
+  }
+
+  @Override
+  public boolean isAborted() {
+    return this.abortable == null? this.aborted: this.abortable.isAborted();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/330b0d05/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestInstancePending.java
----------------------------------------------------------------------
diff --git a/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestInstancePending.java
b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestInstancePending.java
new file mode 100644
index 0000000..e67c9fd
--- /dev/null
+++ b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestInstancePending.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestInstancePending {
+  @Test(timeout = 1000)
+  public void test() throws Exception {
+    final InstancePending<String> pending = new InstancePending<>();
+    final AtomicReference<String> getResultRef = new AtomicReference<>();
+
+    new Thread() {
+      @Override
+      public void run() {
+        getResultRef.set(pending.get());
+      }
+    }.start();
+
+    Thread.sleep(100);
+    Assert.assertNull(getResultRef.get());
+
+    pending.prepare("abc");
+    Thread.sleep(100);
+    Assert.assertEquals("abc", getResultRef.get());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/330b0d05/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMetrics.java
b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMetrics.java
new file mode 100644
index 0000000..2811cc5
--- /dev/null
+++ b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMetrics.java
@@ -0,0 +1,80 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestZKMetrics {
+
+  @Test
+  public void testRegisterExceptions() {
+    MetricsZooKeeperSource zkSource = mock(MetricsZooKeeperSourceImpl.class);
+    ZKMetrics metricsZK = new ZKMetrics(zkSource);
+    metricsZK.registerAuthFailedException();
+    metricsZK.registerConnectionLossException();
+    metricsZK.registerConnectionLossException();
+    metricsZK.registerDataInconsistencyException();
+    metricsZK.registerInvalidACLException();
+    metricsZK.registerNoAuthException();
+    metricsZK.registerOperationTimeoutException();
+    metricsZK.registerOperationTimeoutException();
+    metricsZK.registerRuntimeInconsistencyException();
+    metricsZK.registerSessionExpiredException();
+    metricsZK.registerSystemErrorException();
+    metricsZK.registerSystemErrorException();
+    metricsZK.registerFailedZKCall();
+
+    verify(zkSource, times(1)).incrementAuthFailedCount();
+    // ConnectionLoss Exception was registered twice.
+    verify(zkSource, times(2)).incrementConnectionLossCount();
+    verify(zkSource, times(1)).incrementDataInconsistencyCount();
+    verify(zkSource, times(1)).incrementInvalidACLCount();
+    verify(zkSource, times(1)).incrementNoAuthCount();
+    // OperationTimeout Exception was registered twice.
+    verify(zkSource, times(2)).incrementOperationTimeoutCount();
+    verify(zkSource, times(1)).incrementRuntimeInconsistencyCount();
+    verify(zkSource, times(1)).incrementSessionExpiredCount();
+    // SystemError Exception was registered twice.
+    verify(zkSource, times(2)).incrementSystemErrorCount();
+    verify(zkSource, times(1)).incrementTotalFailedZKCalls();
+  }
+
+  @Test
+  public void testLatencyHistogramUpdates() {
+    MetricsZooKeeperSource zkSource = mock(MetricsZooKeeperSourceImpl.class);
+    ZKMetrics metricsZK = new ZKMetrics(zkSource);
+    long latency = 100;
+
+    metricsZK.registerReadOperationLatency(latency);
+    metricsZK.registerReadOperationLatency(latency);
+    metricsZK.registerWriteOperationLatency(latency);
+    metricsZK.registerSyncOperationLatency(latency);
+    // Read Operation Latency update was registered twice.
+    verify(zkSource, times(2)).recordReadOperationLatency(latency);
+    verify(zkSource, times(1)).recordWriteOperationLatency(latency);
+    verify(zkSource, times(1)).recordSyncOperationLatency(latency);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/330b0d05/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
----------------------------------------------------------------------
diff --git a/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
new file mode 100644
index 0000000..7006040
--- /dev/null
+++ b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.security.Superusers;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooDefs.Perms;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ *
+ */
+@Category({SmallTests.class})
+public class TestZKUtil {
+
+  @Test
+  public void testUnsecure() throws ZooKeeperConnectionException, IOException {
+    Configuration conf = HBaseConfiguration.create();
+    conf.set(Superusers.SUPERUSER_CONF_KEY, "user1");
+    String node = "/hbase/testUnsecure";
+    ZKWatcher watcher = new ZKWatcher(conf, node, null, false);
+    List<ACL> aclList = ZKUtil.createACL(watcher, node, false);
+    Assert.assertEquals(aclList.size(), 1);
+    Assert.assertTrue(aclList.contains(Ids.OPEN_ACL_UNSAFE.iterator().next()));
+  }
+
+  @Test
+  public void testSecuritySingleSuperuser() throws ZooKeeperConnectionException, IOException
{
+    Configuration conf = HBaseConfiguration.create();
+    conf.set(Superusers.SUPERUSER_CONF_KEY, "user1");
+    String node = "/hbase/testSecuritySingleSuperuser";
+    ZKWatcher watcher = new ZKWatcher(conf, node, null, false);
+    List<ACL> aclList = ZKUtil.createACL(watcher, node, true);
+    Assert.assertEquals(aclList.size(), 2); // 1+1, since ACL will be set for the creator
by default
+    Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "user1"))));
+    Assert.assertTrue(aclList.contains(Ids.CREATOR_ALL_ACL.iterator().next()));
+  }
+
+  @Test
+  public void testCreateACL() throws ZooKeeperConnectionException, IOException {
+    Configuration conf = HBaseConfiguration.create();
+    conf.set(Superusers.SUPERUSER_CONF_KEY, "user1,@group1,user2,@group2,user3");
+    String node = "/hbase/testCreateACL";
+    ZKWatcher watcher = new ZKWatcher(conf, node, null, false);
+    List<ACL> aclList = ZKUtil.createACL(watcher, node, true);
+    Assert.assertEquals(aclList.size(), 4); // 3+1, since ACL will be set for the creator
by default
+    Assert.assertFalse(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "@group1"))));
+    Assert.assertFalse(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "@group2"))));
+    Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "user1"))));
+    Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "user2"))));
+    Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "user3"))));
+  }
+
+  @Test
+  public void testCreateACLWithSameUser() throws ZooKeeperConnectionException, IOException
{
+    Configuration conf = HBaseConfiguration.create();
+    conf.set(Superusers.SUPERUSER_CONF_KEY, "user4,@group1,user5,user6");
+    UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser("user4"));
+    String node = "/hbase/testCreateACL";
+    ZKWatcher watcher = new ZKWatcher(conf, node, null, false);
+    List<ACL> aclList = ZKUtil.createACL(watcher, node, true);
+    Assert.assertEquals(aclList.size(), 3); // 3, since service user the same as one of superuser
+    Assert.assertFalse(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "@group1"))));
+    Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("auth", ""))));
+    Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "user5"))));
+    Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "user6"))));
+  }
+
+  @Test(expected = KeeperException.SystemErrorException.class)
+  public void testInterruptedDuringAction()
+      throws ZooKeeperConnectionException, IOException, KeeperException, InterruptedException
{
+    final RecoverableZooKeeper recoverableZk = Mockito.mock(RecoverableZooKeeper.class);
+    ZKWatcher zkw = new ZKWatcher(HBaseConfiguration.create(), "unittest", null) {
+      @Override
+      public RecoverableZooKeeper getRecoverableZooKeeper() {
+        return recoverableZk;
+      }
+    };
+    Mockito.doThrow(new InterruptedException()).when(recoverableZk)
+        .getChildren(zkw.znodePaths.baseZNode, null);
+    ZKUtil.listChildrenNoWatch(zkw, zkw.znodePaths.baseZNode);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/330b0d05/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKWatcher.java
b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKWatcher.java
new file mode 100644
index 0000000..bd4575d
--- /dev/null
+++ b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKWatcher.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.zookeeper;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ SmallTests.class })
+public class TestZKWatcher {
+
+  @Test
+  public void testIsClientReadable() throws IOException {
+    ZKWatcher watcher =
+      new ZKWatcher(HBaseConfiguration.create(), "testIsClientReadable", null, false);
+
+    assertTrue(watcher.isClientReadable(watcher.znodePaths.baseZNode));
+    assertTrue(watcher.isClientReadable(watcher.znodePaths.getZNodeForReplica(0)));
+    assertTrue(watcher.isClientReadable(watcher.znodePaths.masterAddressZNode));
+    assertTrue(watcher.isClientReadable(watcher.znodePaths.clusterIdZNode));
+    assertTrue(watcher.isClientReadable(watcher.znodePaths.tableZNode));
+    assertTrue(
+      watcher.isClientReadable(ZNodePaths.joinZNode(watcher.znodePaths.tableZNode, "foo")));
+    assertTrue(watcher.isClientReadable(watcher.znodePaths.rsZNode));
+
+    assertFalse(watcher.isClientReadable(watcher.znodePaths.tableLockZNode));
+    assertFalse(watcher.isClientReadable(watcher.znodePaths.balancerZNode));
+    assertFalse(watcher.isClientReadable(watcher.znodePaths.regionNormalizerZNode));
+    assertFalse(watcher.isClientReadable(watcher.znodePaths.clusterStateZNode));
+    assertFalse(watcher.isClientReadable(watcher.znodePaths.drainingZNode));
+    assertFalse(watcher.isClientReadable(watcher.znodePaths.splitLogZNode));
+    assertFalse(watcher.isClientReadable(watcher.znodePaths.backupMasterAddressesZNode));
+
+    watcher.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/330b0d05/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b423085..ee35212 100755
--- a/pom.xml
+++ b/pom.xml
@@ -92,6 +92,7 @@
     <module>hbase-metrics</module>
     <module>hbase-spark-it</module>
     <module>hbase-backup</module>
+    <module>hbase-zookeeper</module>
   </modules>
   <!--Add apache snapshots in case we want to use unreleased versions of plugins:
       e.g. surefire 2.18-SNAPSHOT-->
@@ -1789,6 +1790,11 @@
         <type>test-jar</type>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <artifactId>hbase-zookeeper</artifactId>
+        <groupId>org.apache.hbase</groupId>
+        <version>${project.version}</version>
+      </dependency>
       <!-- General dependencies -->
       <dependency>
         <groupId>com.github.stephenc.findbugs</groupId>


Mime
View raw message