hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1203515 [4/4] - in /hbase/trunk: ./ conf/ security/ security/src/ security/src/main/ security/src/main/java/ security/src/main/java/org/ security/src/main/java/org/apache/ security/src/main/java/org/apache/hadoop/ security/src/main/java/or...
Date Fri, 18 Nov 2011 07:34:45 GMT
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java?rev=1203515&r1=1203514&r2=1203515&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java Fri Nov 18 07:34:43
2011
@@ -567,6 +567,35 @@ public class ZKUtil {
   }
 
   /**
+   * Returns the date of child znodes of the specified znode.  Also sets a watch on
+   * the specified znode which will capture a NodeDeleted event on the specified
+   * znode as well as NodeChildrenChanged if any children of the specified znode
+   * are created or deleted.
+   *
+   * Returns null if the specified node does not exist.  Otherwise returns a
+   * list of children of the specified node.  If the node exists but it has no
+   * children, an empty list will be returned.
+   *
+   * @param zkw zk reference
+   * @param znode path of node to list and watch children of
+   * @return list of data of children of the specified node, an empty list if the node
+   *          exists but has no children, and null if the node does not exist
+   * @throws KeeperException if unexpected zookeeper exception
+   */
+  public static List<NodeAndData> getChildDataAndWatchForNewChildren(
+      ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
+    List<String> nodes =
+      ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
+    List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
+    for (String node: nodes) {
+      String nodePath = ZKUtil.joinZNode(baseNode, node);
+      byte [] data = ZKUtil.getDataAndWatch(zkw, nodePath);
+      newNodes.add(new NodeAndData(nodePath, data));
+    }
+    return newNodes;
+  }
+
+  /**
    * Update the data of an existing node with the expected version to have the
    * specified data.
    *

Modified: hbase/trunk/src/main/resources/hbase-default.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/resources/hbase-default.xml?rev=1203515&r1=1203514&r2=1203515&view=diff
==============================================================================
--- hbase/trunk/src/main/resources/hbase-default.xml (original)
+++ hbase/trunk/src/main/resources/hbase-default.xml Fri Nov 18 07:34:43 2011
@@ -558,6 +558,39 @@
     specified in hbase.regionserver.keytab.file
     </description>
   </property>
+
+  <!-- Additional configuration specific to HBase security -->
+  <property>
+    <name>hadoop.policy.file</name>
+    <value>hbase-policy.xml</value>
+    <description>The policy configuration file used by RPC servers to make
+      authorization decisions on client requests.  Only used when HBase
+      security is enabled.
+    </description>
+  </property>
+  <property>
+    <name>hbase.superuser</name>
+    <value></value>
+    <description>List of users or groups (comma-separated), who are allowed
+    full privileges, regardless of stored ACLs, across the cluster.
+    Only used when HBase security is enabled.
+    </description>
+  </property>
+  <property>
+    <name>hbase.auth.key.update.interval</name>
+    <value>86400000</value>
+    <description>The update interval for master key for authentication tokens 
+    in servers in milliseconds.  Only used when HBase security is enabled.
+    </description>
+  </property>
+  <property>
+    <name>hbase.auth.token.max.lifetime</name>
+    <value>604800000</value>
+    <description>The maximum lifetime in milliseconds after which an
+    authentication token expires.  Only used when HBase security is enabled.
+    </description>
+  </property>
+
   <property>
     <name>zookeeper.session.timeout</name>
     <value>180000</value>

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=1203515&r1=1203514&r2=1203515&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java Fri Nov 18 07:34:43
2011
@@ -222,7 +222,7 @@ public class MiniHBaseCluster {
       t.start();
       t.waitForServerOnline();
     } catch (InterruptedException ie) {
-      throw new IOException("Interrupted executing UserGroupInformation.doAs()", ie);
+      throw new IOException("Interrupted adding regionserver to cluster", ie);
     }
     return t;
   }
@@ -295,7 +295,7 @@ public class MiniHBaseCluster {
       t.start();
       t.waitForServerOnline();
     } catch (InterruptedException ie) {
-      throw new IOException("Interrupted executing UserGroupInformation.doAs()", ie);
+      throw new IOException("Interrupted adding master to cluster", ie);
     }
     return t;
   }

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java?rev=1203515&r1=1203514&r2=1203515&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java Fri Nov 18
07:34:43 2011
@@ -573,6 +573,9 @@ public class PerformanceEvaluation {
     // Add a Class from the hbase.jar so it gets registered too.
     TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
       org.apache.hadoop.hbase.util.Bytes.class);
+
+    TableMapReduceUtil.initCredentials(job);
+
     job.waitForCompletion(true);
   }
 

Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java?rev=1203515&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java Fri
Nov 18 07:34:43 2011
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.zookeeper;
+
+import static org.junit.Assert.*;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ */
+public class TestZKLeaderManager {
+  private static Log LOG = LogFactory.getLog(TestZKLeaderManager.class);
+
+  private static final String LEADER_ZNODE =
+      "/test/" + TestZKLeaderManager.class.getSimpleName();
+
+  private static class MockAbortable implements Abortable {
+    private boolean aborted;
+
+    @Override
+    public void abort(String why, Throwable e) {
+      aborted = true;
+      LOG.fatal("Aborting during test: "+why, e);
+      fail("Aborted during test: " + why);
+    }
+
+    @Override
+    public boolean isAborted() {
+      return aborted;
+    }
+  }
+
+  private static class MockLeader extends Thread implements Stoppable {
+    private boolean stopped;
+    private ZooKeeperWatcher watcher;
+    private ZKLeaderManager zkLeader;
+    private AtomicBoolean master = new AtomicBoolean(false);
+    private int index;
+
+    public MockLeader(ZooKeeperWatcher watcher, int index) {
+      setDaemon(true);
+      setName("TestZKLeaderManager-leader-" + index);
+      this.index = index;
+      this.watcher = watcher;
+      this.zkLeader = new ZKLeaderManager(watcher, LEADER_ZNODE,
+          Bytes.toBytes(index), this);
+    }
+
+    public boolean isMaster() {
+      return master.get();
+    }
+
+    public int getIndex() {
+      return index;
+    }
+
+    public ZooKeeperWatcher getWatcher() {
+      return watcher;
+    }
+
+    public void run() {
+      while (!stopped) {
+        zkLeader.start();
+        zkLeader.waitToBecomeLeader();
+        master.set(true);
+
+        while (master.get() && !stopped) {
+          try {
+            Thread.sleep(200);
+          } catch (InterruptedException ignored) {}
+        }
+      }
+    }
+
+    public void abdicate() {
+      zkLeader.stepDownAsLeader();
+      master.set(false);
+    }
+
+    @Override
+    public void stop(String why) {
+      stopped = true;
+      abdicate();
+      watcher.close();
+    }
+
+    @Override
+    public boolean isStopped() {
+      return stopped;
+    }
+  }
+
+  private static HBaseTestingUtility TEST_UTIL;
+  private static MockLeader[] CANDIDATES;
+
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
+    TEST_UTIL = new HBaseTestingUtility();
+    TEST_UTIL.startMiniZKCluster();
+    Configuration conf = TEST_UTIL.getConfiguration();
+
+    // use an abortable to fail the test in the case of any KeeperExceptions
+    MockAbortable abortable = new MockAbortable();
+    CANDIDATES = new MockLeader[3];
+    for (int i = 0; i < 3; i++) {
+      ZooKeeperWatcher watcher = newZK(conf, "server"+i, abortable);
+      CANDIDATES[i] = new MockLeader(watcher, i);
+      CANDIDATES[i].start();
+    }
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniZKCluster();
+  }
+
+  @Test
+  public void testLeaderSelection() throws Exception {
+    MockLeader currentLeader = getCurrentLeader();
+    // one leader should have been found
+    assertNotNull("Leader should exist", currentLeader);
+    LOG.debug("Current leader index is "+currentLeader.getIndex());
+
+    byte[] znodeData = ZKUtil.getData(CANDIDATES[0].getWatcher(), LEADER_ZNODE);
+    assertNotNull("Leader znode should contain leader index", znodeData);
+    assertTrue("Leader znode should not be empty", znodeData.length > 0);
+    int storedIndex = Bytes.toInt(znodeData);
+    LOG.debug("Stored leader index in ZK is "+storedIndex);
+    assertEquals("Leader znode should match leader index",
+        currentLeader.getIndex(), storedIndex);
+
+    // force a leader transition
+    currentLeader.abdicate();
+    assertFalse(currentLeader.isMaster());
+
+    // check for new leader
+    currentLeader = getCurrentLeader();
+    // one leader should have been found
+    assertNotNull("New leader should exist after abdication", currentLeader);
+    LOG.debug("New leader index is "+currentLeader.getIndex());
+
+    znodeData = ZKUtil.getData(CANDIDATES[0].getWatcher(), LEADER_ZNODE);
+    assertNotNull("Leader znode should contain leader index", znodeData);
+    assertTrue("Leader znode should not be empty", znodeData.length > 0);
+    storedIndex = Bytes.toInt(znodeData);
+    LOG.debug("Stored leader index in ZK is "+storedIndex);
+    assertEquals("Leader znode should match leader index",
+        currentLeader.getIndex(), storedIndex);
+
+    // force another transition by stopping the current
+    currentLeader.stop("Stopping for test");
+    assertFalse(currentLeader.isMaster());
+
+    // check for new leader
+    currentLeader = getCurrentLeader();
+    // one leader should have been found
+    assertNotNull("New leader should exist after stop", currentLeader);
+    LOG.debug("New leader index is "+currentLeader.getIndex());
+
+    znodeData = ZKUtil.getData(CANDIDATES[0].getWatcher(), LEADER_ZNODE);
+    assertNotNull("Leader znode should contain leader index", znodeData);
+    assertTrue("Leader znode should not be empty", znodeData.length > 0);
+    storedIndex = Bytes.toInt(znodeData);
+    LOG.debug("Stored leader index in ZK is "+storedIndex);
+    assertEquals("Leader znode should match leader index",
+        currentLeader.getIndex(), storedIndex);
+
+    // with a second stop we can guarantee that a previous leader has resumed leading
+    currentLeader.stop("Stopping for test");
+    assertFalse(currentLeader.isMaster());
+
+    // check for new
+    currentLeader = getCurrentLeader();
+    assertNotNull("New leader should exist", currentLeader);
+  }
+
+  private MockLeader getCurrentLeader() throws Exception {
+    MockLeader currentLeader = null;
+    outer:
+    // wait up to 2 secs for initial leader
+    for (int i = 0; i < 20; i++) {
+      for (int j = 0; j < CANDIDATES.length; j++) {
+        if (CANDIDATES[j].isMaster()) {
+          // should only be one leader
+          if (currentLeader != null) {
+            fail("Both candidate "+currentLeader.getIndex()+" and "+j+" claim to be leader!");
+          }
+          currentLeader = CANDIDATES[j];
+        }
+      }
+      if (currentLeader != null) {
+        break outer;
+      }
+      Thread.sleep(100);
+    }
+    return currentLeader;
+  }
+
+  private static ZooKeeperWatcher newZK(Configuration conf, String name,
+      Abortable abort) throws Exception {
+    Configuration copy = HBaseConfiguration.create(conf);
+    ZooKeeperWatcher zk = new ZooKeeperWatcher(copy, name, abort);
+    return zk;
+  }
+}



Mime
View raw message