hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject svn commit: r1452257 [11/14] - in /hbase/branches/0.94: security/src/main/java/org/apache/hadoop/hbase/security/access/ security/src/test/java/org/apache/hadoop/hbase/security/access/ src/main/jamon/org/apache/hadoop/hbase/tmpl/master/ src/main/java/or...
Date Mon, 04 Mar 2013 11:24:53 GMT
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java Mon Mar  4 11:24:50 2013
@@ -42,6 +42,8 @@ import org.apache.hadoop.hbase.client.HT
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
@@ -92,6 +94,14 @@ public class TestMasterObserver {
     private boolean postStartMasterCalled;
     private boolean startCalled;
     private boolean stopCalled;
+    private boolean preSnapshotCalled;
+    private boolean postSnapshotCalled;
+    private boolean preCloneSnapshotCalled;
+    private boolean postCloneSnapshotCalled;
+    private boolean preRestoreSnapshotCalled;
+    private boolean postRestoreSnapshotCalled;
+    private boolean preDeleteSnapshotCalled;
+    private boolean postDeleteSnapshotCalled;
 
     public void enableBypass(boolean bypass) {
       this.bypass = bypass;
@@ -124,6 +134,14 @@ public class TestMasterObserver {
       postBalanceCalled = false;
       preBalanceSwitchCalled = false;
       postBalanceSwitchCalled = false;
+      preSnapshotCalled = false;
+      postSnapshotCalled = false;
+      preCloneSnapshotCalled = false;
+      postCloneSnapshotCalled = false;
+      preRestoreSnapshotCalled = false;
+      postRestoreSnapshotCalled = false;
+      preDeleteSnapshotCalled = false;
+      postDeleteSnapshotCalled = false;
     }
 
     @Override
@@ -463,10 +481,82 @@ public class TestMasterObserver {
     public boolean wasStarted() { return startCalled; }
 
     public boolean wasStopped() { return stopCalled; }
+
+    @Override
+    public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      preSnapshotCalled = true;
+    }
+
+    @Override
+    public void postSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      postSnapshotCalled = true;
+    }
+
+    public boolean wasSnapshotCalled() {
+      return preSnapshotCalled && postSnapshotCalled;
+    }
+
+    @Override
+    public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      preCloneSnapshotCalled = true;
+    }
+
+    @Override
+    public void postCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      postCloneSnapshotCalled = true;
+    }
+
+    public boolean wasCloneSnapshotCalled() {
+      return preCloneSnapshotCalled && postCloneSnapshotCalled;
+    }
+
+    @Override
+    public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      preRestoreSnapshotCalled = true;
+    }
+
+    @Override
+    public void postRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+        throws IOException {
+      postRestoreSnapshotCalled = true;
+    }
+
+    public boolean wasRestoreSnapshotCalled() {
+      return preRestoreSnapshotCalled && postRestoreSnapshotCalled;
+    }
+
+    @Override
+    public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot) throws IOException {
+      preDeleteSnapshotCalled = true;
+    }
+
+    @Override
+    public void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+        final SnapshotDescription snapshot) throws IOException {
+      postDeleteSnapshotCalled = true;
+    }
+
+    public boolean wasDeleteSnapshotCalled() {
+      return preDeleteSnapshotCalled && postDeleteSnapshotCalled;
+    }
   }
 
   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static byte[] TEST_SNAPSHOT = Bytes.toBytes("observed_snapshot");
   private static byte[] TEST_TABLE = Bytes.toBytes("observed_table");
+  private static byte[] TEST_CLONE = Bytes.toBytes("observed_clone");
   private static byte[] TEST_FAMILY = Bytes.toBytes("fam1");
   private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
 
@@ -475,6 +565,8 @@ public class TestMasterObserver {
     Configuration conf = UTIL.getConfiguration();
     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
         CPMasterObserver.class.getName());
+    // Enable snapshot
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
     // We need more than one data server on this test
     UTIL.startMiniCluster(2);
   }
@@ -719,6 +811,63 @@ public class TestMasterObserver {
         cp.wasBalanceCalled());
   }
 
+  @Test
+  public void testSnapshotOperations() throws Exception {
+    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
+    HMaster master = cluster.getMaster();
+    MasterCoprocessorHost host = master.getCoprocessorHost();
+    CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
+        CPMasterObserver.class.getName());
+    cp.resetStates();
+
+    // create a table
+    HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+
+    // delete table if exists
+    if (admin.tableExists(TEST_TABLE)) {
+      UTIL.deleteTable(TEST_TABLE);
+    }
+
+    admin.createTable(htd);
+    admin.disableTable(TEST_TABLE);
+    assertTrue(admin.isTableDisabled(TEST_TABLE));
+
+    try {
+      // Test snapshot operation
+      assertFalse("Coprocessor should not have been called yet",
+        cp.wasSnapshotCalled());
+      admin.snapshot(TEST_SNAPSHOT, TEST_TABLE);
+      assertTrue("Coprocessor should have been called on snapshot",
+        cp.wasSnapshotCalled());
+
+      // Test clone operation
+      admin.cloneSnapshot(TEST_SNAPSHOT, TEST_CLONE);
+      assertTrue("Coprocessor should have been called on snapshot clone",
+        cp.wasCloneSnapshotCalled());
+      assertFalse("Coprocessor restore should not have been called on snapshot clone",
+        cp.wasRestoreSnapshotCalled());
+      admin.disableTable(TEST_CLONE);
+      assertTrue(admin.isTableDisabled(TEST_TABLE));
+      admin.deleteTable(TEST_CLONE);
+
+      // Test restore operation
+      cp.resetStates();
+      admin.restoreSnapshot(TEST_SNAPSHOT);
+      assertTrue("Coprocessor should have been called on snapshot restore",
+        cp.wasRestoreSnapshotCalled());
+      assertFalse("Coprocessor clone should not have been called on snapshot restore",
+        cp.wasCloneSnapshotCalled());
+
+      admin.deleteSnapshot(TEST_SNAPSHOT);
+      assertTrue("Coprocessor should have been called on snapshot delete",
+        cp.wasDeleteSnapshotCalled());
+    } finally {
+      admin.deleteTable(TEST_TABLE);
+    }
+  }
+
   private void waitForRITtoBeZero(HMaster master) throws IOException {
     // wait for assignments to finish
     AssignmentManager mgr = master.getAssignmentManager();

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test that we propagate errors through an dispatcher exactly once via different failure
+ * injection mechanisms.
+ */
+@Category(SmallTests.class)
+public class TestForeignExceptionDispatcher {
+  private static final Log LOG = LogFactory.getLog(TestForeignExceptionDispatcher.class);
+
+  /**
+   * Exception thrown from the test
+   */
+  final ForeignException EXTEXN = new ForeignException("FORTEST", new IllegalArgumentException("FORTEST"));
+  final ForeignException EXTEXN2 = new ForeignException("FORTEST2", new IllegalArgumentException("FORTEST2"));
+
+  /**
+   * Tests that a dispatcher only dispatches only the first exception, and does not propagate
+   * subsequent exceptions.
+   */
+  @Test
+  public void testErrorPropagation() {
+    ForeignExceptionListener listener1 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionListener listener2 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionDispatcher dispatcher = new ForeignExceptionDispatcher();
+
+    // add the listeners
+    dispatcher.addListener(listener1);
+    dispatcher.addListener(listener2);
+
+    // create an artificial error
+    dispatcher.receive(EXTEXN);
+
+    // make sure the listeners got the error
+    Mockito.verify(listener1, Mockito.times(1)).receive(EXTEXN);
+    Mockito.verify(listener2, Mockito.times(1)).receive(EXTEXN);
+
+    // make sure that we get an exception
+    try {
+      dispatcher.rethrowException();
+      fail("Monitor should have thrown an exception after getting error.");
+    } catch (ForeignException ex) {
+      assertTrue("Got an unexpected exception:" + ex, ex.getCause() == EXTEXN.getCause());
+      LOG.debug("Got the testing exception!");
+    }
+
+    // push another error, which should be not be passed to listeners
+    dispatcher.receive(EXTEXN2);
+    Mockito.verify(listener1, Mockito.never()).receive(EXTEXN2);
+    Mockito.verify(listener2, Mockito.never()).receive(EXTEXN2);
+  }
+
+  @Test
+  public void testSingleDispatcherWithTimer() {
+    ForeignExceptionListener listener1 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionListener listener2 = Mockito.mock(ForeignExceptionListener.class);
+
+    ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
+
+    // add the listeners
+    monitor.addListener(listener1);
+    monitor.addListener(listener2);
+
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(monitor, 1000);
+    timer.start();
+    timer.trigger();
+
+    assertTrue("Monitor didn't get timeout", monitor.hasException());
+
+    // verify that that we propagated the error
+    Mockito.verify(listener1).receive(Mockito.any(ForeignException.class));
+    Mockito.verify(listener2).receive(Mockito.any(ForeignException.class));
+  }
+
+  /**
+   * Test that the dispatcher can receive an error via the timer mechanism.
+   */
+  @Test
+  public void testAttemptTimer() {
+    ForeignExceptionListener listener1 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionListener listener2 = Mockito.mock(ForeignExceptionListener.class);
+    ForeignExceptionDispatcher orchestrator = new ForeignExceptionDispatcher();
+
+    // add the listeners
+    orchestrator.addListener(listener1);
+    orchestrator.addListener(listener2);
+
+    // now create a timer and check for that error
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(orchestrator, 1000);
+    timer.start();
+    timer.trigger();
+    // make sure that we got the timer error
+    Mockito.verify(listener1, Mockito.times(1)).receive(Mockito.any(ForeignException.class));
+    Mockito.verify(listener2, Mockito.times(1)).receive(Mockito.any(ForeignException.class));
+  }
+}
\ No newline at end of file

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * Test that we correctly serialize exceptions from a remote source
+ */
+@Category(SmallTests.class)
+public class TestForeignExceptionSerialization {
+  private static final String srcName = "someNode";
+
+  /**
+   * Verify that we get back similar stack trace information before an after serialization.
+   * @throws InvalidProtocolBufferException
+   */
+  @Test
+  public void testSimpleException() throws InvalidProtocolBufferException {
+    String data = "some bytes";
+    ForeignException in = new ForeignException("SRC", new IllegalArgumentException(data));
+    // check that we get the data back out
+    ForeignException e = ForeignException.deserialize(ForeignException.serialize(srcName, in));
+    assertNotNull(e);
+
+    // now check that we get the right stack trace
+    StackTraceElement elem = new StackTraceElement(this.getClass().toString(), "method", "file", 1);
+    in.setStackTrace(new StackTraceElement[] { elem });
+    e = ForeignException.deserialize(ForeignException.serialize(srcName, in));
+
+    assertNotNull(e);
+    assertEquals("Stack trace got corrupted", elem, e.getCause().getStackTrace()[0]);
+    assertEquals("Got an unexpectedly long stack trace", 1, e.getCause().getStackTrace().length);
+  }
+
+  /**
+   * Compare that a generic exception's stack trace has the same stack trace elements after
+   * serialization and deserialization
+   * @throws InvalidProtocolBufferException
+   */
+  @Test
+  public void testRemoteFromLocal() throws InvalidProtocolBufferException {
+    String errorMsg = "some message";
+    Exception generic = new Exception(errorMsg);
+    generic.printStackTrace();
+    assertTrue(generic.getMessage().contains(errorMsg));
+
+    ForeignException e = ForeignException.deserialize(ForeignException.serialize(srcName, generic));
+    assertArrayEquals("Local stack trace got corrupted", generic.getStackTrace(), e.getCause().getStackTrace());
+
+    e.printStackTrace(); // should have ForeignException and source node in it.
+    assertTrue(e.getCause().getCause() == null);
+
+    // verify that original error message is present in Foreign exception message
+    assertTrue(e.getCause().getMessage().contains(errorMsg));
+  }
+
+}

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import static org.junit.Assert.fail;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test the {@link TimeoutExceptionInjector} to ensure we fulfill contracts
+ */
+@Category(SmallTests.class)
+public class TestTimeoutExceptionInjector {
+
+  private static final Log LOG = LogFactory.getLog(TestTimeoutExceptionInjector.class);
+
+  /**
+   * Test that a manually triggered timer fires an exception.
+   */
+  @Test(timeout = 1000)
+  public void testTimerTrigger() {
+    final long time = 10000000; // pick a value that is very far in the future
+    ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time);
+    timer.start();
+    timer.trigger();
+    Mockito.verify(listener, Mockito.times(1)).receive(Mockito.any(ForeignException.class));
+  }
+
+  /**
+   * Test that a manually triggered exception with data fires with the data in receiveError.
+   */
+  @Test
+  public void testTimerPassesOnErrorInfo() {
+    final long time = 1000000;
+    ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time);
+    timer.start();
+    timer.trigger();
+    Mockito.verify(listener).receive(Mockito.any(ForeignException.class));
+  }
+
+  /**
+   * Demonstrate TimeoutExceptionInjector semantics -- completion means no more exceptions passed to
+   * error listener.
+   */
+  @Test(timeout = 1000)
+  public void testStartAfterComplete() throws InterruptedException {
+    final long time = 10;
+    ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time);
+    timer.complete();
+    try {
+      timer.start();
+      fail("Timer should fail to start after complete.");
+    } catch (IllegalStateException e) {
+      LOG.debug("Correctly failed timer: " + e.getMessage());
+    }
+    Thread.sleep(time + 1);
+    Mockito.verifyZeroInteractions(listener);
+  }
+
+  /**
+   * Demonstrate TimeoutExceptionInjector semantics -- triggering fires exception and completes
+   * the timer.
+   */
+  @Test(timeout = 1000)
+  public void testStartAfterTrigger() throws InterruptedException {
+    final long time = 10;
+    ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class);
+    TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time);
+    timer.trigger();
+    try {
+      timer.start();
+      fail("Timer should fail to start after complete.");
+    } catch (IllegalStateException e) {
+      LOG.debug("Correctly failed timer: " + e.getMessage());
+    }
+    Thread.sleep(time * 2);
+    Mockito.verify(listener, Mockito.times(1)).receive(Mockito.any(ForeignException.class));
+    Mockito.verifyNoMoreInteractions(listener);
+  }
+}

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import junit.framework.TestCase;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.io.FileLink;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test that FileLink switches between alternate locations
+ * when the current location moves or gets deleted.
+ */
+@Category(MediumTests.class)
+public class TestFileLink {
+  /**
+   * Test, on HDFS, that the FileLink is still readable
+   * even when the current file gets renamed.
+   */
+  @Test
+  public void testHDFSLinkReadDuringRename() throws Exception {
+    HBaseTestingUtility testUtil = new HBaseTestingUtility();
+    Configuration conf = testUtil.getConfiguration();
+    conf.setInt("dfs.blocksize", 1024 * 1024);
+    conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024);
+
+    testUtil.startMiniDFSCluster(1);
+    MiniDFSCluster cluster = testUtil.getDFSCluster();
+    FileSystem fs = cluster.getFileSystem();
+    assertEquals("hdfs", fs.getUri().getScheme());
+
+    try {
+      testLinkReadDuringRename(fs, testUtil.getDefaultRootDirPath());
+    } finally {
+      testUtil.shutdownMiniCluster();
+    }
+  }
+
+  /**
+   * Test, on a local filesystem, that the FileLink is still readable
+   * even when the current file gets renamed.
+   */
+  @Test
+  public void testLocalLinkReadDuringRename() throws IOException {
+    HBaseTestingUtility testUtil = new HBaseTestingUtility();
+    FileSystem fs = testUtil.getTestFileSystem();
+    assertEquals("file", fs.getUri().getScheme());
+    testLinkReadDuringRename(fs, testUtil.getDataTestDir());
+  }
+
+  /**
+   * Test that link is still readable even when the current file gets renamed.
+   */
+  private void testLinkReadDuringRename(FileSystem fs, Path rootDir) throws IOException {
+    Path originalPath = new Path(rootDir, "test.file");
+    Path archivedPath = new Path(rootDir, "archived.file");
+
+    writeSomeData(fs, originalPath, 256 << 20, (byte)2);
+
+    List<Path> files = new ArrayList<Path>();
+    files.add(originalPath);
+    files.add(archivedPath);
+
+    FileLink link = new FileLink(files);
+    FSDataInputStream in = link.open(fs);
+    try {
+      byte[] data = new byte[8192];
+      long size = 0;
+
+      // Read from origin
+      int n = in.read(data);
+      dataVerify(data, n, (byte)2);
+      size += n;
+
+      // Move origin to archive
+      assertFalse(fs.exists(archivedPath));
+      fs.rename(originalPath, archivedPath);
+      assertFalse(fs.exists(originalPath));
+      assertTrue(fs.exists(archivedPath));
+
+      // Try to read to the end
+      while ((n = in.read(data)) > 0) {
+        dataVerify(data, n, (byte)2);
+        size += n;
+      }
+
+      assertEquals(256 << 20, size);
+    } finally {
+      in.close();
+      if (fs.exists(originalPath)) fs.delete(originalPath);
+      if (fs.exists(archivedPath)) fs.delete(archivedPath);
+    }
+  }
+
+  /**
+   * Test that link is still readable even when the current file gets deleted.
+   *
+   * NOTE: This test is valid only on HDFS.
+   * When a file is deleted from a local file-system, it is simply 'unlinked'.
+   * The inode, which contains the file's data, is not deleted until all
+   * processes have finished with it.
+   * In HDFS when the request exceed the cached block locations,
+   * a query to the namenode is performed, using the filename,
+   * and the deleted file doesn't exists anymore (FileNotFoundException).
+   */
+  @Test
+  public void testHDFSLinkReadDuringDelete() throws Exception {
+    HBaseTestingUtility testUtil = new HBaseTestingUtility();
+    Configuration conf = testUtil.getConfiguration();
+    conf.setInt("dfs.blocksize", 1024 * 1024);
+    conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024);
+
+    testUtil.startMiniDFSCluster(1);
+    MiniDFSCluster cluster = testUtil.getDFSCluster();
+    FileSystem fs = cluster.getFileSystem();
+    assertEquals("hdfs", fs.getUri().getScheme());
+
+    try {
+      List<Path> files = new ArrayList<Path>();
+      for (int i = 0; i < 3; i++) {
+        Path path = new Path(String.format("test-data-%d", i));
+        writeSomeData(fs, path, 1 << 20, (byte)i);
+        files.add(path);
+      }
+
+      FileLink link = new FileLink(files);
+      FSDataInputStream in = link.open(fs);
+      try {
+        byte[] data = new byte[8192];
+        int n;
+
+        // Switch to file 1
+        n = in.read(data);
+        dataVerify(data, n, (byte)0);
+        fs.delete(files.get(0));
+        skipBuffer(in, (byte)0);
+
+        // Switch to file 2
+        n = in.read(data);
+        dataVerify(data, n, (byte)1);
+        fs.delete(files.get(1));
+        skipBuffer(in, (byte)1);
+
+        // Switch to file 3
+        n = in.read(data);
+        dataVerify(data, n, (byte)2);
+        fs.delete(files.get(2));
+        skipBuffer(in, (byte)2);
+
+        // No more files available
+        try {
+          n = in.read(data);
+          assert(n <= 0);
+        } catch (FileNotFoundException e) {
+          assertTrue(true);
+        }
+      } finally {
+        in.close();
+      }
+    } finally {
+      testUtil.shutdownMiniCluster();
+    }
+  }
+
+  /**
+   * Write up to 'size' bytes with value 'v' into a new file called 'path'.
+   */
+  private void writeSomeData (FileSystem fs, Path path, long size, byte v) throws IOException {
+    byte[] data = new byte[4096];
+    for (int i = 0; i < data.length; i++) {
+      data[i] = v;
+    }
+
+    FSDataOutputStream stream = fs.create(path);
+    try {
+      long written = 0;
+      while (written < size) {
+        stream.write(data, 0, data.length);
+        written += data.length;
+      }
+    } finally {
+      stream.close();
+    }
+  }
+
+  /**
+   * Verify that all bytes in 'data' have 'v' as value.
+   */
+  private static void dataVerify(byte[] data, int n, byte v) {
+    for (int i = 0; i < n; ++i) {
+      assertEquals(v, data[i]);
+    }
+  }
+
+  private static void skipBuffer(FSDataInputStream in, byte v) throws IOException {
+    byte[] data = new byte[8192];
+    try {
+      int n;
+      while ((n = in.read(data)) == data.length) {
+        for (int i = 0; i < data.length; ++i) {
+          if (data[i] != v)
+            throw new Exception("File changed");
+        }
+      }
+    } catch (Exception e) {
+    }
+  }
+}

Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java Mon Mar  4 11:24:50 2013
@@ -19,6 +19,9 @@
  */
 package org.apache.hadoop.hbase.io.hfile;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Map;
@@ -28,19 +31,18 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.LruBlockCache.EvictionThread;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.TestSchemaMetrics;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
-import org.junit.experimental.categories.Category;
-import static org.junit.Assert.*;
-
 /**
  * Tests the concurrent LruBlockCache.<p>
  *
@@ -77,7 +79,6 @@ public class TestLruBlockCache {
 
   @Test
   public void testBackgroundEvictionThread() throws Exception {
-
     long maxSize = 100000;
     long blockSize = calculateBlockSizeDefault(maxSize, 9); // room for 9, will evict
 
@@ -85,6 +86,14 @@ public class TestLruBlockCache {
 
     CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
 
+    EvictionThread evictionThread = cache.getEvictionThread();
+    assertTrue(evictionThread != null);
+
+    // Make sure eviction thread has entered run method
+    while (!evictionThread.isEnteringRun()) {
+      Thread.sleep(1);
+    }
+
     // Add all the blocks
     for (CachedItem block : blocks) {
       cache.cacheBlock(block.cacheKey, block);

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test the HFileLink Cleaner.
+ * HFiles with links cannot be deleted until a link is present.
+ */
+@Category(SmallTests.class)
+public class TestHFileLinkCleaner {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @Test
+  public void testHFileLinkCleaning() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDataTestDir().toString());
+    conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName());
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(conf);
+
+    final String tableName = "test-table";
+    final String tableLinkName = "test-link";
+    final String hfileName = "1234567890";
+    final String familyName = "cf";
+
+    HRegionInfo hri = new HRegionInfo(Bytes.toBytes(tableName));
+    HRegionInfo hriLink = new HRegionInfo(Bytes.toBytes(tableLinkName));
+
+    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
+    Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
+          tableName, hri.getEncodedName(), familyName);
+    Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
+          tableLinkName, hriLink.getEncodedName(), familyName);
+
+    // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf);
+    Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName);
+    fs.mkdirs(familyPath);
+    Path hfilePath = new Path(familyPath, hfileName);
+    fs.createNewFile(hfilePath);
+
+    // Create link to hfile
+    Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName,
+                                        hriLink.getEncodedName(), familyName);
+    fs.mkdirs(familyLinkPath);
+    HFileLink.create(conf, fs, familyLinkPath, hri, hfileName);
+    Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName);
+    assertTrue(fs.exists(linkBackRefDir));
+    FileStatus[] backRefs = fs.listStatus(linkBackRefDir);
+    assertEquals(1, backRefs.length);
+    Path linkBackRef = backRefs[0].getPath();
+
+    // Initialize cleaner
+    final long ttl = 1000;
+    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
+    Server server = new DummyServer();
+    HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir);
+
+    // Link backref cannot be removed
+    cleaner.chore();
+    assertTrue(fs.exists(linkBackRef));
+    assertTrue(fs.exists(hfilePath));
+
+    // Link backref can be removed
+    fs.rename(new Path(rootDir, tableLinkName), new Path(archiveDir, tableLinkName));
+    cleaner.chore();
+    assertFalse("Link should be deleted", fs.exists(linkBackRef));
+
+    // HFile can be removed
+    Thread.sleep(ttl * 2);
+    cleaner.chore();
+    assertFalse("HFile should be deleted", fs.exists(hfilePath));
+
+    // Remove everything
+    for (int i = 0; i < 4; ++i) {
+      Thread.sleep(ttl * 2);
+      cleaner.chore();
+    }
+    assertFalse("HFile should be deleted", fs.exists(new Path(archiveDir, tableName)));
+    assertFalse("Link should be deleted", fs.exists(new Path(archiveDir, tableLinkName)));
+
+    cleaner.interrupt();
+  }
+
+  private static Path getFamilyDirPath (final Path rootDir, final String table,
+    final String region, final String family) {
+    return new Path(new Path(new Path(rootDir, table), region), family);
+  }
+
+  static class DummyServer implements Server {
+
+    @Override
+    public Configuration getConfiguration() {
+      return TEST_UTIL.getConfiguration();
+    }
+
+    @Override
+    public ZooKeeperWatcher getZooKeeper() {
+      try {
+        return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+      return null;
+    }
+
+    @Override
+    public CatalogTracker getCatalogTracker() {
+      return null;
+    }
+
+    @Override
+    public ServerName getServerName() {
+      return new ServerName("regionserver,60020,000000");
+    }
+
+    @Override
+    public void abort(String why, Throwable e) {}
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+
+    @Override
+    public void stop(String why) {}
+
+    @Override
+    public boolean isStopped() {
+      return false;
+    }
+  }
+}

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,382 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.snapshot.DisabledTableSnapshotHandler;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Test the master-related aspects of a snapshot
+ */
+@Category(MediumTests.class)
+public class TestSnapshotFromMaster {
+
+  private static final Log LOG = LogFactory.getLog(TestSnapshotFromMaster.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final int NUM_RS = 2;
+  private static Path rootDir;
+  private static Path snapshots;
+  private static FileSystem fs;
+  private static HMaster master;
+
+  // for hfile archiving test.
+  private static Path archiveDir;
+  private static final String STRING_TABLE_NAME = "test";
+  private static final byte[] TEST_FAM = Bytes.toBytes("fam");
+  private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
+  // refresh the cache every 1/2 second
+  private static final long cacheRefreshPeriod = 500;
+
+  /**
+   * Setup the config for the cluster
+   */
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    setupConf(UTIL.getConfiguration());
+    UTIL.startMiniCluster(NUM_RS);
+    fs = UTIL.getDFSCluster().getFileSystem();
+    master = UTIL.getMiniHBaseCluster().getMaster();
+    rootDir = master.getMasterFileSystem().getRootDir();
+    snapshots = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+  }
+
+  private static void setupConf(Configuration conf) {
+    // disable the ui
+    conf.setInt("hbase.regionsever.info.port", -1);
+    // change the flush size to a small amount, regulating number of store files
+    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
+    // so make sure we get a compaction when doing a load, but keep around some
+    // files in the store
+    conf.setInt("hbase.hstore.compaction.min", 3);
+    conf.setInt("hbase.hstore.compactionThreshold", 5);
+    // block writes if we get to 12 store files
+    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
+    // drop the number of attempts for the hbase admin
+    conf.setInt("hbase.client.retries.number", 1);
+    // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner)
+    conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
+    conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, "");
+    // Enable snapshot
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod);
+
+    // prevent aggressive region split
+    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+      ConstantSizeRegionSplitPolicy.class.getName());
+  }
+
+  @Before
+  public void setup() throws Exception {
+    UTIL.createTable(TABLE_NAME, TEST_FAM);
+    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(null);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    UTIL.deleteTable(TABLE_NAME);
+
+    // delete the archive directory, if its exists
+    if (fs.exists(archiveDir)) {
+      if (!fs.delete(archiveDir, true)) {
+        throw new IOException("Couldn't delete archive directory (" + archiveDir
+            + " for an unknown reason");
+      }
+    }
+
+    // delete the snapshot directory, if its exists
+    if (fs.exists(snapshots)) {
+      if (!fs.delete(snapshots, true)) {
+        throw new IOException("Couldn't delete snapshots directory (" + snapshots
+            + " for an unknown reason");
+      }
+    }
+  }
+
+  @AfterClass
+  public static void cleanupTest() throws Exception {
+    try {
+      UTIL.shutdownMiniCluster();
+    } catch (Exception e) {
+      // NOOP;
+    }
+  }
+
+  /**
+   * Test that the contract from the master for checking on a snapshot are valid.
+   * <p>
+   * <ol>
+   * <li>If a snapshot fails with an error, we expect to get the source error.</li>
+   * <li>If there is no snapshot name supplied, we should get an error.</li>
+   * <li>If asking about a snapshot has hasn't occurred, you should get an error.</li>
+   * </ol>
+   */
+  @Test(timeout = 15000)
+  public void testIsDoneContract() throws Exception {
+
+    String snapshotName = "asyncExpectedFailureTest";
+
+    // check that we get an exception when looking up snapshot where one hasn't happened
+    SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(),
+      UnknownSnapshotException.class);
+
+    // and that we get the same issue, even if we specify a name
+    SnapshotDescription desc = SnapshotDescription.newBuilder()
+      .setName(snapshotName).build();
+    SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc),
+      UnknownSnapshotException.class);
+
+    // set a mock handler to simulate a snapshot
+    DisabledTableSnapshotHandler mockHandler = Mockito.mock(DisabledTableSnapshotHandler.class);
+    Mockito.when(mockHandler.getException()).thenReturn(null);
+    Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
+    Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true));
+
+    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(mockHandler);
+
+    // if we do a lookup without a snapshot name, we should fail - you should always know your name
+    SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(),
+      UnknownSnapshotException.class);
+
+    // then do the lookup for the snapshot that it is done
+    boolean isDone = master.isSnapshotDone(new HSnapshotDescription(desc));
+    assertTrue("Snapshot didn't complete when it should have.", isDone);
+
+    // now try the case where we are looking for a snapshot we didn't take
+    desc = SnapshotDescription.newBuilder().setName("Not A Snapshot").build();
+    SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc),
+      UnknownSnapshotException.class);
+
+    // then create a snapshot to the fs and make sure that we can find it when checking done
+    snapshotName = "completed";
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    desc = desc.toBuilder().setName(snapshotName).build();
+    SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshotDir, fs);
+
+    isDone = master.isSnapshotDone(new HSnapshotDescription(desc));
+    assertTrue("Completed, on-disk snapshot not found", isDone);
+  }
+
+  @Test
+  public void testGetCompletedSnapshots() throws Exception {
+    // first check when there are no snapshots
+    List<HSnapshotDescription> snapshots = master.getCompletedSnapshots();
+    assertEquals("Found unexpected number of snapshots", 0, snapshots.size());
+
+    // write one snapshot to the fs
+    String snapshotName = "completed";
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
+    SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);
+
+    // check that we get one snapshot
+    snapshots = master.getCompletedSnapshots();
+    assertEquals("Found unexpected number of snapshots", 1, snapshots.size());
+    List<HSnapshotDescription> expected = Lists.newArrayList(new HSnapshotDescription(snapshot));
+    assertEquals("Returned snapshots don't match created snapshots", expected, snapshots);
+
+    // write a second snapshot
+    snapshotName = "completed_two";
+    snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
+    SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);
+    expected.add(new HSnapshotDescription(snapshot));
+
+    // check that we get one snapshot
+    snapshots = master.getCompletedSnapshots();
+    assertEquals("Found unexpected number of snapshots", 2, snapshots.size());
+    assertEquals("Returned snapshots don't match created snapshots", expected, snapshots);
+  }
+
+  @Test
+  public void testDeleteSnapshot() throws Exception {
+
+    String snapshotName = "completed";
+    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
+
+    try {
+      master.deleteSnapshot(new HSnapshotDescription(snapshot));
+      fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist");
+    } catch (IOException e) {
+      LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage());
+    }
+
+    // write one snapshot to the fs
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);
+
+    // then delete the existing snapshot,which shouldn't cause an exception to be thrown
+    master.deleteSnapshot(new HSnapshotDescription(snapshot));
+  }
+
+  /**
+   * Test that the snapshot hfile archive cleaner works correctly. HFiles that are in snapshots
+   * should be retained, while those that are not in a snapshot should be deleted.
+   * @throws Exception on failure
+   */
+  @Test
+  public void testSnapshotHFileArchiving() throws Exception {
+    HBaseAdmin admin = UTIL.getHBaseAdmin();
+    // make sure we don't fail on listing snapshots
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+    // load the table
+    UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
+
+    // disable the table so we can take a snapshot
+    admin.disableTable(TABLE_NAME);
+
+    // take a snapshot of the table
+    String snapshotName = "snapshot";
+    byte[] snapshotNameBytes = Bytes.toBytes(snapshotName);
+    admin.snapshot(snapshotNameBytes, TABLE_NAME);
+
+    Configuration conf = master.getConfiguration();
+    LOG.info("After snapshot File-System state");
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // ensure we only have one snapshot
+    SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshotNameBytes, TABLE_NAME);
+
+    // renable the table so we can compact the regions
+    admin.enableTable(TABLE_NAME);
+
+    // compact the files so we get some archived files for the table we just snapshotted
+    List<HRegion> regions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
+    for (HRegion region : regions) {
+      region.waitForFlushesAndCompactions(); // enable can trigger a compaction, wait for it.
+      region.compactStores();
+    }
+    LOG.info("After compaction File-System state");
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // make sure the cleaner has run
+    LOG.debug("Running hfile cleaners");
+    ensureHFileCleanersRun();
+    LOG.info("After cleaners File-System state: " + rootDir);
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // get the snapshot files for the table
+    Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    FileStatus[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
+    // check that the files in the archive contain the ones that we need for the snapshot
+    LOG.debug("Have snapshot hfiles:");
+    for (FileStatus file : snapshotHFiles) {
+      LOG.debug(file.getPath());
+    }
+    // get the archived files for the table
+    Collection<String> files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME);
+
+    // and make sure that there is a proper subset
+    for (FileStatus file : snapshotHFiles) {
+      assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file.getPath(),
+        files.contains(file.getPath().getName()));
+    }
+
+    // delete the existing snapshot
+    admin.deleteSnapshot(snapshotNameBytes);
+    SnapshotTestingUtils.assertNoSnapshots(admin);
+
+    // make sure that we don't keep around the hfiles that aren't in a snapshot
+    // make sure we wait long enough to refresh the snapshot hfile
+    List<BaseHFileCleanerDelegate> delegates = UTIL.getMiniHBaseCluster().getMaster()
+        .getHFileCleaner().cleanersChain;
+    for (BaseHFileCleanerDelegate delegate: delegates) {
+      if (delegate instanceof SnapshotHFileCleaner) {
+        ((SnapshotHFileCleaner)delegate).getFileCacheForTesting().triggerCacheRefreshForTesting();
+      }
+    }
+    // run the cleaner again
+    LOG.debug("Running hfile cleaners");
+    ensureHFileCleanersRun();
+    LOG.info("After delete snapshot cleaners run File-System state");
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME);
+    assertEquals("Still have some hfiles in the archive, when their snapshot has been deleted.", 0,
+      files.size());
+  }
+
+  /**
+   * @return all the HFiles for a given table that have been archived
+   * @throws IOException on expected failure
+   */
+  private final Collection<String> getArchivedHFiles(Path archiveDir, Path rootDir,
+      FileSystem fs, String tableName) throws IOException {
+    Path tableArchive = new Path(archiveDir, tableName);
+    FileStatus[] archivedHFiles = SnapshotTestingUtils.listHFiles(fs, tableArchive);
+    List<String> files = new ArrayList<String>(archivedHFiles.length);
+    LOG.debug("Have archived hfiles: " + tableArchive);
+    for (FileStatus file : archivedHFiles) {
+      LOG.debug(file.getPath());
+      files.add(file.getPath().getName());
+    }
+    // sort the archived files
+
+    Collections.sort(files);
+    return files;
+  }
+
+  /**
+   * Make sure the {@link HFileCleaner HFileCleaners} run at least once
+   */
+  private static void ensureHFileCleanersRun() {
+    UTIL.getHBaseCluster().getMaster().getHFileCleaner().chore();
+  }
+}

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that we correctly reload the cache, filter directories, etc.
+ */
+@Category(MediumTests.class)
+public class TestSnapshotFileCache {
+
+  private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static FileSystem fs;
+  private static Path rootDir;
+
+  @BeforeClass
+  public static void startCluster() throws Exception {
+    UTIL.startMiniDFSCluster(1);
+    fs = UTIL.getDFSCluster().getFileSystem();
+    rootDir = UTIL.getDefaultRootDirPath();
+  }
+
+  @AfterClass
+  public static void stopCluster() throws Exception {
+    UTIL.shutdownMiniDFSCluster();
+  }
+
+  @After
+  public void cleanupFiles() throws Exception {
+    // cleanup the snapshot directory
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    fs.delete(snapshotDir, true);
+  }
+
+  @Test(timeout = 10000000)
+  public void testLoadAndDelete() throws Exception {
+    // don't refresh the cache unless we tell it to
+    long period = Long.MAX_VALUE;
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+        "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+    Path snapshot = new Path(snapshotDir, "snapshot");
+    Path region = new Path(snapshot, "7e91021");
+    Path family = new Path(region, "fam");
+    Path file1 = new Path(family, "file1");
+    Path file2 = new Path(family, "file2");
+
+    // create two hfiles under the snapshot
+    fs.create(file1);
+    fs.create(file2);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // then make sure the cache finds them
+    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
+    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+    String not = "file-shouldn't-be-found";
+    assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not));
+
+    // make sure we get a little bit of separation in the modification times
+    // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little
+    Thread.sleep(10);
+
+    LOG.debug("Deleting snapshot.");
+    // then delete the snapshot and make sure that we can still find the files
+    if (!fs.delete(snapshot, true)) {
+      throw new IOException("Couldn't delete " + snapshot + " for an unknown reason.");
+    }
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+
+    LOG.debug("Checking to see if file is deleted.");
+    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
+    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+
+    // then trigger a refresh
+    cache.triggerCacheRefreshForTesting();
+    // and not it shouldn't find those files
+    assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
+      cache.contains(file1.getName()));
+    assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
+      cache.contains(file2.getName()));
+
+    fs.delete(snapshotDir, true);
+  }
+
+  @Test
+  public void testLoadsTmpDir() throws Exception {
+    // don't refresh the cache unless we tell it to
+    long period = Long.MAX_VALUE;
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+        "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+    // create a file in a 'completed' snapshot
+    Path snapshot = new Path(snapshotDir, "snapshot");
+    Path region = new Path(snapshot, "7e91021");
+    Path family = new Path(region, "fam");
+    Path file1 = new Path(family, "file1");
+    fs.create(file1);
+
+    // create an 'in progress' snapshot
+    SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
+    snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+    region = new Path(snapshot, "7e91021");
+    family = new Path(region, "fam");
+    Path file2 = new Path(family, "file2");
+    fs.create(file2);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // then make sure the cache finds both files
+    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
+    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+  }
+
+  @Test
+  public void testJustFindLogsDirectory() throws Exception {
+    // don't refresh the cache unless we tell it to
+    long period = Long.MAX_VALUE;
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+        "test-snapshot-file-cache-refresh", new SnapshotFileCache.SnapshotFileInspector() {
+            public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+                throws IOException {
+              return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
+            }
+        });
+
+    // create a file in a 'completed' snapshot
+    Path snapshot = new Path(snapshotDir, "snapshot");
+    Path region = new Path(snapshot, "7e91021");
+    Path family = new Path(region, "fam");
+    Path file1 = new Path(family, "file1");
+    fs.create(file1);
+
+    // and another file in the logs directory
+    Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
+    Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
+    fs.create(log);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    // then make sure the cache only finds the log files
+    assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
+      cache.contains(file1.getName()));
+    assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
+  }
+
+  @Test
+  public void testReloadModifiedDirectory() throws IOException {
+    // don't refresh the cache unless we tell it to
+    long period = Long.MAX_VALUE;
+    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+        "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+    Path snapshot = new Path(snapshotDir, "snapshot");
+    Path region = new Path(snapshot, "7e91021");
+    Path family = new Path(region, "fam");
+    Path file1 = new Path(family, "file1");
+    Path file2 = new Path(family, "file2");
+
+    // create two hfiles under the snapshot
+    fs.create(file1);
+    fs.create(file2);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+    assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
+
+    // now delete the snapshot and add a file with a different name
+    fs.delete(snapshot, true);
+    Path file3 = new Path(family, "new_file");
+    fs.create(file3);
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+    assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
+  }
+
+  class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
+    public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
+      Collection<String> files =  new HashSet<String>();
+      files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
+      files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
+      return files;
+    }
+  };
+}

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot
+ */
+@Category(SmallTests.class)
+public class TestSnapshotHFileCleaner {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @AfterClass
+  public static void cleanup() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(conf);
+    // cleanup
+    fs.delete(rootDir, true);
+  }
+
+  @Test
+  public void testFindsSnapshotFilesWhenCleaning() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
+    Path rootDir = FSUtils.getRootDir(conf);
+    Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
+
+    FileSystem fs = FileSystem.get(conf);
+    SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
+    cleaner.setConf(conf);
+
+    // write an hfile to the snapshot directory
+    String snapshotName = "snapshot";
+    byte[] snapshot = Bytes.toBytes(snapshotName);
+    String table = "table";
+    byte[] tableName = Bytes.toBytes(table);
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    HRegionInfo mockRegion = new HRegionInfo(tableName);
+    Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
+    Path familyDir = new Path(regionSnapshotDir, "family");
+    // create a reference to a supposedly valid hfile
+    String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
+    Path refFile = new Path(familyDir, hfile);
+
+    // make sure the reference file exists
+    fs.create(refFile);
+
+    // create the hfile in the archive
+    fs.mkdirs(archivedHfileDir);
+    fs.createNewFile(new Path(archivedHfileDir, hfile));
+
+    // make sure that the file isn't deletable
+    assertFalse(cleaner.isFileDeletable(new Path(hfile)));
+  }
+}
\ No newline at end of file

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that the snapshot log cleaner finds logs referenced in a snapshot
+ */
+@Category(SmallTests.class)
+public class TestSnapshotLogCleaner {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @AfterClass
+  public static void cleanup() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(conf);
+    // cleanup
+    fs.delete(rootDir, true);
+  }
+
+  @Test
+  public void testFindsSnapshotFilesWhenCleaning() throws IOException {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
+    Path rootDir = FSUtils.getRootDir(conf);
+    FileSystem fs = FileSystem.get(conf);
+    SnapshotLogCleaner cleaner = new SnapshotLogCleaner();
+    cleaner.setConf(conf);
+
+    // write an hfile to the snapshot directory
+    String snapshotName = "snapshot";
+    byte[] snapshot = Bytes.toBytes(snapshotName);
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    Path snapshotLogDir = new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME);
+    String timestamp = "1339643343027";
+    String hostFromMaster = "localhost%2C59648%2C1339643336601";
+
+    Path hostSnapshotLogDir = new Path(snapshotLogDir, hostFromMaster);
+    String snapshotlogfile = hostFromMaster + "." + timestamp + ".hbase";
+
+    // add the reference to log in the snapshot
+    fs.create(new Path(hostSnapshotLogDir, snapshotlogfile));
+
+    // now check to see if that log file would get deleted.
+    Path oldlogDir = new Path(rootDir, ".oldlogs");
+    Path logFile = new Path(oldlogDir, snapshotlogfile);
+    fs.create(logFile);
+
+    // make sure that the file isn't deletable
+    assertFalse(cleaner.isFileDeletable(logFile));
+  }
+}
\ No newline at end of file

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.zookeeper.KeeperException;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test basic snapshot manager functionality
+ */
+@Category(SmallTests.class)
+public class TestSnapshotManager {
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  MasterServices services = Mockito.mock(MasterServices.class);
+  ProcedureCoordinator coordinator = Mockito.mock(ProcedureCoordinator.class);
+  ExecutorService pool = Mockito.mock(ExecutorService.class);
+  MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class);
+  FileSystem fs;
+  {
+    try {
+      fs = UTIL.getTestFileSystem();
+    } catch (IOException e) {
+      throw new RuntimeException("Couldn't get test filesystem", e);
+    }
+  }
+
+   private SnapshotManager getNewManager() throws IOException, KeeperException {
+    return getNewManager(UTIL.getConfiguration());
+  }
+
+  private SnapshotManager getNewManager(final Configuration conf) throws IOException, KeeperException {
+    Mockito.reset(services);
+    Mockito.when(services.getConfiguration()).thenReturn(conf);
+    Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
+    Mockito.when(mfs.getFileSystem()).thenReturn(fs);
+    Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir());
+    return new SnapshotManager(services, coordinator, pool);
+  }
+
+  @Test
+  public void testInProcess() throws KeeperException, IOException {
+    SnapshotManager manager = getNewManager();
+    TakeSnapshotHandler handler = Mockito.mock(TakeSnapshotHandler.class);
+    assertFalse("Manager is in process when there is no current handler", manager.isTakingSnapshot());
+    manager.setSnapshotHandlerForTesting(handler);
+    Mockito.when(handler.isFinished()).thenReturn(false);
+    assertTrue("Manager isn't in process when handler is running", manager.isTakingSnapshot());
+    Mockito.when(handler.isFinished()).thenReturn(true);
+    assertFalse("Manager is process when handler isn't running", manager.isTakingSnapshot());
+  }
+
+  /**
+   * Verify the snapshot support based on the configuration.
+   */
+  @Test
+  public void testSnapshotSupportConfiguration() throws Exception {
+    // No configuration (no cleaners, not enabled): snapshot feature disabled
+    Configuration conf = new Configuration();
+    SnapshotManager manager = getNewManager(conf);
+    assertFalse("Snapshot should be disabled with no configuration", isSnapshotSupported(manager));
+
+    // force snapshot feature to be enabled
+    conf = new Configuration();
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    manager = getNewManager(conf);
+    assertTrue("Snapshot should be enabled", isSnapshotSupported(manager));
+
+    // force snapshot feature to be disabled
+    conf = new Configuration();
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
+    manager = getNewManager(conf);
+    assertFalse("Snapshot should be disabled", isSnapshotSupported(manager));
+
+    // force snapshot feature to be disabled, even if cleaners are present
+    conf = new Configuration();
+    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
+      SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName());
+    conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, SnapshotLogCleaner.class.getName());
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
+    manager = getNewManager(conf);
+    assertFalse("Snapshot should be disabled", isSnapshotSupported(manager));
+
+    // cleaners are present, but missing snapshot enabled property
+    conf = new Configuration();
+    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
+      SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName());
+    conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, SnapshotLogCleaner.class.getName());
+    manager = getNewManager(conf);
+    assertTrue("Snapshot should be enabled, because cleaners are present",
+      isSnapshotSupported(manager));
+
+    // Create a "test snapshot"
+    Path rootDir = UTIL.getDataTestDir();
+    Path testSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
+      "testSnapshotSupportConfiguration", rootDir);
+    fs.mkdirs(testSnapshotDir);
+    try {
+      // force snapshot feature to be disabled, but snapshots are present
+      conf = new Configuration();
+      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
+      manager = getNewManager(conf);
+      fail("Master should not start when snapshot is disabled, but snapshots are present");
+    } catch (UnsupportedOperationException e) {
+      // expected
+    } finally {
+      fs.delete(testSnapshotDir, true);
+    }
+  }
+
+  private boolean isSnapshotSupported(final SnapshotManager manager) {
+    try {
+      manager.checkSnapshotSupport();
+      return true;
+    } catch (UnsupportedOperationException e) {
+      return false;
+    }
+  }
+}



Mime
View raw message