hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject svn commit: r1452257 [2/14] - in /hbase/branches/0.94: security/src/main/java/org/apache/hadoop/hbase/security/access/ security/src/test/java/org/apache/hadoop/hbase/security/access/ src/main/jamon/org/apache/hadoop/hbase/tmpl/master/ src/main/java/org...
Date Mon, 04 Mar 2013 11:24:53 GMT
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The ForeignExceptionListener is an interface for objects that can receive a ForeignException.
+ * <p>
+ * Implementations must be thread-safe, because this is expected to be used to propagate exceptions
+ * from foreign threads.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface ForeignExceptionListener {
+
+  /**
+   * Receive a ForeignException.
+   * <p>
+   * Implementers must ensure that this method is thread-safe.
+   * @param e exception causing the error. Implementations must accept and handle null here.
+   */
+  public void receive(ForeignException e);
+}
\ No newline at end of file

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * This is an interface for a cooperative exception throwing mechanism.  Implementations are
+ * containers that holds an exception from a separate thread. This can be used to receive
+ * exceptions from 'foreign' threads or from separate 'foreign' processes.
+ * <p>
+ * To use, one would pass an implementation of this object to a long running method and
+ * periodically check by calling {@link #rethrowException()}.  If any foreign exceptions have
+ * been received, the calling thread is then responsible for handling the rethrown exception.
+ * <p>
+ * One could use the boolean {@link #hasException()} to determine if there is an exceptoin as well.
+ * <p>
+ * NOTE: This is very similar to the InterruptedException/interrupt/interrupted pattern.  There,
+ * the notification state is bound to a Thread.  Using this, applications receive Exceptions in
+ * the snare.  The snare is referenced and checked by multiple threads which enables exception 
+ * notification in all the involved threads/processes. 
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface ForeignExceptionSnare {
+
+  /**
+   * Rethrow an exception currently held by the {@link ForeignExceptionSnare}. If there is
+   * no exception this is a no-op
+   *
+   * @throws ForeignException
+   *           all exceptions from remote sources are procedure exceptions
+   */
+  public void rethrowException() throws ForeignException;
+
+  /**
+   * Non-exceptional form of {@link #rethrowException()}. Checks to see if any
+   * process to which the exception checkers is bound has created an error that
+   * would cause a failure.
+   *
+   * @return <tt>true</tt> if there has been an error,<tt>false</tt> otherwise
+   */
+  public boolean hasException();
+
+  /**
+   * Get the value of the captured exception.
+   *
+   * @return the captured foreign exception or null if no exception captured.
+   */
+  public ForeignException getException();
+}

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Exception for timeout of a task.
+ * @see TimeoutExceptionInjector
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+@SuppressWarnings("serial")
+public class TimeoutException extends Exception {
+
+  private final String sourceName;
+  private final long start;
+  private final long end;
+  private final long expected;
+
+  /**
+   * Exception indicating that an operation attempt has timed out
+   * @param start time the operation started (ms since epoch)
+   * @param end time the timeout was triggered (ms since epoch)
+   * @param expected expected amount of time for the operation to complete (ms) (ideally, expected <= end-start)
+   */
+  public TimeoutException(String sourceName, long start, long end, long expected) {
+    super("Timeout elapsed! Source:" + sourceName + " Start:" + start + ", End:" + end
+        + ", diff:" + (end - start) + ", max:" + expected + " ms");
+    this.sourceName = sourceName;
+    this.start = start;
+    this.end = end;
+    this.expected = expected;
+  }
+
+  public long getStart() {
+    return start;
+  }
+
+  public long getEnd() {
+    return end;
+  }
+
+  public long getMaxAllowedOperationTime() {
+    return expected;
+  }
+
+  public String getSourceName() {
+    return sourceName;
+  }
+}
\ No newline at end of file

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.errorhandling;
+
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * Time a given process/operation and report a failure if the elapsed time exceeds the max allowed
+ * time.
+ * <p>
+ * The timer won't start tracking time until calling {@link #start()}. If {@link #complete()} or
+ * {@link #trigger()} is called before {@link #start()}, calls to {@link #start()} will fail.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class TimeoutExceptionInjector {
+
+  private static final Log LOG = LogFactory.getLog(TimeoutExceptionInjector.class);
+
+  private final long maxTime;
+  private volatile boolean complete;
+  private final Timer timer;
+  private final TimerTask timerTask;
+  private long start = -1;
+
+  /**
+   * Create a generic timer for a task/process.
+   * @param listener listener to notify if the process times out
+   * @param maxTime max allowed running time for the process. Timer starts on calls to
+   *          {@link #start()}
+   */
+  public TimeoutExceptionInjector(final ForeignExceptionListener listener, final long maxTime) {
+    this.maxTime = maxTime;
+    timer = new Timer();
+    timerTask = new TimerTask() {
+      @Override
+      public void run() {
+        // ensure we don't run this task multiple times
+        synchronized (this) {
+          // quick exit if we already marked the task complete
+          if (TimeoutExceptionInjector.this.complete) return;
+          // mark the task is run, to avoid repeats
+          TimeoutExceptionInjector.this.complete = true;
+        }
+        long end = EnvironmentEdgeManager.currentTimeMillis();
+        TimeoutException tee =  new TimeoutException(
+            "Timeout caused Foreign Exception", start, end, maxTime);
+        String source = "timer-" + timer;
+        listener.receive(new ForeignException(source, tee));
+      }
+    };
+  }
+
+  public long getMaxTime() {
+    return maxTime;
+  }
+
+  /**
+   * For all time forward, do not throw an error because the process has completed.
+   */
+  public void complete() {
+    // warn if the timer is already marked complete. This isn't going to be thread-safe, but should
+    // be good enough and its not worth locking just for a warning.
+    if (this.complete) {
+      LOG.warn("Timer already marked completed, ignoring!");
+      return;
+    }
+    LOG.debug("Marking timer as complete - no error notifications will be received for this timer.");
+    synchronized (this.timerTask) {
+      this.complete = true;
+    }
+    this.timer.cancel();
+  }
+
+  /**
+   * Start a timer to fail a process if it takes longer than the expected time to complete.
+   * <p>
+   * Non-blocking.
+   * @throws IllegalStateException if the timer has already been marked done via {@link #complete()}
+   *           or {@link #trigger()}
+   */
+  public synchronized void start() throws IllegalStateException {
+    if (this.start >= 0) {
+      LOG.warn("Timer already started, can't be started again. Ignoring second request.");
+      return;
+    }
+    LOG.debug("Scheduling process timer to run in: " + maxTime + " ms");
+    timer.schedule(timerTask, maxTime);
+    this.start = EnvironmentEdgeManager.currentTimeMillis();
+  }
+
+  /**
+   * Trigger the timer immediately.
+   * <p>
+   * Exposed for testing.
+   */
+  public void trigger() {
+    synchronized (timerTask) {
+      if (this.complete) {
+        LOG.warn("Timer already completed, not triggering.");
+        return;
+      }
+      LOG.debug("Triggering timer immediately!");
+      this.timer.cancel();
+      this.timerTask.run();
+    }
+  }
+}
\ No newline at end of file

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java Mon Mar  4 11:24:50 2013
@@ -128,6 +128,8 @@ public abstract class EventHandler imple
     C_M_DELETE_FAMILY         (45),   // Client asking Master to delete family of table
     C_M_MODIFY_FAMILY         (46),   // Client asking Master to modify family of table
     C_M_CREATE_TABLE          (47),   // Client asking Master to create a table
+    C_M_SNAPSHOT_TABLE        (48),   // Client asking Master to snapshot an offline table
+    C_M_RESTORE_SNAPSHOT      (49),   // Client asking Master to snapshot an offline table
 
     // Updates from master to ZK. This is done by the master and there is
     // nothing to process by either Master or RS

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java Mon Mar  4 11:24:50 2013
@@ -136,6 +136,8 @@ public class ExecutorService {
       case C_M_ENABLE_TABLE:
       case C_M_MODIFY_TABLE:
       case C_M_CREATE_TABLE:
+      case C_M_SNAPSHOT_TABLE:
+      case C_M_RESTORE_SNAPSHOT:
         return ExecutorType.MASTER_TABLE_OPERATIONS;
 
       // RegionServer executor services

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/FileLink.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/FileLink.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/FileLink.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/FileLink.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,455 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.util.Collection;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.FileNotFoundException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+
+/**
+ * The FileLink is a sort of hardlink, that allows to access a file given a set of locations.
+ *
+ * <p><b>The Problem:</b>
+ * <ul>
+ *  <li>
+ *    HDFS doesn't have support for hardlinks, and this make impossible to referencing
+ *    the same data blocks using different names.
+ *  </li>
+ *  <li>
+ *    HBase store files in one location (e.g. table/region/family/) and when the file is not
+ *    needed anymore (e.g. compaction, region deletetion, ...) moves it to an archive directory.
+ *  </li>
+ * </ul>
+ * If we want to create a reference to a file, we need to remember that it can be in its
+ * original location or in the archive folder.
+ * The FileLink class tries to abstract this concept and given a set of locations
+ * it is able to switch between them making this operation transparent for the user.
+ * More concrete implementations of the FileLink are the {@link HFileLink} and the {@link HLogLink}.
+ *
+ * <p><b>Back-references:</b>
+ * To help the {@link CleanerChore} to keep track of the links to a particular file,
+ * during the FileLink creation, a new file is placed inside a back-reference directory.
+ * There's one back-reference directory for each file that has links,
+ * and in the directory there's one file per link.
+ *
+ * <p>HFileLink Example
+ * <ul>
+ *  <li>
+ *      /hbase/table/region-x/cf/file-k
+ *      (Original File)
+ *  </li>
+ *  <li>
+ *      /hbase/table-cloned/region-y/cf/file-k.region-x.table
+ *     (HFileLink to the original file)
+ *  </li>
+ *  <li>
+ *      /hbase/table-2nd-cloned/region-z/cf/file-k.region-x.table
+ *      (HFileLink to the original file)
+ *  </li>
+ *  <li>
+ *      /hbase/.archive/table/region-x/.links-file-k/region-y.table-cloned
+ *      (Back-reference to the link in table-cloned)
+ *  </li>
+ *  <li>
+ *      /hbase/.archive/table/region-x/.links-file-k/region-z.table-cloned
+ *      (Back-reference to the link in table-2nd-cloned)
+ *  </li>
+ * </ul>
+ */
+@InterfaceAudience.Private
+public class FileLink {
+  private static final Log LOG = LogFactory.getLog(FileLink.class);
+
+  /** Define the Back-reference directory name prefix: .links-<hfile>/ */
+  public static final String BACK_REFERENCES_DIRECTORY_PREFIX = ".links-";
+
+  /**
+   * FileLink InputStream that handles the switch between the original path
+   * and the alternative locations, when the file is moved.
+   */
+  private static class FileLinkInputStream extends InputStream
+      implements Seekable, PositionedReadable {
+    private FSDataInputStream in = null;
+    private Path currentPath = null;
+    private long pos = 0;
+
+    private final FileLink fileLink;
+    private final int bufferSize;
+    private final FileSystem fs;
+
+    public FileLinkInputStream(final FileSystem fs, final FileLink fileLink)
+        throws IOException {
+      this(fs, fileLink, fs.getConf().getInt("io.file.buffer.size", 4096));
+    }
+
+    public FileLinkInputStream(final FileSystem fs, final FileLink fileLink, int bufferSize)
+        throws IOException {
+      this.bufferSize = bufferSize;
+      this.fileLink = fileLink;
+      this.fs = fs;
+
+      this.in = tryOpen();
+    }
+
+    @Override
+    public int read() throws IOException {
+      int res;
+      try {
+        res = in.read();
+      } catch (FileNotFoundException e) {
+        res = tryOpen().read();
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().read();
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().read();
+      }
+      if (res > 0) pos += 1;
+      return res;
+    }
+
+    @Override
+    public int read(byte b[]) throws IOException {
+       return read(b, 0, b.length);
+    }
+
+    @Override
+    public int read(byte b[], int off, int len) throws IOException {
+      int n;
+      try {
+        n = in.read(b, off, len);
+      } catch (FileNotFoundException e) {
+        n = tryOpen().read(b, off, len);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(b, off, len);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(b, off, len);
+      }
+      if (n > 0) pos += n;
+      assert(in.getPos() == pos);
+      return n;
+    }
+
+    @Override
+    public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+      int n;
+      try {
+        n = in.read(position, buffer, offset, length);
+      } catch (FileNotFoundException e) {
+        n = tryOpen().read(position, buffer, offset, length);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(position, buffer, offset, length);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(position, buffer, offset, length);
+      }
+      return n;
+    }
+
+    @Override
+    public void readFully(long position, byte[] buffer) throws IOException {
+      readFully(position, buffer, 0, buffer.length);
+    }
+
+    @Override
+    public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
+      try {
+        in.readFully(position, buffer, offset, length);
+      } catch (FileNotFoundException e) {
+        tryOpen().readFully(position, buffer, offset, length);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().readFully(position, buffer, offset, length);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().readFully(position, buffer, offset, length);
+      }
+    }
+
+    @Override
+    public long skip(long n) throws IOException {
+      long skipped;
+
+      try {
+        skipped = in.skip(n);
+      } catch (FileNotFoundException e) {
+        skipped = tryOpen().skip(n);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        skipped = tryOpen().skip(n);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        skipped = tryOpen().skip(n);
+      }
+
+      if (skipped > 0) pos += skipped;
+      return skipped;
+    }
+
+    @Override
+    public int available() throws IOException {
+      try {
+        return in.available();
+      } catch (FileNotFoundException e) {
+        return tryOpen().available();
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        return tryOpen().available();
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        return tryOpen().available();
+      }
+    }
+
+    @Override
+    public void seek(long pos) throws IOException {
+      try {
+        in.seek(pos);
+      } catch (FileNotFoundException e) {
+        tryOpen().seek(pos);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().seek(pos);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().seek(pos);
+      }
+      this.pos = pos;
+    }
+
+    @Override
+    public long getPos() throws IOException {
+      return pos;
+    }
+
+    @Override
+    public boolean seekToNewSource(long targetPos) throws IOException {
+      boolean res;
+      try {
+        res = in.seekToNewSource(targetPos);
+      } catch (FileNotFoundException e) {
+        res = tryOpen().seekToNewSource(targetPos);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().seekToNewSource(targetPos);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().seekToNewSource(targetPos);
+      }
+      if (res) pos = targetPos;
+      return res;
+    }
+
+    @Override
+    public void close() throws IOException {
+      in.close();
+    }
+
+    @Override
+    public synchronized void mark(int readlimit) {
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+      throw new IOException("mark/reset not supported");
+    }
+
+    @Override
+    public boolean markSupported() {
+      return false;
+    }
+
+    /**
+     * Try to open the file from one of the available locations.
+     *
+     * @return FSDataInputStream stream of the opened file link
+     * @throws IOException on unexpected error, or file not found.
+     */
+    private FSDataInputStream tryOpen() throws IOException {
+      for (Path path: fileLink.getLocations()) {
+        if (path.equals(currentPath)) continue;
+        try {
+          in = fs.open(path, bufferSize);
+          in.seek(pos);
+          assert(in.getPos() == pos) : "Link unable to seek to the right position=" + pos;
+          if (LOG.isTraceEnabled()) {
+            if (currentPath != null) {
+              LOG.debug("link open path=" + path);
+            } else {
+              LOG.trace("link switch from path=" + currentPath + " to path=" + path);
+            }
+          }
+          currentPath = path;
+          return(in);
+        } catch (FileNotFoundException e) {
+          // Try another file location
+        }
+      }
+      throw new FileNotFoundException("Unable to open link: " + fileLink);
+    }
+  }
+
+  private Path[] locations = null;
+
+  protected FileLink() {
+    this.locations = null;
+  }
+
+  /**
+   * @param originPath Original location of the file to link
+   * @param alternativePaths Alternative locations to look for the linked file
+   */
+  public FileLink(Path originPath, Path... alternativePaths) {
+    setLocations(originPath, alternativePaths);
+  }
+
+  /**
+   * @param locations locations to look for the linked file
+   */
+  public FileLink(final Collection<Path> locations) {
+    this.locations = locations.toArray(new Path[locations.size()]);
+  }
+
+  /**
+   * @return the locations to look for the linked file.
+   */
+  public Path[] getLocations() {
+    return locations;
+  }
+
+  public String toString() {
+    StringBuilder str = new StringBuilder(getClass().getName());
+    str.append(" locations=[");
+    int i = 0;
+    for (Path location: locations) {
+      if (i++ > 0) str.append(", ");
+      str.append(location.toString());
+    }
+    str.append("]");
+    return str.toString();
+  }
+
+  /**
+   * @return the path of the first available link.
+   */
+  public Path getAvailablePath(FileSystem fs) throws IOException {
+    for (Path path: locations) {
+      if (fs.exists(path)) {
+        return path;
+      }
+    }
+    throw new FileNotFoundException("Unable to open link: " + this);
+  }
+
+  /**
+   * Get the FileStatus of the referenced file.
+   *
+   * @param fs {@link FileSystem} on which to get the file status
+   * @return InputStream for the hfile link.
+   * @throws IOException on unexpected error.
+   */
+  public FileStatus getFileStatus(FileSystem fs) throws IOException {
+    for (Path path: locations) {
+      try {
+        return fs.getFileStatus(path);
+      } catch (FileNotFoundException e) {
+        // Try another file location
+      }
+    }
+    throw new FileNotFoundException("Unable to open link: " + this);
+  }
+
+  /**
+   * Open the FileLink for read.
+   * <p>
+   * It uses a wrapper of FSDataInputStream that is agnostic to the location
+   * of the file, even if the file switches between locations.
+   *
+   * @param fs {@link FileSystem} on which to open the FileLink
+   * @return InputStream for reading the file link.
+   * @throws IOException on unexpected error.
+   */
+  public FSDataInputStream open(final FileSystem fs) throws IOException {
+    return new FSDataInputStream(new FileLinkInputStream(fs, this));
+  }
+
+  /**
+   * Open the FileLink for read.
+   * <p>
+   * It uses a wrapper of FSDataInputStream that is agnostic to the location
+   * of the file, even if the file switches between locations.
+   *
+   * @param fs {@link FileSystem} on which to open the FileLink
+   * @param bufferSize the size of the buffer to be used.
+   * @return InputStream for reading the file link.
+   * @throws IOException on unexpected error.
+   */
+  public FSDataInputStream open(final FileSystem fs, int bufferSize) throws IOException {
+    return new FSDataInputStream(new FileLinkInputStream(fs, this, bufferSize));
+  }
+
+  /**
+   * NOTE: This method must be used only in the constructor!
+   * It creates a List with the specified locations for the link.
+   */
+  protected void setLocations(Path originPath, Path... alternativePaths) {
+    assert this.locations == null : "Link locations already set";
+    this.locations = new Path[1 + alternativePaths.length];
+    this.locations[0] = originPath;
+    for (int i = 0; i < alternativePaths.length; i++) {
+      this.locations[i + 1] = alternativePaths[i];
+    }
+  }
+
+  /**
+   * Get the directory to store the link back references
+   *
+   * <p>To simplify the reference count process, during the FileLink creation
+   * a back-reference is added to the back-reference directory of the specified file.
+   *
+   * @param storeDir Root directory for the link reference folder
+   * @param fileName File Name with links
+   * @return Path for the link back references.
+   */
+  public static Path getBackReferencesDir(final Path storeDir, final String fileName) {
+    return new Path(storeDir, BACK_REFERENCES_DIRECTORY_PREFIX + fileName);
+  }
+
+  /**
+   * Get the referenced file name from the reference link directory path.
+   *
+   * @param dirPath Link references directory path
+   * @return Name of the file referenced
+   */
+  public static String getBackReferenceFileName(final Path dirPath) {
+    return dirPath.getName().substring(BACK_REFERENCES_DIRECTORY_PREFIX.length());
+  }
+
+  /**
+   * Checks if the specified directory path is a back reference links folder.
+   *
+   * @param dirPath Directory path to verify
+   * @return True if the specified directory is a link references folder
+   */
+  public static boolean isBackReferencesDir(final Path dirPath) {
+    if (dirPath == null) return false;
+    return dirPath.getName().startsWith(BACK_REFERENCES_DIRECTORY_PREFIX);
+  }
+}
+

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,371 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+
+/**
+ * HFileLink describes a link to an hfile.
+ *
+ * An hfile can be served from a region or from the hfile archive directory (/hbase/.archive)
+ * HFileLink allows to access the referenced hfile regardless of the location where it is.
+ *
+ * <p>Searches for hfiles in the following order and locations:
+ * <ul>
+ *  <li>/hbase/table/region/cf/hfile</li>
+ *  <li>/hbase/.archive/table/region/cf/hfile</li>
+ * </ul>
+ *
+ * The link checks first in the original path if it is not present
+ * it fallbacks to the archived path.
+ */
+@InterfaceAudience.Private
+public class HFileLink extends FileLink {
+  private static final Log LOG = LogFactory.getLog(HFileLink.class);
+
+  /**
+   * A non-capture group, for HFileLink, so that this can be embedded.
+   * The HFileLink describe a link to an hfile in a different table/region
+   * and the name is in the form: table=region-hfile.
+   * <p>
+   * Table name is ([a-zA-Z_0-9][a-zA-Z_0-9.-]*), so '=' is an invalid character for the table name.
+   * Region name is ([a-f0-9]+), so '-' is an invalid character for the region name.
+   * HFile is ([0-9a-f]+(?:_SeqId_[0-9]+_)?) covering the plain hfiles (uuid)
+   * and the bulk loaded (_SeqId_[0-9]+_) hfiles.
+   */
+  public static final String LINK_NAME_REGEX =
+    String.format("%s=%s-%s", HTableDescriptor.VALID_USER_TABLE_REGEX,
+      HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFile.HFILE_NAME_REGEX);
+
+  /** Define the HFile Link name parser in the form of: table=region-hfile */
+  private static final Pattern LINK_NAME_PATTERN =
+    Pattern.compile(String.format("^(%s)=(%s)-(%s)$", HTableDescriptor.VALID_USER_TABLE_REGEX,
+      HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFile.HFILE_NAME_REGEX));
+
+  /**
+   * The pattern should be used for hfile and reference links
+   * that can be found in /hbase/table/region/family/
+   */
+  private static final Pattern REF_OR_HFILE_LINK_PATTERN =
+    Pattern.compile(String.format("^(%s)=(%s)-(.+)$", HTableDescriptor.VALID_USER_TABLE_REGEX,
+      HRegionInfo.ENCODED_REGION_NAME_REGEX));
+
+  private final Path archivePath;
+  private final Path originPath;
+  private final Path tempPath;
+
+  /**
+   * @param conf {@link Configuration} from which to extract specific archive locations
+   * @param path The path of the HFile Link.
+   * @throws IOException on unexpected error.
+   */
+  public HFileLink(Configuration conf, Path path) throws IOException {
+    this(FSUtils.getRootDir(conf), HFileArchiveUtil.getArchivePath(conf), path);
+  }
+
+  /**
+   * @param rootDir Path to the root directory where hbase files are stored
+   * @param archiveDir Path to the hbase archive directory
+   * @param path The path of the HFile Link.
+   */
+  public HFileLink(final Path rootDir, final Path archiveDir, final Path path) {
+    Path hfilePath = getRelativeTablePath(path);
+    this.tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath);
+    this.originPath = new Path(rootDir, hfilePath);
+    this.archivePath = new Path(archiveDir, hfilePath);
+    setLocations(originPath, tempPath, archivePath);
+  }
+
+  /**
+   * @return the origin path of the hfile.
+   */
+  public Path getOriginPath() {
+    return this.originPath;
+  }
+
+  /**
+   * @return the path of the archived hfile.
+   */
+  public Path getArchivePath() {
+    return this.archivePath;
+  }
+
+  /**
+   * @param path Path to check.
+   * @return True if the path is a HFileLink.
+   */
+  public static boolean isHFileLink(final Path path) {
+    return isHFileLink(path.getName());
+  }
+
+
+  /**
+   * @param fileName File name to check.
+   * @return True if the path is a HFileLink.
+   */
+  public static boolean isHFileLink(String fileName) {
+    Matcher m = LINK_NAME_PATTERN.matcher(fileName);
+    if (!m.matches()) return false;
+
+    return m.groupCount() > 2 && m.group(3) != null && m.group(2) != null && m.group(1) != null;
+  }
+
+  /**
+   * Convert a HFileLink path to a table relative path.
+   * e.g. the link: /hbase/test/0123/cf/testtb=4567-abcd
+   *      becomes: /hbase/testtb/4567/cf/abcd
+   *
+   * @param path HFileLink path
+   * @return Relative table path
+   * @throws IOException on unexpected error.
+   */
+  private static Path getRelativeTablePath(final Path path) {
+    // table=region-hfile
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(path.getName());
+    if (!m.matches()) {
+      throw new IllegalArgumentException(path.getName() + " is not a valid HFileLink name!");
+    }
+
+    // Convert the HFileLink name into a real table/region/cf/hfile path.
+    String tableName = m.group(1);
+    String regionName = m.group(2);
+    String hfileName = m.group(3);
+    String familyName = path.getParent().getName();
+    return new Path(new Path(tableName, regionName), new Path(familyName, hfileName));
+  }
+
+  /**
+   * Get the HFile name of the referenced link
+   *
+   * @param fileName HFileLink file name
+   * @return the name of the referenced HFile
+   */
+  public static String getReferencedHFileName(final String fileName) {
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
+    }
+    return(m.group(3));
+  }
+
+  /**
+   * Get the Region name of the referenced link
+   *
+   * @param fileName HFileLink file name
+   * @return the name of the referenced Region
+   */
+  public static String getReferencedRegionName(final String fileName) {
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
+    }
+    return(m.group(2));
+  }
+
+  /**
+   * Get the Table name of the referenced link
+   *
+   * @param fileName HFileLink file name
+   * @return the name of the referenced Table
+   */
+  public static String getReferencedTableName(final String fileName) {
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
+    }
+    return(m.group(1));
+  }
+
+  /**
+   * Create a new HFileLink name
+   *
+   * @param hfileRegionInfo - Linked HFile Region Info
+   * @param hfileName - Linked HFile name
+   * @return file name of the HFile Link
+   */
+  public static String createHFileLinkName(final HRegionInfo hfileRegionInfo,
+      final String hfileName) {
+    return createHFileLinkName(hfileRegionInfo.getTableNameAsString(),
+                      hfileRegionInfo.getEncodedName(), hfileName);
+  }
+
+  /**
+   * Create a new HFileLink name
+   *
+   * @param tableName - Linked HFile table name
+   * @param regionName - Linked HFile region name
+   * @param hfileName - Linked HFile name
+   * @return file name of the HFile Link
+   */
+  public static String createHFileLinkName(final String tableName,
+      final String regionName, final String hfileName) {
+    return String.format("%s=%s-%s", tableName, regionName, hfileName);
+  }
+
+  /**
+   * Create a new HFileLink
+   *
+   * <p>It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param hfileRegionInfo - Linked HFile Region Info
+   * @param hfileName - Linked HFile name
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean create(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final HRegionInfo hfileRegionInfo,
+      final String hfileName) throws IOException {
+    String linkedTable = hfileRegionInfo.getTableNameAsString();
+    String linkedRegion = hfileRegionInfo.getEncodedName();
+    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName);
+  }
+
+  /**
+   * Create a new HFileLink
+   *
+   * <p>It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param linkedTable - Linked Table Name
+   * @param linkedRegion - Linked Region Name
+   * @param hfileName - Linked HFile name
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean create(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final String linkedTable, final String linkedRegion,
+      final String hfileName) throws IOException {
+    String familyName = dstFamilyPath.getName();
+    String regionName = dstFamilyPath.getParent().getName();
+    String tableName = dstFamilyPath.getParent().getParent().getName();
+
+    String name = createHFileLinkName(linkedTable, linkedRegion, hfileName);
+    String refName = createBackReferenceName(tableName, regionName);
+
+    // Make sure the destination directory exists
+    fs.mkdirs(dstFamilyPath);
+
+    // Make sure the FileLink reference directory exists
+    Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
+          linkedTable, linkedRegion, familyName);
+    Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
+    fs.mkdirs(backRefssDir);
+
+    // Create the reference for the link
+    Path backRefPath = new Path(backRefssDir, refName);
+    fs.createNewFile(backRefPath);
+    try {
+      // Create the link
+      return fs.createNewFile(new Path(dstFamilyPath, name));
+    } catch (IOException e) {
+      LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
+      // Revert the reference if the link creation failed
+      fs.delete(backRefPath, false);
+      throw e;
+    }
+  }
+
+  /**
+   * Create a new HFileLink starting from a hfileLink name
+   *
+   * <p>It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final String hfileLinkName) throws IOException {
+    Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
+    }
+    return create(conf, fs, dstFamilyPath, m.group(1), m.group(2), m.group(3));
+  }
+
+  /**
+   * Create the back reference name
+   */
+  private static String createBackReferenceName(final String tableName, final String regionName) {
+    return regionName + "." + tableName;
+  }
+
+  /**
+   * Get the full path of the HFile referenced by the back reference
+   *
+   * @param rootDir root hbase directory
+   * @param linkRefPath Link Back Reference path
+   * @return full path of the referenced hfile
+   * @throws IOException on unexpected error.
+   */
+  public static Path getHFileFromBackReference(final Path rootDir, final Path linkRefPath) {
+    int separatorIndex = linkRefPath.getName().indexOf('.');
+    String linkRegionName = linkRefPath.getName().substring(0, separatorIndex);
+    String linkTableName = linkRefPath.getName().substring(separatorIndex + 1);
+    String hfileName = getBackReferenceFileName(linkRefPath.getParent());
+    Path familyPath = linkRefPath.getParent().getParent();
+    Path regionPath = familyPath.getParent();
+    Path tablePath = regionPath.getParent();
+
+    String linkName = createHFileLinkName(tablePath.getName(), regionPath.getName(), hfileName);
+    Path linkTableDir = FSUtils.getTablePath(rootDir, linkTableName);
+    Path regionDir = HRegion.getRegionDir(linkTableDir, linkRegionName);
+    return new Path(new Path(regionDir, familyPath.getName()), linkName);
+  }
+
+  /**
+   * Get the full path of the HFile referenced by the back reference
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param linkRefPath Link Back Reference path
+   * @return full path of the referenced hfile
+   * @throws IOException on unexpected error.
+   */
+  public static Path getHFileFromBackReference(final Configuration conf, final Path linkRefPath)
+      throws IOException {
+    return getHFileFromBackReference(FSUtils.getRootDir(conf), linkRefPath);
+  }
+}

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * HLogLink describes a link to a WAL.
+ *
+ * An hlog can be in /hbase/.logs/<server>/<hlog>
+ * or it can be in /hbase/.oldlogs/<hlog>
+ *
+ * The link checks first in the original path,
+ * if it is not present it fallbacks to the archived path.
+ */
+@InterfaceAudience.Private
+public class HLogLink extends FileLink {
+  /**
+   * @param conf {@link Configuration} from which to extract specific archive locations
+   * @param serverName Region Server owner of the log
+   * @param logName WAL file name
+   * @throws IOException on unexpected error.
+   */
+  public HLogLink(final Configuration conf,
+      final String serverName, final String logName) throws IOException {
+    this(FSUtils.getRootDir(conf), serverName, logName);
+  }
+
+  /**
+   * @param rootDir Path to the root directory where hbase files are stored
+   * @param serverName Region Server owner of the log
+   * @param logName WAL file name
+   */
+  public HLogLink(final Path rootDir, final String serverName, final String logName) {
+    final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+    final Path logDir = new Path(new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), serverName);
+    setLocations(new Path(logDir, logName), new Path(oldLogDir, logName));
+  }
+
+  /**
+   * @param originPath Path to the wal in the log directory
+   * @param archivePath Path to the wal in the archived log directory
+   */
+  public HLogLink(final Path originPath, final Path archivePath) {
+    setLocations(originPath, archivePath);
+  }
+}

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java Mon Mar  4 11:24:50 2013
@@ -58,10 +58,12 @@ public class HalfStoreFileReader extends
   private boolean firstKeySeeked = false;
 
   /**
-   * @param fs
-   * @param p
+   * Creates a half file reader for a normal hfile.
+   * @param fs fileystem to read from
+   * @param p path to hfile
    * @param cacheConf
-   * @param r
+   * @param r original reference file (contains top or bottom)
+   * @param preferredEncodingInCache
    * @throws IOException
    */
   public HalfStoreFileReader(final FileSystem fs, final Path p,
@@ -78,6 +80,30 @@ public class HalfStoreFileReader extends
     this.top = Reference.isTopFileRegion(r.getFileRegion());
   }
 
+  /**
+   * Creates a half file reader for a hfile referred to by an hfilelink.
+   * @param fs fileystem to read from
+   * @param p path to hfile
+   * @param link
+   * @param cacheConf
+   * @param r original reference file (contains top or bottom)
+   * @param preferredEncodingInCache
+   * @throws IOException
+   */
+  public HalfStoreFileReader(final FileSystem fs, final Path p, final HFileLink link,
+      final CacheConfig cacheConf, final Reference r,
+      DataBlockEncoding preferredEncodingInCache) throws IOException {
+    super(fs, p, link, link.getFileStatus(fs).getLen(), cacheConf, preferredEncodingInCache, true);
+    // This is not actual midkey for this half-file; its just border
+    // around which we split top and bottom.  Have to look in files to find
+    // actual last and first keys for bottom and top halves.  Half-files don't
+    // have an actual midkey themselves. No midkey is how we indicate file is
+    // not splittable.
+    this.splitkey = r.getSplitKey();
+    // Is it top or bottom half?
+    this.top = Reference.isTopFileRegion(r.getFileRegion());
+  }
+
   protected boolean isTop() {
     return this.top;
   }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java Mon Mar  4 11:24:50 2013
@@ -90,6 +90,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ProtoUtil;
 import org.apache.hadoop.io.MapWritable;
@@ -269,6 +270,10 @@ public class HbaseObjectWritable impleme
     
     addToMap(FuzzyRowFilter.class, code++);
 
+    // we aren't going to bump the rpc version number.
+    // we don't want to cause incompatiblity with older 0.94/0.92 clients.
+    addToMap(HSnapshotDescription.class, code);
+
     // make sure that this is the last statement in this static block
     NEXT_CLASS_CODE = code;
   }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Mon Mar  4 11:24:50 2013
@@ -180,18 +180,18 @@ public class HFile {
   static final AtomicLong checksumFailures = new AtomicLong();
 
   // For getting more detailed stats on FS latencies
-  // If, for some reason, the metrics subsystem stops polling for latencies, 
+  // If, for some reason, the metrics subsystem stops polling for latencies,
   // I don't want data to pile up in a memory leak
   // so, after LATENCY_BUFFER_SIZE items have been enqueued for processing,
   // fs latency stats will be dropped (and this behavior will be logged)
   private static final int LATENCY_BUFFER_SIZE = 5000;
-  private static final BlockingQueue<Long> fsReadLatenciesNanos = 
+  private static final BlockingQueue<Long> fsReadLatenciesNanos =
       new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
-  private static final BlockingQueue<Long> fsWriteLatenciesNanos = 
+  private static final BlockingQueue<Long> fsWriteLatenciesNanos =
       new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
-  private static final BlockingQueue<Long> fsPreadLatenciesNanos = 
+  private static final BlockingQueue<Long> fsPreadLatenciesNanos =
       new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
-  
+
   public static final void offerReadLatency(long latencyNanos, boolean pread) {
     if (pread) {
       fsPreadLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
@@ -203,30 +203,30 @@ public class HFile {
       readOps.incrementAndGet();
     }
   }
-  
+
   public static final void offerWriteLatency(long latencyNanos) {
     fsWriteLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
-    
+
     writeTimeNano.addAndGet(latencyNanos);
     writeOps.incrementAndGet();
   }
-  
+
   public static final Collection<Long> getReadLatenciesNanos() {
-    final List<Long> latencies = 
+    final List<Long> latencies =
         Lists.newArrayListWithCapacity(fsReadLatenciesNanos.size());
     fsReadLatenciesNanos.drainTo(latencies);
     return latencies;
   }
 
   public static final Collection<Long> getPreadLatenciesNanos() {
-    final List<Long> latencies = 
+    final List<Long> latencies =
         Lists.newArrayListWithCapacity(fsPreadLatenciesNanos.size());
     fsPreadLatenciesNanos.drainTo(latencies);
     return latencies;
   }
-  
+
   public static final Collection<Long> getWriteLatenciesNanos() {
-    final List<Long> latencies = 
+    final List<Long> latencies =
         Lists.newArrayListWithCapacity(fsWriteLatenciesNanos.size());
     fsWriteLatenciesNanos.drainTo(latencies);
     return latencies;
@@ -572,7 +572,7 @@ public class HFile {
     HFileSystem hfs = null;
     FSDataInputStream fsdis = fs.open(path);
     FSDataInputStream fsdisNoFsChecksum = fsdis;
-    // If the fs is not an instance of HFileSystem, then create an 
+    // If the fs is not an instance of HFileSystem, then create an
     // instance of HFileSystem that wraps over the specified fs.
     // In this case, we will not be able to avoid checksumming inside
     // the filesystem.
@@ -592,6 +592,39 @@ public class HFile {
   }
 
   /**
+   * @param fs A file system
+   * @param path Path to HFile
+   * @param fsdis an open checksummed stream of path's file
+   * @param fsdisNoFsChecksum an open unchecksummed stream of path's file
+   * @param size max size of the trailer.
+   * @param cacheConf Cache configuration for hfile's contents
+   * @param preferredEncodingInCache Preferred in-cache data encoding algorithm.
+   * @param closeIStream boolean for closing file after the getting the reader version.
+   * @return A version specific Hfile Reader
+   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
+   */
+  public static Reader createReaderWithEncoding(
+      FileSystem fs, Path path, FSDataInputStream fsdis,
+      FSDataInputStream fsdisNoFsChecksum, long size, CacheConfig cacheConf,
+      DataBlockEncoding preferredEncodingInCache, boolean closeIStream)
+      throws IOException {
+    HFileSystem hfs = null;
+
+    // If the fs is not an instance of HFileSystem, then create an
+    // instance of HFileSystem that wraps over the specified fs.
+    // In this case, we will not be able to avoid checksumming inside
+    // the filesystem.
+    if (!(fs instanceof HFileSystem)) {
+      hfs = new HFileSystem(fs);
+    } else {
+      hfs = (HFileSystem)fs;
+    }
+    return pickReaderVersion(path, fsdis, fsdisNoFsChecksum, size,
+                             closeIStream, cacheConf,
+                             preferredEncodingInCache, hfs);
+  }
+
+  /**
    * @param fs filesystem
    * @param path Path to file to read
    * @param cacheConf This must not be null.  @see {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#CacheConfig(Configuration)}

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Mon Mar  4 11:24:50 2013
@@ -581,15 +581,21 @@ public class LruBlockCache implements Bl
     return this.stats.getEvictedCount();
   }
 
+  EvictionThread getEvictionThread() {
+    return this.evictionThread;
+  }
+
   /*
    * Eviction thread.  Sits in waiting state until an eviction is triggered
    * when the cache size grows above the acceptable level.<p>
    *
    * Thread is triggered into action by {@link LruBlockCache#runEviction()}
    */
-  private static class EvictionThread extends HasThread {
+  static class EvictionThread extends HasThread {
     private WeakReference<LruBlockCache> cache;
     private boolean go = true;
+    // flag set after enter the run method, used for test
+    private boolean enteringRun = false;
 
     public EvictionThread(LruBlockCache cache) {
       super(Thread.currentThread().getName() + ".LruBlockCache.EvictionThread");
@@ -599,6 +605,7 @@ public class LruBlockCache implements Bl
 
     @Override
     public void run() {
+      enteringRun = true;
       while (this.go) {
         synchronized(this) {
           try {
@@ -621,6 +628,13 @@ public class LruBlockCache implements Bl
       this.go = false;
       interrupt();
     }
+
+    /**
+     * Used for the test.
+     */
+    boolean isEnteringRun() {
+      return this.enteringRun;
+    }
   }
 
   /*

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java Mon Mar  4 11:24:50 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.client.co
 import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
 import org.apache.hadoop.hbase.security.TokenInfo;
 import org.apache.hadoop.hbase.security.KerberosInfo;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -286,4 +287,22 @@ public interface HMasterInterface extend
    */
   public ExecResult execCoprocessor(Exec call)
       throws IOException;
+
+  public long snapshot(final HSnapshotDescription snapshot)
+    throws IOException;
+
+  public List<HSnapshotDescription> getCompletedSnapshots()
+    throws IOException;
+
+  public void deleteSnapshot(final HSnapshotDescription snapshot)
+    throws IOException;
+
+  public boolean isSnapshotDone(final HSnapshotDescription snapshot)
+    throws IOException;
+
+  public void restoreSnapshot(final HSnapshotDescription request)
+    throws IOException;
+
+  public boolean isRestoreSnapshotDone(final HSnapshotDescription request)
+    throws IOException;
 }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Mon Mar  4 11:24:50 2013
@@ -44,6 +44,7 @@ import javax.management.ObjectName;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Chore;
 import org.apache.hadoop.hbase.ClusterStatus;
@@ -92,13 +93,18 @@ import org.apache.hadoop.hbase.master.ha
 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.replication.regionserver.Replication;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 import org.apache.hadoop.hbase.util.HasThread;
 import org.apache.hadoop.hbase.util.InfoServer;
@@ -225,6 +231,9 @@ Server {
   private long masterStartTime;
   private long masterActiveTime;
 
+  // monitor for snapshot of hbase tables
+  private SnapshotManager snapshotManager;
+
   /**
    * MX Bean for MasterInfo
    */
@@ -406,6 +415,7 @@ Server {
       if (this.serverManager != null) this.serverManager.stop();
       if (this.assignmentManager != null) this.assignmentManager.stop();
       if (this.fileSystemManager != null) this.fileSystemManager.stop();
+      if (this.snapshotManager != null) this.snapshotManager.stop("server shutting down.");
       this.zooKeeper.close();
     }
     LOG.info("HMaster main thread exiting");
@@ -467,6 +477,9 @@ Server {
         ", sessionid=0x" +
         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
         ", cluster-up flag was=" + wasUp);
+
+    // create the snapshot manager
+    this.snapshotManager = new SnapshotManager(this);
   }
 
   // Check if we should stop every second.
@@ -1989,4 +2002,125 @@ Server {
     String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
     return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
   }
+
+  /**
+   * Exposed for TESTING!
+   * @return the underlying snapshot manager
+   */
+  public SnapshotManager getSnapshotManagerForTesting() {
+    return this.snapshotManager;
+   }
+
+
+  /**
+   * Triggers an asynchronous attempt to take a snapshot.
+   * {@inheritDoc}
+   */
+  @Override
+  public long snapshot(final HSnapshotDescription request) throws IOException {
+    LOG.debug("Submitting snapshot request for:" +
+        SnapshotDescriptionUtils.toString(request.getProto()));
+    try {
+      this.snapshotManager.checkSnapshotSupport();
+    } catch (UnsupportedOperationException e) {
+      throw new IOException(e);
+    }
+
+    // get the snapshot information
+    SnapshotDescription snapshot = SnapshotDescriptionUtils.validate(request.getProto(),
+      this.conf);
+
+    snapshotManager.takeSnapshot(snapshot);
+
+    // send back the max amount of time the client should wait for the snapshot to complete
+    long waitTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(),
+      SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
+    return waitTime;
+  }
+
+  /**
+   * List the currently available/stored snapshots. Any in-progress snapshots are ignored
+   */
+  @Override
+  public List<HSnapshotDescription> getCompletedSnapshots() throws IOException {
+    List<HSnapshotDescription> availableSnapshots = new ArrayList<HSnapshotDescription>();
+    List<SnapshotDescription> snapshots = snapshotManager.getCompletedSnapshots();
+
+    // convert to writables
+    for (SnapshotDescription snapshot: snapshots) {
+      availableSnapshots.add(new HSnapshotDescription(snapshot));
+    }
+
+    return availableSnapshots;
+  }
+
+  /**
+   * Execute Delete Snapshot operation.
+   * @throws ServiceException wrapping SnapshotDoesNotExistException if specified snapshot did not
+   * exist.
+   */
+  @Override
+  public void deleteSnapshot(final HSnapshotDescription request) throws IOException {
+    try {
+      this.snapshotManager.checkSnapshotSupport();
+    } catch (UnsupportedOperationException e) {
+      throw new IOException(e);
+    }
+
+    snapshotManager.deleteSnapshot(request.getProto());
+  }
+
+  /**
+   * Checks if the specified snapshot is done.
+   * @return true if the snapshot is in file system ready to use,
+   * false if the snapshot is in the process of completing
+   * @throws ServiceException wrapping UnknownSnapshotException if invalid snapshot, or
+   * a wrapped HBaseSnapshotException with progress failure reason.
+   */
+  @Override
+  public boolean isSnapshotDone(final HSnapshotDescription request) throws IOException {
+    LOG.debug("Checking to see if snapshot from request:" +
+      SnapshotDescriptionUtils.toString(request.getProto()) + " is done");
+    return snapshotManager.isSnapshotDone(request.getProto());
+  }
+
+  /**
+   * Execute Restore/Clone snapshot operation.
+   *
+   * <p>If the specified table exists a "Restore" is executed, replacing the table
+   * schema and directory data with the content of the snapshot.
+   * The table must be disabled, or a UnsupportedOperationException will be thrown.
+   *
+   * <p>If the table doesn't exist a "Clone" is executed, a new table is created
+   * using the schema at the time of the snapshot, and the content of the snapshot.
+   *
+   * <p>The restore/clone operation does not require copying HFiles. Since HFiles
+   * are immutable the table can point to and use the same files as the original one.
+   */
+  @Override
+  public void restoreSnapshot(final HSnapshotDescription request) throws IOException {
+    try {
+      this.snapshotManager.checkSnapshotSupport();
+    } catch (UnsupportedOperationException e) {
+      throw new IOException(e);
+    }
+
+    snapshotManager.restoreSnapshot(request.getProto());
+  }
+
+  /**
+   * Returns the status of the requested snapshot restore/clone operation.
+   * This method is not exposed to the user, it is just used internally by HBaseAdmin
+   * to verify if the restore is completed.
+   *
+   * No exceptions are thrown if the restore is not running, the result will be "done".
+   *
+   * @return done <tt>true</tt> if the restore/clone operation is completed.
+   * @throws RestoreSnapshotExcepton if the operation failed.
+   */
+  @Override
+  public boolean isRestoreSnapshotDone(final HSnapshotDescription request) throws IOException {
+    return !snapshotManager.isRestoringTable(request.getProto());
+  }
 }
+

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java Mon Mar  4 11:24:50 2013
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.coprocessor.*;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 
 import java.io.IOException;
 
@@ -629,4 +630,146 @@ public class MasterCoprocessorHost
       }
     }
   }
+
+  public void preSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preCloneSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preCloneSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postCloneSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postCloneSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preRestoreSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preRestoreSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postRestoreSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postRestoreSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preDeleteSnapshot(ctx, snapshot);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
+    ObserverContext<MasterCoprocessorEnvironment> ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postDeleteSnapshot(ctx, snapshot);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
 }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Mon Mar  4 11:24:50 2013
@@ -76,6 +76,8 @@ public class MasterFileSystem {
   private final Path oldLogDir;
   // root hbase directory on the FS
   private final Path rootdir;
+  // hbase temp directory used for table construction and deletion
+  private final Path tempdir;
   // create the split log lock
   final Lock splitLogLock = new ReentrantLock();
   final boolean distributedLogSplitting;
@@ -94,6 +96,7 @@ public class MasterFileSystem {
     // default localfs.  Presumption is that rootdir is fully-qualified before
     // we get to here with appropriate fs scheme.
     this.rootdir = FSUtils.getRootDir(conf);
+    this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
     // Cover both bases, the old way of setting default fs and the new.
     // We're supposed to run on 0.20 and 0.21 anyways.
     this.fs = this.rootdir.getFileSystem(conf);
@@ -130,6 +133,9 @@ public class MasterFileSystem {
     // check if the root directory exists
     checkRootDir(this.rootdir, conf, this.fs);
 
+    // check if temp directory exists and clean it
+    checkTempDir(this.tempdir, conf, this.fs);
+
     Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
 
     // Make sure the region servers can archive their old logs
@@ -178,6 +184,13 @@ public class MasterFileSystem {
   }
 
   /**
+   * @return HBase temp dir.
+   */
+  public Path getTempDir() {
+    return this.tempdir;
+  }
+
+  /**
    * @return The unique identifier generated for this cluster
    */
   public String getClusterId() {
@@ -385,6 +398,32 @@ public class MasterFileSystem {
     }
   }
 
+  /**
+   * Make sure the hbase temp directory exists and is empty.
+   * NOTE that this method is only executed once just after the master becomes the active one.
+   */
+  private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
+      throws IOException {
+    // If the temp directory exists, clear the content (left over, from the previous run)
+    if (fs.exists(tmpdir)) {
+      // Archive table in temp, maybe left over from failed deletion,
+      // if not the cleaner will take care of them.
+      for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
+        for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
+          HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
+        }
+      }
+      if (!fs.delete(tmpdir, true)) {
+        throw new IOException("Unable to clean the temp directory: " + tmpdir);
+      }
+    }
+
+    // Create the temp directory
+    if (!fs.mkdirs(tmpdir)) {
+      throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
+    }
+  }
+
   private static void bootstrap(final Path rd, final Configuration c)
   throws IOException {
     LOG.info("BOOTSTRAP: creating ROOT and first META regions");
@@ -451,6 +490,37 @@ public class MasterFileSystem {
     fs.delete(new Path(rootdir, Bytes.toString(tableName)), true);
   }
 
+  /**
+   * Move the specified file/directory to the hbase temp directory.
+   * @param path The path of the file/directory to move
+   * @return The temp location of the file/directory moved
+   * @throws IOException in case of file-system failure
+   */
+  public Path moveToTemp(final Path path) throws IOException {
+    Path tempPath = new Path(this.tempdir, path.getName());
+
+    // Ensure temp exists
+    if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) {
+      throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
+    }
+
+    if (!fs.rename(path, tempPath)) {
+      throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
+    }
+
+    return tempPath;
+  }
+
+  /**
+   * Move the specified table to the hbase temp directory
+   * @param tableName Table name to move
+   * @return The temp location of the table moved
+   * @throws IOException in case of file-system failure
+   */
+  public Path moveTableToTemp(byte[] tableName) throws IOException {
+    return moveToTemp(HTableDescriptor.getTableDir(this.rootdir, tableName));
+  }
+
   public void updateRegionInfo(HRegionInfo region) {
     // TODO implement this.  i think this is currently broken in trunk i don't
     //      see this getting updated.

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+/**
+ * Watch the current snapshot under process
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface SnapshotSentinel {
+
+  /**
+   * Check to see if the snapshot is finished, where finished may be success or failure.
+   * @return <tt>false</tt> if the snapshot is still in progress, <tt>true</tt> if the snapshot has
+   *         finished
+   */
+  public boolean isFinished();
+
+  /**
+   * Actively cancel a running snapshot.
+   * @param why Reason for cancellation.
+   */
+  public void cancel(String why);
+
+  /**
+   * @return the description of the snapshot being run
+   */
+  public SnapshotDescription getSnapshot();
+
+  /**
+   * Get the exception that caused the snapshot to fail, if the snapshot has failed.
+   * @return {@link ForeignException} that caused the snapshot to fail, or <tt>null</tt> if the
+   *  snapshot is still in progress or has succeeded
+   */
+  public ForeignException getExceptionIfFailed();
+
+}

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java Mon Mar  4 11:24:50 2013
@@ -82,7 +82,10 @@ public abstract class CleanerChore<T ext
     if (logCleaners != null) {
       for (String className : logCleaners) {
         T logCleaner = newFileCleaner(className, conf);
-        if (logCleaner != null) this.cleanersChain.add(logCleaner);
+        if (logCleaner != null) {
+          LOG.debug("initialize cleaner=" + className);
+          this.cleanersChain.add(logCleaner);
+        }
       }
     }
   }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java Mon Mar  4 11:24:50 2013
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 /**
  * This Chore, every time it runs, will clear the HFiles in the hfile archive
@@ -46,6 +47,9 @@ public class HFileCleaner extends Cleane
 
   @Override
   protected boolean validate(Path file) {
+    if (HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent())) {
+      return true;
+    }
     return StoreFile.validateStoreFileName(file.getName());
   }
 }



Mime
View raw message