accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [06/12] git commit: ACCUMULO-3213 Archive files that the garbage collector would otherwise delete.
Date Fri, 31 Oct 2014 21:27:05 GMT
ACCUMULO-3213 Archive files that the garbage collector would otherwise delete.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ce1a649
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ce1a649
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ce1a649

Branch: refs/heads/1.6
Commit: 7ce1a649fe22120833c9c0505c4a1a4c41986eca
Parents: 1890bea
Author: Josh Elser <elserj@apache.org>
Authored: Tue Oct 28 21:52:21 2014 -0400
Committer: Josh Elser <elserj@apache.org>
Committed: Fri Oct 31 14:59:47 2014 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/core/conf/Property.java |  1 +
 .../apache/accumulo/server/ServerConstants.java |  8 ++-
 .../accumulo/gc/SimpleGarbageCollector.java     | 75 ++++++++++++++++++--
 3 files changed, 78 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index ad83454..56f3d9c 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -289,6 +289,7 @@ public enum Property {
   GC_PORT("gc.port.client", "50091", PropertyType.PORT, "The listening port for the garbage
collector's monitor service"),
   GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT, "The number of threads
used to delete files"),
   GC_TRASH_IGNORE("gc.trash.ignore", "false", PropertyType.BOOLEAN, "Do not use the Trash,
even if it is configured"),
+  GC_FILE_ARCHIVE("gc.file.archive", "false", PropertyType.BOOLEAN, "Archive any files/directories
instead of moving to the HDFS trash or deleting"),
 
   // properties that are specific to the monitor server behavior
   MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX, "Properties in this category affect
the behavior of the monitor web server."),

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 880e2db..d4c0b32 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -112,6 +112,8 @@ public class ServerConstants {
   public static final String TABLE_DIR = "tables";
   public static final String RECOVERY_DIR = "recovery";
   public static final String WAL_DIR = "wal";
+  public static final String WALOG_ARCHIVE_DIR = "walogArchive";
+  public static final String FILE_ARCHIVE_DIR = "fileArchive";
 
   public static String[] getTablesDirs() {
     return VolumeConfiguration.prefix(getBaseUris(), TABLE_DIR);
@@ -126,7 +128,11 @@ public class ServerConstants {
   }
 
   public static String[] getWalogArchives() {
-    return VolumeConfiguration.prefix(getBaseUris(), "walogArchive");
+    return VolumeConfiguration.prefix(getBaseUris(), WALOG_ARCHIVE_DIR);
+  }
+
+  public static String[] getFileArchives() {
+    return VolumeConfiguration.prefix(getBaseUris(), FILE_ARCHIVE_DIR);
   }
 
   public static Path getInstanceIdLocation(Volume v) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 9b4af58..308d7b9 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -70,6 +70,7 @@ import org.apache.accumulo.core.util.ServerServices;
 import org.apache.accumulo.core.util.ServerServices.Service;
 import org.apache.accumulo.core.util.SslConnectionParams;
 import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.Volume;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
 import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
@@ -225,6 +226,15 @@ public class SimpleGarbageCollector implements Iface {
   }
 
   /**
+   * Should files be archived (as opposed to preserved in trash)
+   *
+   * @return True if files should be archived, false otherwise
+   */
+  boolean shouldArchiveFiles() {
+    return config.getBoolean(Property.GC_FILE_ARCHIVE);
+  }
+
+  /**
    * Initializes this garbage collector.
    *
    * @param fs
@@ -597,13 +607,68 @@ public class SimpleGarbageCollector implements Iface {
    *           if the volume manager encountered a problem
    */
   boolean moveToTrash(Path path) throws IOException {
-    if (!isUsingTrash())
-      return false;
-    try {
-      return fs.moveToTrash(path);
-    } catch (FileNotFoundException ex) {
+    if (shouldArchiveFiles()) {
+      return archiveFile(path);
+    } else {
+      if (!isUsingTrash())
+        return false;
+      try {
+        return fs.moveToTrash(path);
+      } catch (FileNotFoundException ex) {
+        return false;
+      }
+    }
+  }
+
+  /**
+   * Move a file, that would otherwise be deleted, to the archive directory for files
+   *
+   * @param fileToArchive
+   *          Path to file that is to be archived
+   * @return True if the file was successfully moved to the file archive directory, false
otherwise
+   */
+  boolean archiveFile(Path fileToArchive) throws IOException {
+    // Figure out what the base path this volume uses on this FileSystem
+    Volume sourceVolume = fs.getVolumeByPath(fileToArchive);
+    String sourceVolumeBasePath = sourceVolume.getBasePath();
+
+    log.debug("Base path for volume: " + sourceVolumeBasePath);
+
+    // Get the path for the file we want to archive
+    String sourcePathBasePath = fileToArchive.toUri().getPath();
+
+    // Strip off the common base path for the file to archive
+    String relativeVolumePath = sourcePathBasePath.substring(sourceVolumeBasePath.length());
+    if (Path.SEPARATOR_CHAR == relativeVolumePath.charAt(0)) {
+      if (relativeVolumePath.length() > 1) {
+        relativeVolumePath = relativeVolumePath.substring(1);
+      } else {
+        relativeVolumePath = "";
+      }
+    }
+
+    log.debug("Computed relative path for file to archive: " + relativeVolumePath);
+
+    // The file archive path on this volume (we can't archive this file to a different volume)
+    Path archivePath = new Path(sourceVolumeBasePath, ServerConstants.FILE_ARCHIVE_DIR);
+
+    log.debug("File archive path: " + archivePath);
+
+    fs.mkdirs(archivePath);
+
+    // Preserve the path beneath the Volume's base directory (e.g. tables/1/A_0000001.rf)
+    Path fileArchivePath = new Path(archivePath, relativeVolumePath);
+
+    log.debug("Create full path of " + fileArchivePath + " from " + archivePath + " and "
+ relativeVolumePath);
+
+    // Make sure that it doesn't already exist, something is wrong.
+    if (fs.exists(fileArchivePath)) {
+      log.warn("Tried to archive file, but it already exists: " + fileArchivePath);
       return false;
     }
+
+    log.debug("Moving " + fileToArchive + " to " + fileArchivePath);
+    return fs.rename(fileToArchive, fileArchivePath);
   }
 
   private void getZooLock(HostAndPort addr) throws KeeperException, InterruptedException
{


Mime
View raw message