Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 43CCE17913 for ; Fri, 31 Oct 2014 21:27:01 +0000 (UTC) Received: (qmail 49382 invoked by uid 500); 31 Oct 2014 21:27:01 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 49332 invoked by uid 500); 31 Oct 2014 21:27:01 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 49229 invoked by uid 99); 31 Oct 2014 21:27:01 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 31 Oct 2014 21:27:01 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id B996A92A213; Fri, 31 Oct 2014 21:27:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Fri, 31 Oct 2014 21:27:05 -0000 Message-Id: In-Reply-To: <9eef2352847744aa85ee6d4dc2d6d0db@git.apache.org> References: <9eef2352847744aa85ee6d4dc2d6d0db@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [06/12] git commit: ACCUMULO-3213 Archive files that the garbage collector would otherwise delete. ACCUMULO-3213 Archive files that the garbage collector would otherwise delete. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ce1a649 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ce1a649 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ce1a649 Branch: refs/heads/1.6 Commit: 7ce1a649fe22120833c9c0505c4a1a4c41986eca Parents: 1890bea Author: Josh Elser Authored: Tue Oct 28 21:52:21 2014 -0400 Committer: Josh Elser Committed: Fri Oct 31 14:59:47 2014 -0400 ---------------------------------------------------------------------- .../org/apache/accumulo/core/conf/Property.java | 1 + .../apache/accumulo/server/ServerConstants.java | 8 ++- .../accumulo/gc/SimpleGarbageCollector.java | 75 ++++++++++++++++++-- 3 files changed, 78 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/core/src/main/java/org/apache/accumulo/core/conf/Property.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index ad83454..56f3d9c 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -289,6 +289,7 @@ public enum Property { GC_PORT("gc.port.client", "50091", PropertyType.PORT, "The listening port for the garbage collector's monitor service"), GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT, "The number of threads used to delete files"), GC_TRASH_IGNORE("gc.trash.ignore", "false", PropertyType.BOOLEAN, "Do not use the Trash, even if it is configured"), + GC_FILE_ARCHIVE("gc.file.archive", "false", PropertyType.BOOLEAN, "Archive any files/directories instead of moving to the HDFS trash or deleting"), // properties that are specific to the monitor server behavior MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the monitor web server."), http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java index 880e2db..d4c0b32 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java @@ -112,6 +112,8 @@ public class ServerConstants { public static final String TABLE_DIR = "tables"; public static final String RECOVERY_DIR = "recovery"; public static final String WAL_DIR = "wal"; + public static final String WALOG_ARCHIVE_DIR = "walogArchive"; + public static final String FILE_ARCHIVE_DIR = "fileArchive"; public static String[] getTablesDirs() { return VolumeConfiguration.prefix(getBaseUris(), TABLE_DIR); @@ -126,7 +128,11 @@ public class ServerConstants { } public static String[] getWalogArchives() { - return VolumeConfiguration.prefix(getBaseUris(), "walogArchive"); + return VolumeConfiguration.prefix(getBaseUris(), WALOG_ARCHIVE_DIR); + } + + public static String[] getFileArchives() { + return VolumeConfiguration.prefix(getBaseUris(), FILE_ARCHIVE_DIR); } public static Path getInstanceIdLocation(Volume v) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ce1a649/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java ---------------------------------------------------------------------- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 9b4af58..308d7b9 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -70,6 +70,7 @@ import org.apache.accumulo.core.util.ServerServices; import org.apache.accumulo.core.util.ServerServices.Service; import org.apache.accumulo.core.util.SslConnectionParams; import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason; import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher; @@ -225,6 +226,15 @@ public class SimpleGarbageCollector implements Iface { } /** + * Should files be archived (as opposed to preserved in trash) + * + * @return True if files should be archived, false otherwise + */ + boolean shouldArchiveFiles() { + return config.getBoolean(Property.GC_FILE_ARCHIVE); + } + + /** * Initializes this garbage collector. * * @param fs @@ -597,13 +607,68 @@ public class SimpleGarbageCollector implements Iface { * if the volume manager encountered a problem */ boolean moveToTrash(Path path) throws IOException { - if (!isUsingTrash()) - return false; - try { - return fs.moveToTrash(path); - } catch (FileNotFoundException ex) { + if (shouldArchiveFiles()) { + return archiveFile(path); + } else { + if (!isUsingTrash()) + return false; + try { + return fs.moveToTrash(path); + } catch (FileNotFoundException ex) { + return false; + } + } + } + + /** + * Move a file, that would otherwise be deleted, to the archive directory for files + * + * @param fileToArchive + * Path to file that is to be archived + * @return True if the file was successfully moved to the file archive directory, false otherwise + */ + boolean archiveFile(Path fileToArchive) throws IOException { + // Figure out what the base path this volume uses on this FileSystem + Volume sourceVolume = fs.getVolumeByPath(fileToArchive); + String sourceVolumeBasePath = sourceVolume.getBasePath(); + + log.debug("Base path for volume: " + sourceVolumeBasePath); + + // Get the path for the file we want to archive + String sourcePathBasePath = fileToArchive.toUri().getPath(); + + // Strip off the common base path for the file to archive + String relativeVolumePath = sourcePathBasePath.substring(sourceVolumeBasePath.length()); + if (Path.SEPARATOR_CHAR == relativeVolumePath.charAt(0)) { + if (relativeVolumePath.length() > 1) { + relativeVolumePath = relativeVolumePath.substring(1); + } else { + relativeVolumePath = ""; + } + } + + log.debug("Computed relative path for file to archive: " + relativeVolumePath); + + // The file archive path on this volume (we can't archive this file to a different volume) + Path archivePath = new Path(sourceVolumeBasePath, ServerConstants.FILE_ARCHIVE_DIR); + + log.debug("File archive path: " + archivePath); + + fs.mkdirs(archivePath); + + // Preserve the path beneath the Volume's base directory (e.g. tables/1/A_0000001.rf) + Path fileArchivePath = new Path(archivePath, relativeVolumePath); + + log.debug("Create full path of " + fileArchivePath + " from " + archivePath + " and " + relativeVolumePath); + + // Make sure that it doesn't already exist, something is wrong. + if (fs.exists(fileArchivePath)) { + log.warn("Tried to archive file, but it already exists: " + fileArchivePath); return false; } + + log.debug("Moving " + fileToArchive + " to " + fileArchivePath); + return fs.rename(fileToArchive, fileArchivePath); } private void getZooLock(HostAndPort addr) throws KeeperException, InterruptedException {