accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [22/53] [abbrv] git commit: ACCUMULO-656, ACCUMULO-658 Update package names
Date Fri, 06 Sep 2013 18:22:50 GMT
ACCUMULO-656,ACCUMULO-658 Update package names


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c67460ee
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c67460ee
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c67460ee

Branch: refs/heads/ACCUMULO-210
Commit: c67460eeafa7deb888fc293c3ea8dd91d4373d45
Parents: fe95fc1
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Tue Jul 30 18:48:13 2013 -0400
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Fri Sep 6 14:20:11 2013 -0400

----------------------------------------------------------------------
 LICENSE                                         |    2 +-
 .../gc/GarbageCollectWriteAheadLogs.java        |  305 +
 .../accumulo/gc/SimpleGarbageCollector.java     |  768 ++
 .../server/gc/GarbageCollectWriteAheadLogs.java |  305 -
 .../server/gc/SimpleGarbageCollector.java       |  768 --
 .../apache/accumulo/gc/TestConfirmDeletes.java  |  139 +
 .../accumulo/server/gc/TestConfirmDeletes.java  |  139 -
 .../accumulo/monitor/DedupedLogEvent.java       |   68 +
 .../accumulo/monitor/EmbeddedWebServer.java     |   94 +
 .../org/apache/accumulo/monitor/LogService.java |  147 +
 .../org/apache/accumulo/monitor/Monitor.java    |  668 ++
 .../accumulo/monitor/ZooKeeperStatus.java       |  138 +
 .../accumulo/monitor/servlets/BasicServlet.java |  269 +
 .../monitor/servlets/DefaultServlet.java        |  374 +
 .../monitor/servlets/GcStatusServlet.java       |   72 +
 .../accumulo/monitor/servlets/JSONServlet.java  |  103 +
 .../accumulo/monitor/servlets/LogServlet.java   |  106 +
 .../monitor/servlets/MasterServlet.java         |  199 +
 .../monitor/servlets/OperationServlet.java      |  172 +
 .../monitor/servlets/PreciseNumberType.java     |   32 +
 .../monitor/servlets/ProblemServlet.java        |  193 +
 .../accumulo/monitor/servlets/ShellServlet.java |  331 +
 .../monitor/servlets/TServersServlet.java       |  368 +
 .../monitor/servlets/TablesServlet.java         |  189 +
 .../accumulo/monitor/servlets/VisServlet.java   |  236 +
 .../accumulo/monitor/servlets/XMLServlet.java   |  178 +
 .../accumulo/monitor/servlets/trace/Basic.java  |  111 +
 .../monitor/servlets/trace/ListType.java        |   76 +
 .../servlets/trace/NullKeyValueIterator.java    |   39 +
 .../monitor/servlets/trace/NullScanner.java     |  106 +
 .../monitor/servlets/trace/ShowTrace.java       |  159 +
 .../servlets/trace/ShowTraceLinkType.java       |   47 +
 .../monitor/servlets/trace/Summary.java         |  166 +
 .../org/apache/accumulo/monitor/util/Table.java |  233 +
 .../accumulo/monitor/util/TableColumn.java      |   48 +
 .../apache/accumulo/monitor/util/TableRow.java  |   68 +
 .../monitor/util/celltypes/CellType.java        |   35 +
 .../monitor/util/celltypes/CompactionsType.java |   72 +
 .../monitor/util/celltypes/DateTimeType.java    |   66 +
 .../monitor/util/celltypes/DurationType.java    |   51 +
 .../monitor/util/celltypes/NumberType.java      |  118 +
 .../monitor/util/celltypes/PercentageType.java  |   40 +
 .../util/celltypes/ProgressChartType.java       |   59 +
 .../monitor/util/celltypes/StringType.java      |   40 +
 .../monitor/util/celltypes/TServerLinkType.java |   55 +
 .../monitor/util/celltypes/TableLinkType.java   |   56 +
 .../monitor/util/celltypes/TableStateType.java  |   57 +
 .../server/monitor/DedupedLogEvent.java         |   68 -
 .../server/monitor/EmbeddedWebServer.java       |   94 -
 .../accumulo/server/monitor/LogService.java     |  146 -
 .../apache/accumulo/server/monitor/Monitor.java |  669 --
 .../server/monitor/ZooKeeperStatus.java         |  138 -
 .../server/monitor/servlets/BasicServlet.java   |  269 -
 .../server/monitor/servlets/DefaultServlet.java |  374 -
 .../monitor/servlets/GcStatusServlet.java       |   70 -
 .../server/monitor/servlets/JSONServlet.java    |  103 -
 .../server/monitor/servlets/LogServlet.java     |  105 -
 .../server/monitor/servlets/MasterServlet.java  |  199 -
 .../monitor/servlets/OperationServlet.java      |  172 -
 .../monitor/servlets/PreciseNumberType.java     |   33 -
 .../server/monitor/servlets/ProblemServlet.java |  191 -
 .../server/monitor/servlets/ShellServlet.java   |  330 -
 .../monitor/servlets/TServersServlet.java       |  368 -
 .../server/monitor/servlets/TablesServlet.java  |  189 -
 .../server/monitor/servlets/VisServlet.java     |  236 -
 .../server/monitor/servlets/XMLServlet.java     |  178 -
 .../server/monitor/servlets/trace/Basic.java    |  111 -
 .../server/monitor/servlets/trace/ListType.java |   77 -
 .../servlets/trace/NullKeyValueIterator.java    |   39 -
 .../monitor/servlets/trace/NullScanner.java     |  106 -
 .../monitor/servlets/trace/ShowTrace.java       |  160 -
 .../servlets/trace/ShowTraceLinkType.java       |   47 -
 .../server/monitor/servlets/trace/Summary.java  |  164 -
 .../accumulo/server/monitor/util/Table.java     |  233 -
 .../server/monitor/util/TableColumn.java        |   48 -
 .../accumulo/server/monitor/util/TableRow.java  |   68 -
 .../server/monitor/util/celltypes/CellType.java |   35 -
 .../monitor/util/celltypes/CompactionsType.java |   72 -
 .../monitor/util/celltypes/DateTimeType.java    |   66 -
 .../monitor/util/celltypes/DurationType.java    |   51 -
 .../monitor/util/celltypes/NumberType.java      |  118 -
 .../monitor/util/celltypes/PercentageType.java  |   40 -
 .../util/celltypes/ProgressChartType.java       |   59 -
 .../monitor/util/celltypes/StringType.java      |   40 -
 .../monitor/util/celltypes/TServerLinkType.java |   55 -
 .../monitor/util/celltypes/TableLinkType.java   |   56 -
 .../monitor/util/celltypes/TableStateType.java  |   57 -
 server/monitor/src/main/resources/web/down.gif  |  Bin 0 -> 164 bytes
 .../monitor/src/main/resources/web/favicon.png  |  Bin 0 -> 2195 bytes
 .../src/main/resources/web/flot/LICENSE.txt     |   22 +
 .../src/main/resources/web/flot/excanvas.js     | 1427 +++
 .../src/main/resources/web/flot/excanvas.min.js |    1 +
 .../resources/web/flot/jquery.colorhelpers.js   |  179 +
 .../web/flot/jquery.colorhelpers.min.js         |    1 +
 .../resources/web/flot/jquery.flot.crosshair.js |  167 +
 .../web/flot/jquery.flot.crosshair.min.js       |    1 +
 .../web/flot/jquery.flot.fillbetween.js         |  183 +
 .../web/flot/jquery.flot.fillbetween.min.js     |    1 +
 .../resources/web/flot/jquery.flot.image.js     |  238 +
 .../resources/web/flot/jquery.flot.image.min.js |    1 +
 .../src/main/resources/web/flot/jquery.flot.js  | 2599 ++++++
 .../main/resources/web/flot/jquery.flot.min.js  |    6 +
 .../resources/web/flot/jquery.flot.navigate.js  |  336 +
 .../web/flot/jquery.flot.navigate.min.js        |    1 +
 .../main/resources/web/flot/jquery.flot.pie.js  |  750 ++
 .../resources/web/flot/jquery.flot.pie.min.js   |    1 +
 .../resources/web/flot/jquery.flot.resize.js    |   60 +
 .../web/flot/jquery.flot.resize.min.js          |    1 +
 .../resources/web/flot/jquery.flot.selection.js |  344 +
 .../web/flot/jquery.flot.selection.min.js       |    1 +
 .../resources/web/flot/jquery.flot.stack.js     |  184 +
 .../resources/web/flot/jquery.flot.stack.min.js |    1 +
 .../resources/web/flot/jquery.flot.symbol.js    |   70 +
 .../web/flot/jquery.flot.symbol.min.js          |    1 +
 .../resources/web/flot/jquery.flot.threshold.js |  103 +
 .../web/flot/jquery.flot.threshold.min.js       |    1 +
 .../src/main/resources/web/flot/jquery.js       | 8316 ++++++++++++++++++
 .../src/main/resources/web/flot/jquery.min.js   |   23 +
 .../monitor/src/main/resources/web/functions.js |   21 +
 .../monitor/src/main/resources/web/screen.css   |  388 +
 server/monitor/src/main/resources/web/up.gif    |  Bin 0 -> 165 bytes
 server/monitor/src/main/resources/web/vis.js    |  409 +
 .../accumulo/monitor/ShowTraceLinkTypeTest.java |   64 +
 .../accumulo/monitor/ZooKeeperStatusTest.java   |   56 +
 .../apache/accumulo/server/master/Master.java   |    2 +-
 server/server/src/main/resources/web/down.gif   |  Bin 164 -> 0 bytes
 .../server/src/main/resources/web/favicon.png   |  Bin 2195 -> 0 bytes
 .../src/main/resources/web/flot/LICENSE.txt     |   22 -
 .../src/main/resources/web/flot/excanvas.js     | 1427 ---
 .../src/main/resources/web/flot/excanvas.min.js |    1 -
 .../resources/web/flot/jquery.colorhelpers.js   |  179 -
 .../web/flot/jquery.colorhelpers.min.js         |    1 -
 .../resources/web/flot/jquery.flot.crosshair.js |  167 -
 .../web/flot/jquery.flot.crosshair.min.js       |    1 -
 .../web/flot/jquery.flot.fillbetween.js         |  183 -
 .../web/flot/jquery.flot.fillbetween.min.js     |    1 -
 .../resources/web/flot/jquery.flot.image.js     |  238 -
 .../resources/web/flot/jquery.flot.image.min.js |    1 -
 .../src/main/resources/web/flot/jquery.flot.js  | 2599 ------
 .../main/resources/web/flot/jquery.flot.min.js  |    6 -
 .../resources/web/flot/jquery.flot.navigate.js  |  336 -
 .../web/flot/jquery.flot.navigate.min.js        |    1 -
 .../main/resources/web/flot/jquery.flot.pie.js  |  750 --
 .../resources/web/flot/jquery.flot.pie.min.js   |    1 -
 .../resources/web/flot/jquery.flot.resize.js    |   60 -
 .../web/flot/jquery.flot.resize.min.js          |    1 -
 .../resources/web/flot/jquery.flot.selection.js |  344 -
 .../web/flot/jquery.flot.selection.min.js       |    1 -
 .../resources/web/flot/jquery.flot.stack.js     |  184 -
 .../resources/web/flot/jquery.flot.stack.min.js |    1 -
 .../resources/web/flot/jquery.flot.symbol.js    |   70 -
 .../web/flot/jquery.flot.symbol.min.js          |    1 -
 .../resources/web/flot/jquery.flot.threshold.js |  103 -
 .../web/flot/jquery.flot.threshold.min.js       |    1 -
 .../src/main/resources/web/flot/jquery.js       | 8316 ------------------
 .../src/main/resources/web/flot/jquery.min.js   |   23 -
 .../server/src/main/resources/web/functions.js  |   21 -
 server/server/src/main/resources/web/screen.css |  388 -
 server/server/src/main/resources/web/up.gif     |  Bin 165 -> 0 bytes
 server/server/src/main/resources/web/vis.js     |  409 -
 .../server/monitor/ShowTraceLinkTypeTest.java   |   62 -
 .../server/monitor/ZooKeeperStatusTest.java     |   56 -
 .../java/org/apache/accumulo/start/Main.java    |    8 +-
 .../apache/accumulo/test/GetMasterStats.java    |    2 +-
 .../continuous/ContinuousStatsCollector.java    |    2 +-
 .../test/functional/GarbageCollectorIT.java     |    2 +-
 .../accumulo/test/functional/ReadWriteIT.java   |    2 +-
 167 files changed, 22819 insertions(+), 22810 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/c67460ee/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 8759447..7af5fb8 100644
--- a/LICENSE
+++ b/LICENSE
@@ -216,7 +216,7 @@ a "3-clause BSD" license. For details, see
 
 This product bundles JQuery and Flot, which are each available under
 The MIT License (MIT). For details, see
-  server/src/main/resources/web/flot/
+  server/monitor/src/main/resources/web/flot/
 
 The binary distribution of this product bundles jline, which is available
 under the following "3-clause BSD" license:

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c67460ee/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
new file mode 100644
index 0000000..92021fa
--- /dev/null
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.gc;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.gc.thrift.GCStatus;
+import org.apache.accumulo.core.gc.thrift.GcCycleStats;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.zookeeper.KeeperException;
+
+public class GarbageCollectWriteAheadLogs {
+  private static final Logger log = Logger.getLogger(GarbageCollectWriteAheadLogs.class);
+  
+  private final Instance instance;
+  private final VolumeManager fs;
+  
+  private boolean useTrash;
+  
+  GarbageCollectWriteAheadLogs(Instance instance, VolumeManager fs, boolean useTrash) throws IOException {
+    this.instance = instance;
+    this.fs = fs;
+  }
+  
+  public void collect(GCStatus status) {
+    
+    Span span = Trace.start("scanServers");
+    try {
+      
+      Set<Path> sortedWALogs = getSortedWALogs();
+      
+      status.currentLog.started = System.currentTimeMillis();
+      
+      Map<Path,String> fileToServerMap = new HashMap<Path,String>();
+      int count = scanServers(fileToServerMap);
+      long fileScanStop = System.currentTimeMillis();
+      log.info(String.format("Fetched %d files from %d servers in %.2f seconds", fileToServerMap.size(), count,
+          (fileScanStop - status.currentLog.started) / 1000.));
+      status.currentLog.candidates = fileToServerMap.size();
+      span.stop();
+      
+      span = Trace.start("removeMetadataEntries");
+      try {
+        count = removeMetadataEntries(fileToServerMap, sortedWALogs, status);
+      } catch (Exception ex) {
+        log.error("Unable to scan metadata table", ex);
+        return;
+      } finally {
+        span.stop();
+      }
+      
+      long logEntryScanStop = System.currentTimeMillis();
+      log.info(String.format("%d log entries scanned in %.2f seconds", count, (logEntryScanStop - fileScanStop) / 1000.));
+      
+      span = Trace.start("removeFiles");
+      Map<String,ArrayList<Path>> serverToFileMap = mapServersToFiles(fileToServerMap);
+      
+      count = removeFiles(serverToFileMap, sortedWALogs, status);
+      
+      long removeStop = System.currentTimeMillis();
+      log.info(String.format("%d total logs removed from %d servers in %.2f seconds", count, serverToFileMap.size(), (removeStop - logEntryScanStop) / 1000.));
+      status.currentLog.finished = removeStop;
+      status.lastLog = status.currentLog;
+      status.currentLog = new GcCycleStats();
+      span.stop();
+      
+    } catch (Exception e) {
+      log.error("exception occured while garbage collecting write ahead logs", e);
+    } finally {
+      span.stop();
+    }
+  }
+  
+  boolean holdsLock(InetSocketAddress addr) {
+    try {
+      String zpath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + org.apache.accumulo.core.util.AddressUtil.toString(addr);
+      List<String> children = ZooReaderWriter.getInstance().getChildren(zpath);
+      return !(children == null || children.isEmpty());
+    } catch (KeeperException.NoNodeException ex) {
+      return false;
+    } catch (Exception ex) {
+      log.debug(ex, ex);
+      return true;
+    }
+  }
+  
+  private int removeFiles(Map<String,ArrayList<Path>> serverToFileMap, Set<Path> sortedWALogs, final GCStatus status) {
+    AccumuloConfiguration conf = instance.getConfiguration();
+    for (Entry<String,ArrayList<Path>> entry : serverToFileMap.entrySet()) {
+      if (entry.getKey().isEmpty()) {
+        // old-style log entry, just remove it
+        for (Path path : entry.getValue()) {
+          log.debug("Removing old-style WAL " + path);
+          try {
+            if (!useTrash || !fs.moveToTrash(path))
+              fs.deleteRecursively(path);
+            status.currentLog.deleted++;
+          } catch (FileNotFoundException ex) {
+            // ignored
+          } catch (IOException ex) {
+            log.error("Unable to delete wal " + path + ": " + ex);
+          }
+        }
+      } else {
+        InetSocketAddress address = AddressUtil.parseAddress(entry.getKey());
+        if (!holdsLock(address)) {
+          for (Path path : entry.getValue()) {
+            log.debug("Removing WAL for offline server " + path);
+            try {
+              if (!useTrash || !fs.moveToTrash(path))
+                fs.deleteRecursively(path);
+              status.currentLog.deleted++;
+            } catch (FileNotFoundException ex) {
+              // ignored
+            } catch (IOException ex) {
+              log.error("Unable to delete wal " + path + ": " + ex);
+            }
+          }
+          continue;
+        } else {
+          Client tserver = null;
+          try {
+            tserver = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+            tserver.removeLogs(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), paths2strings(entry.getValue()));
+            log.debug("deleted " + entry.getValue() + " from " + entry.getKey());
+            status.currentLog.deleted += entry.getValue().size();
+          } catch (TException e) {
+            log.warn("Error talking to " + address + ": " + e);
+          } finally {
+            if (tserver != null)
+              ThriftUtil.returnClient(tserver);
+          }
+        }
+      }
+    }
+    
+    for (Path swalog : sortedWALogs) {
+      log.debug("Removing sorted WAL " + swalog);
+      try {
+        if (!useTrash || !fs.moveToTrash(swalog)) {
+          fs.deleteRecursively(swalog);
+        }
+      } catch (FileNotFoundException ex) {
+        // ignored
+      } catch (IOException ioe) {
+        try {
+          if (fs.exists(swalog)) {
+            log.error("Unable to delete sorted walog " + swalog + ": " + ioe);
+          }
+        } catch (IOException ex) {
+          log.error("Unable to check for the existence of " + swalog, ex);
+        }
+      }
+    }
+    
+    return 0;
+  }
+  
+  private List<String> paths2strings(ArrayList<Path> paths) {
+    List<String> result = new ArrayList<String>(paths.size());
+    for (Path path : paths)
+      result.add(path.toString());
+    return result;
+  }
+  
+  private static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap) {
+    Map<String,ArrayList<Path>> result = new HashMap<String,ArrayList<Path>>();
+    for (Entry<Path,String> fileServer : fileToServerMap.entrySet()) {
+      ArrayList<Path> files = result.get(fileServer.getValue());
+      if (files == null) {
+        files = new ArrayList<Path>();
+        result.put(fileServer.getValue(), files);
+      }
+      files.add(fileServer.getKey());
+    }
+    return result;
+  }
+  
+  private static int removeMetadataEntries(Map<Path,String> fileToServerMap, Set<Path> sortedWALogs, GCStatus status) throws IOException, KeeperException,
+      InterruptedException {
+    int count = 0;
+    Iterator<MetadataTableUtil.LogEntry> iterator = MetadataTableUtil.getLogEntries(SystemCredentials.get());
+    while (iterator.hasNext()) {
+      for (String entry : iterator.next().logSet) {
+        String parts[] = entry.split("/", 2);
+        String filename = parts[1];
+        Path path;
+        if (filename.contains(":"))
+          path = new Path(filename);
+        else
+          path = new Path(ServerConstants.getWalDirs()[0] + filename);
+        
+        if (fileToServerMap.remove(path) != null)
+          status.currentLog.inUse++;
+        
+        sortedWALogs.remove(path);
+        
+        count++;
+      }
+    }
+    return count;
+  }
+  
+  private int scanServers(Map<Path,String> fileToServerMap) throws Exception {
+    Set<String> servers = new HashSet<String>();
+    for (String walDir : ServerConstants.getWalDirs()) {
+      Path walRoot = new Path(walDir);
+      FileStatus[] listing = fs.listStatus(walRoot);
+      if (listing == null)
+        continue;
+      for (FileStatus status : listing) {
+        String server = status.getPath().getName();
+        servers.add(server);
+        if (status.isDir()) {
+          for (FileStatus file : fs.listStatus(new Path(walRoot, server))) {
+            if (isUUID(file.getPath().getName()))
+              fileToServerMap.put(file.getPath(), server);
+            else {
+              log.info("Ignoring file " + file.getPath() + " because it doesn't look like a uuid");
+            }
+          }
+        } else if (isUUID(server)) {
+          // old-style WAL are not under a directory
+          fileToServerMap.put(status.getPath(), "");
+        } else {
+          log.info("Ignoring file " + status.getPath() + " because it doesn't look like a uuid");
+        }
+      }
+    }
+    return servers.size();
+  }
+  
+  private Set<Path> getSortedWALogs() throws IOException {
+    Set<Path> result = new HashSet<Path>();
+    
+    for (String dir : ServerConstants.getRecoveryDirs()) {
+      Path recoveryDir = new Path(dir);
+      
+      if (fs.exists(recoveryDir)) {
+        for (FileStatus status : fs.listStatus(recoveryDir)) {
+          if (isUUID(status.getPath().getName())) {
+            result.add(status.getPath());
+          } else {
+            log.debug("Ignoring file " + status.getPath() + " because it doesn't look like a uuid");
+          }
+        }
+      }
+    }
+    return result;
+  }
+  
+  static private boolean isUUID(String name) {
+    try {
+      UUID.fromString(name);
+      return true;
+    } catch (IllegalArgumentException ex) {
+      return false;
+    }
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c67460ee/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
new file mode 100644
index 0000000..c096da4
--- /dev/null
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -0,0 +1,768 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.gc;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.gc.thrift.GCMonitorService.Iface;
+import org.apache.accumulo.core.gc.thrift.GCMonitorService.Processor;
+import org.apache.accumulo.core.gc.thrift.GCStatus;
+import org.apache.accumulo.core.gc.thrift.GcCycleStats;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TabletIterator;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.trace.instrument.CountSampler;
+import org.apache.accumulo.trace.instrument.Sampler;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+import com.beust.jcommander.Parameter;
+
+public class SimpleGarbageCollector implements Iface {
+  private static final Text EMPTY_TEXT = new Text();
+  
+  static class Opts extends ServerOpts {
+    @Parameter(names = {"-v", "--verbose"}, description = "extra information will get printed to stdout also")
+    boolean verbose = false;
+    @Parameter(names = {"-s", "--safemode"}, description = "safe mode will not delete files")
+    boolean safeMode = false;
+    @Parameter(names = {"-o", "--offline"},
+        description = "offline mode will run once and check data files directly; this is dangerous if accumulo is running or not shut down properly")
+    boolean offline = false;
+  }
+  
+  // how much of the JVM's available memory should it use gathering candidates
+  private static final float CANDIDATE_MEMORY_PERCENTAGE = 0.75f;
+  private boolean candidateMemExceeded;
+  
+  private static final Logger log = Logger.getLogger(SimpleGarbageCollector.class);
+  
+  private Credentials credentials;
+  private long gcStartDelay;
+  private boolean checkForBulkProcessingFiles;
+  private VolumeManager fs;
+  private boolean useTrash = true;
+  private Opts opts = new Opts();
+  private ZooLock lock;
+  private Key continueKey = null;
+  
+  private GCStatus status = new GCStatus(new GcCycleStats(), new GcCycleStats(), new GcCycleStats(), new GcCycleStats());
+  
+  private int numDeleteThreads;
+  
+  private Instance instance;
+  
+  public static void main(String[] args) throws UnknownHostException, IOException {
+    SecurityUtil.serverLogin();
+    Instance instance = HdfsZooInstance.getInstance();
+    ServerConfiguration serverConf = new ServerConfiguration(instance);
+    final VolumeManager fs = VolumeManagerImpl.get();
+    Accumulo.init(fs, serverConf, "gc");
+    Opts opts = new Opts();
+    opts.parseArgs("gc", args);
+    SimpleGarbageCollector gc = new SimpleGarbageCollector(opts);
+    
+    gc.init(fs, instance, SystemCredentials.get(), serverConf.getConfiguration().getBoolean(Property.GC_TRASH_IGNORE));
+    Accumulo.enableTracing(opts.getAddress(), "gc");
+    gc.run();
+  }
+  
+  public SimpleGarbageCollector(Opts opts) {
+    this.opts = opts;
+  }
+  
+  public void init(VolumeManager fs, Instance instance, Credentials credentials, boolean noTrash) throws IOException {
+    this.fs = fs;
+    this.credentials = credentials;
+    this.instance = instance;
+    
+    gcStartDelay = instance.getConfiguration().getTimeInMillis(Property.GC_CYCLE_START);
+    long gcDelay = instance.getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
+    numDeleteThreads = instance.getConfiguration().getCount(Property.GC_DELETE_THREADS);
+    log.info("start delay: " + (opts.offline ? 0 + " sec (offline)" : gcStartDelay + " milliseconds"));
+    log.info("time delay: " + gcDelay + " milliseconds");
+    log.info("safemode: " + opts.safeMode);
+    log.info("offline: " + opts.offline);
+    log.info("verbose: " + opts.verbose);
+    log.info("memory threshold: " + CANDIDATE_MEMORY_PERCENTAGE + " of " + Runtime.getRuntime().maxMemory() + " bytes");
+    log.info("delete threads: " + numDeleteThreads);
+    useTrash = !noTrash;
+  }
+  
+  private void run() {
+    long tStart, tStop;
+    
+    // Sleep for an initial period, giving the master time to start up and
+    // old data files to be unused
+    if (!opts.offline) {
+      try {
+        getZooLock(startStatsService());
+      } catch (Exception ex) {
+        log.error(ex, ex);
+        System.exit(1);
+      }
+      
+      try {
+        log.debug("Sleeping for " + gcStartDelay + " milliseconds before beginning garbage collection cycles");
+        Thread.sleep(gcStartDelay);
+      } catch (InterruptedException e) {
+        log.warn(e, e);
+        return;
+      }
+    }
+    
+    Sampler sampler = new CountSampler(100);
+    
+    while (true) {
+      if (sampler.next())
+        Trace.on("gc");
+      
+      Span gcSpan = Trace.start("loop");
+      tStart = System.currentTimeMillis();
+      try {
+        // STEP 1: gather candidates
+        System.gc(); // make room
+        candidateMemExceeded = false;
+        checkForBulkProcessingFiles = false;
+        
+        Span candidatesSpan = Trace.start("getCandidates");
+        status.current.started = System.currentTimeMillis();
+        SortedSet<String> candidates;
+        try {
+          candidates = getCandidates();
+          status.current.candidates = candidates.size();
+        } finally {
+          candidatesSpan.stop();
+        }
+        
+        // STEP 2: confirm deletes
+        // WARNING: This line is EXTREMELY IMPORTANT.
+        // You MUST confirm candidates are okay to delete
+        Span confirmDeletesSpan = Trace.start("confirmDeletes");
+        try {
+          confirmDeletes(candidates);
+        status.current.inUse = status.current.candidates - candidates.size();
+        } finally {
+          confirmDeletesSpan.stop();
+        }
+        
+        // STEP 3: delete files
+        if (opts.safeMode) {
+          if (opts.verbose)
+            System.out.println("SAFEMODE: There are " + candidates.size() + " data file candidates marked for deletion.%n"
+                + "          Examine the log files to identify them.%n" + "          They can be removed by executing: bin/accumulo gc --offline%n"
+                + "WARNING:  Do not run the garbage collector in offline mode unless you are positive%n"
+                + "          that the accumulo METADATA table is in a clean state, or that accumulo%n"
+                + "          has not yet been run, in the case of an upgrade.");
+          log.info("SAFEMODE: Listing all data file candidates for deletion");
+          for (String s : candidates)
+            log.info("SAFEMODE: " + s);
+          log.info("SAFEMODE: End candidates for deletion");
+        } else {
+          Span deleteSpan = Trace.start("deleteFiles");
+          try {
+            deleteFiles(candidates);
+            log.info("Number of data file candidates for deletion: " + status.current.candidates);
+            log.info("Number of data file candidates still in use: " + status.current.inUse);
+            log.info("Number of successfully deleted data files: " + status.current.deleted);
+            log.info("Number of data files delete failures: " + status.current.errors);
+          } finally {
+            deleteSpan.stop();
+          }
+          
+          // delete empty dirs of deleted tables
+          // this can occur as a result of cloning
+          cleanUpDeletedTableDirs(candidates);
+        }
+        
+        status.current.finished = System.currentTimeMillis();
+        status.last = status.current;
+        status.current = new GcCycleStats();
+        
+      } catch (Exception e) {
+        log.error(e, e);
+      }
+      tStop = System.currentTimeMillis();
+      log.info(String.format("Collect cycle took %.2f seconds", ((tStop - tStart) / 1000.0)));
+      
+      if (opts.offline)
+        break;
+      
+      if (candidateMemExceeded) {
+        log.info("Gathering of candidates was interrupted due to memory shortage. Bypassing cycle delay to collect the remaining candidates.");
+        continue;
+      }
+      
+      // Clean up any unused write-ahead logs
+      Span waLogs = Trace.start("walogs");
+      try {
+        GarbageCollectWriteAheadLogs walogCollector = new GarbageCollectWriteAheadLogs(instance, fs, useTrash);
+        log.info("Beginning garbage collection of write-ahead logs");
+        walogCollector.collect(status);
+      } catch (Exception e) {
+        log.error(e, e);
+      } finally {
+        waLogs.stop();
+      }
+      gcSpan.stop();
+      
+      // we just made a lot of changes to the !METADATA table: flush them out
+      try {
+        Connector connector = instance.getConnector(credentials.getPrincipal(), credentials.getToken());
+        connector.tableOperations().compact(MetadataTable.NAME, null, null, true, true);
+        connector.tableOperations().compact(RootTable.NAME, null, null, true, true);
+      } catch (Exception e) {
+        log.warn(e, e);
+      }
+      
+      Trace.offNoFlush();
+      try {
+        long gcDelay = instance.getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
+        log.debug("Sleeping for " + gcDelay + " milliseconds");
+        Thread.sleep(gcDelay);
+      } catch (InterruptedException e) {
+        log.warn(e, e);
+        return;
+      }
+    }
+  }
+  
+  private boolean moveToTrash(Path path) throws IOException {
+    if (!useTrash)
+      return false;
+    try {
+      return fs.moveToTrash(path);
+    } catch (FileNotFoundException ex) {
+      return false;
+    }
+  }
+  
+  /*
+   * this method removes deleted table dirs that are empty
+   */
+  private void cleanUpDeletedTableDirs(SortedSet<String> candidates) throws Exception {
+    
+    HashSet<String> tableIdsWithDeletes = new HashSet<String>();
+    
+    // find the table ids that had dirs deleted
+    for (String delete : candidates) {
+      if (isDir(delete)) {
+        String tableId = delete.split("/")[1];
+        tableIdsWithDeletes.add(tableId);
+      }
+    }
+    
+    Tables.clearCache(instance);
+    Set<String> tableIdsInZookeeper = Tables.getIdToNameMap(instance).keySet();
+    
+    tableIdsWithDeletes.removeAll(tableIdsInZookeeper);
+    
+    // tableIdsWithDeletes should now contain the set of deleted tables that had dirs deleted
+    
+    for (String delTableId : tableIdsWithDeletes) {
+      // if dir exist and is empty, then empty list is returned...
+      // hadoop 1.0 will return null if the file doesn't exist
+      // hadoop 2.0 will throw an exception if the file does not exist
+      for (String dir : ServerConstants.getTablesDirs()) {
+        FileStatus[] tabletDirs = null;
+        try {
+          tabletDirs = fs.listStatus(new Path(dir + "/" + delTableId));
+        } catch (FileNotFoundException ex) {
+          // ignored
+        }
+        if (tabletDirs == null)
+          continue;
+        
+        if (tabletDirs.length == 0) {
+          Path p = new Path(dir + "/" + delTableId);
+          if (!moveToTrash(p))
+            fs.delete(p);
+        }
+      }
+    }
+  }
+  
+  private void getZooLock(InetSocketAddress addr) throws KeeperException, InterruptedException {
+    String address = addr.getHostName() + ":" + addr.getPort();
+    String path = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZGC_LOCK;
+    
+    LockWatcher lockWatcher = new LockWatcher() {
+      @Override
+      public void lostLock(LockLossReason reason) {
+        Halt.halt("GC lock in zookeeper lost (reason = " + reason + "), exiting!");
+      }
+      
+      @Override
+      public void unableToMonitorLockNode(final Throwable e) {
+        Halt.halt(-1, new Runnable() {
+          
+          @Override
+          public void run() {
+            log.fatal("No longer able to monitor lock node ", e);
+          }
+        });
+        
+      }
+    };
+    
+    while (true) {
+      lock = new ZooLock(path);
+      if (lock.tryLock(lockWatcher, new ServerServices(address, Service.GC_CLIENT).toString().getBytes())) {
+        break;
+      }
+      UtilWaitThread.sleep(1000);
+    }
+  }
+  
+  private InetSocketAddress startStatsService() throws UnknownHostException {
+    Processor<Iface> processor = new Processor<Iface>(TraceWrap.service(this));
+    int port = instance.getConfiguration().getPort(Property.GC_PORT);
+    long maxMessageSize = instance.getConfiguration().getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE);
+    InetSocketAddress result = new InetSocketAddress(opts.getAddress(), port);
+    try {
+      TServerUtils.startTServer(result, processor, this.getClass().getSimpleName(), "GC Monitor Service", 2, 1000, maxMessageSize);
+    } catch (Exception ex) {
+      log.fatal(ex, ex);
+      throw new RuntimeException(ex);
+    }
+    return result;
+  }
+  
+  /**
+   * This method gets a set of candidates for deletion by scanning the METADATA table deleted flag keyspace
+   */
+  SortedSet<String> getCandidates() throws Exception {
+    TreeSet<String> candidates = new TreeSet<String>();
+    
+    if (opts.offline) {
+      checkForBulkProcessingFiles = true;
+      try {
+        for (String validExtension : FileOperations.getValidExtensions()) {
+          for (String dir : ServerConstants.getTablesDirs()) {
+            for (FileStatus stat : fs.globStatus(new Path(dir + "/*/*/*." + validExtension))) {
+              String cand = stat.getPath().toUri().getPath();
+              if (cand.contains(ServerConstants.getRootTabletDir()))
+                continue;
+              candidates.add(cand.substring(dir.length()));
+              log.debug("Offline candidate: " + cand);
+            }
+          }
+        }
+      } catch (IOException e) {
+        log.error("Unable to check the filesystem for offline candidates. Removing all candidates for deletion to be safe.", e);
+        candidates.clear();
+      }
+      return candidates;
+    }
+    
+    checkForBulkProcessingFiles = false;
+    candidates.addAll(getBatch(RootTable.NAME));
+    if (candidateMemExceeded)
+      return candidates;
+    
+    candidates.addAll(getBatch(MetadataTable.NAME));
+    return candidates;
+  }
+  
+  /**
+   * Gets a batch of delete markers from the specified table
+   * 
+   * @param tableName
+   *          the name of the system table to scan (either {@link RootTable.NAME} or {@link MetadataTable.NAME})
+   */
+  private Collection<String> getBatch(String tableName) throws Exception {
+    // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them,
+    // then will never inspect deletes after N
+    Range range = MetadataSchema.DeletesSection.getRange();
+    if (continueKey != null) {
+      if (!range.contains(continueKey)) {
+        // continue key is for some other range
+        return Collections.emptyList();
+      }
+      range = new Range(continueKey, true, range.getEndKey(), range.isEndKeyInclusive());
+      continueKey = null;
+    }
+    
+    Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, Authorizations.EMPTY);
+    scanner.setRange(range);
+    List<String> result = new ArrayList<String>();
+    // find candidates for deletion; chop off the prefix
+    for (Entry<Key,Value> entry : scanner) {
+      String cand = entry.getKey().getRow().toString().substring(MetadataSchema.DeletesSection.getRowPrefix().length());
+      result.add(cand);
+      checkForBulkProcessingFiles |= cand.toLowerCase(Locale.ENGLISH).contains(Constants.BULK_PREFIX);
+      if (almostOutOfMemory()) {
+        candidateMemExceeded = true;
+        log.info("List of delete candidates has exceeded the memory threshold. Attempting to delete what has been gathered so far.");
+        continueKey = entry.getKey();
+        break;
+      }
+    }
+    
+    return result;
+  }
+  
+  static public boolean almostOutOfMemory() {
+    Runtime runtime = Runtime.getRuntime();
+    return runtime.totalMemory() - runtime.freeMemory() > CANDIDATE_MEMORY_PERCENTAGE * runtime.maxMemory();
+  }
+  
+  /**
+   * This method removes candidates from the candidate list under two conditions: 1. They are in the same folder as a bulk processing file, if that option is
+   * selected 2. They are still in use in the file column family in the METADATA table
+   */
+  public void confirmDeletes(SortedSet<String> candidates) throws AccumuloException {
+    confirmDeletes(RootTable.NAME, candidates);
+    confirmDeletes(MetadataTable.NAME, candidates);
+  }
+  
+  private void confirmDeletes(String tableName, SortedSet<String> candidates) throws AccumuloException {
+    Scanner scanner;
+    if (opts.offline) {
+      // TODO
+      throw new RuntimeException("Offline scanner no longer supported");
+      // try {
+      // scanner = new OfflineMetadataScanner(instance.getConfiguration(), fs);
+      // } catch (IOException e) {
+      // throw new IllegalStateException("Unable to create offline metadata scanner", e);
+      // }
+    } else {
+      try {
+        scanner = new IsolatedScanner(instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, Authorizations.EMPTY));
+      } catch (AccumuloSecurityException ex) {
+        throw new AccumuloException(ex);
+      } catch (TableNotFoundException ex) {
+        throw new AccumuloException(ex);
+      }
+    }
+    
+    // skip candidates that are in a bulk processing folder
+    if (checkForBulkProcessingFiles) {
+      
+      log.debug("Checking for bulk processing flags");
+      
+      scanner.setRange(MetadataSchema.BlipSection.getRange());
+      
+      // WARNING: This block is IMPORTANT
+      // You MUST REMOVE candidates that are in the same folder as a bulk
+      // processing flag!
+      
+      for (Entry<Key,Value> entry : scanner) {
+        String blipPath = entry.getKey().getRow().toString().substring(MetadataSchema.BlipSection.getRowPrefix().length());
+        Iterator<String> tailIter = candidates.tailSet(blipPath).iterator();
+        int count = 0;
+        while (tailIter.hasNext()) {
+          if (tailIter.next().startsWith(blipPath)) {
+            count++;
+            tailIter.remove();
+          } else {
+            break;
+          }
+        }
+        
+        if (count > 0)
+          log.debug("Folder has bulk processing flag: " + blipPath);
+        
+      }
+    }
+    
+    // skip candidates that are still in use in the file column family in
+    // the metadata table
+    scanner.clearColumns();
+    scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+    scanner.fetchColumnFamily(ScanFileColumnFamily.NAME);
+    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
+    TabletIterator tabletIterator = new TabletIterator(scanner, MetadataSchema.TabletsSection.getRange(), false, true);
+    
+    while (tabletIterator.hasNext()) {
+      Map<Key,Value> tabletKeyValues = tabletIterator.next();
+      
+      for (Entry<Key,Value> entry : tabletKeyValues.entrySet()) {
+        if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME) || entry.getKey().getColumnFamily().equals(ScanFileColumnFamily.NAME)) {
+          
+          String cf = entry.getKey().getColumnQualifier().toString();
+          String delete = cf;
+          if (!cf.contains(":")) {
+            if (cf.startsWith("../")) {
+              delete = cf.substring(2);
+            } else {
+              String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow()));
+              if (cf.startsWith("/"))
+                delete = "/" + table + cf;
+              else
+                delete = "/" + table + "/" + cf;
+            }
+          }
+          // WARNING: This line is EXTREMELY IMPORTANT.
+          // You MUST REMOVE candidates that are still in use
+          if (candidates.remove(delete))
+            log.debug("Candidate was still in use in the " + tableName + " table: " + delete);
+          
+          String path = delete.substring(0, delete.lastIndexOf('/'));
+          if (candidates.remove(path))
+            log.debug("Candidate was still in use in the " + tableName + " table: " + path);
+        } else if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(entry.getKey())) {
+          String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow()));
+          String delete = "/" + table + entry.getValue().toString();
+          if (candidates.remove(delete))
+            log.debug("Candidate was still in use in the " + tableName + " table: " + delete);
+        } else
+          throw new AccumuloException("Scanner over metadata table returned unexpected column : " + entry.getKey());
+      }
+    }
+  }
+  
+  final static String METADATA_TABLE_DIR = "/" + MetadataTable.ID;
+  
+  private static void putMarkerDeleteMutation(final String delete, final BatchWriter metadataWriter, final BatchWriter rootWriter)
+      throws MutationsRejectedException {
+    BatchWriter writer = delete.contains(METADATA_TABLE_DIR) ? rootWriter : metadataWriter;
+    Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + delete);
+    m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
+    writer.addMutation(m);
+  }
+  
+  /**
+   * This method attempts to do its best to remove files from the filesystem that have been confirmed for deletion.
+   */
+  private void deleteFiles(SortedSet<String> confirmedDeletes) {
+    // create a batchwriter to remove the delete flags for successful
+    // deletes; Need separate writer for the root tablet.
+    BatchWriter writer = null;
+    BatchWriter rootWriter = null;
+    if (!opts.offline) {
+      Connector c;
+      try {
+        c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+        writer = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+        rootWriter = c.createBatchWriter(RootTable.NAME, new BatchWriterConfig());
+      } catch (AccumuloException e) {
+        log.error("Unable to connect to Accumulo to write deletes", e);
+      } catch (AccumuloSecurityException e) {
+        log.error("Unable to connect to Accumulo to write deletes", e);
+      } catch (TableNotFoundException e) {
+        log.error("Unable to create writer to remove file from the " + e.getTableName() + " table", e);
+      }
+    }
+    // when deleting a dir and all files in that dir, only need to delete the dir
+    // the dir will sort right before the files... so remove the files in this case
+    // to minimize namenode ops
+    Iterator<String> cdIter = confirmedDeletes.iterator();
+    String lastDir = null;
+    while (cdIter.hasNext()) {
+      String delete = cdIter.next();
+      if (isDir(delete)) {
+        lastDir = delete;
+      } else if (lastDir != null) {
+        if (delete.startsWith(lastDir)) {
+          log.debug("Ignoring " + delete + " because " + lastDir + " exist");
+          try {
+            putMarkerDeleteMutation(delete, writer, rootWriter);
+          } catch (MutationsRejectedException e) {
+            throw new RuntimeException(e);
+          }
+          cdIter.remove();
+        } else {
+          lastDir = null;
+        }
+        
+      }
+    }
+    
+    final BatchWriter finalWriter = writer;
+    final BatchWriter finalRootWriter = rootWriter;
+    
+    ExecutorService deleteThreadPool = Executors.newFixedThreadPool(numDeleteThreads, new NamingThreadFactory("deleting"));
+    
+    for (final String delete : confirmedDeletes) {
+      
+      Runnable deleteTask = new Runnable() {
+        @Override
+        public void run() {
+          boolean removeFlag;
+          
+          try {
+            Path fullPath;
+            
+            if (delete.contains(":"))
+              fullPath = new Path(delete);
+            else
+              fullPath = fs.getFullPath(ServerConstants.getTablesDirs(), delete);
+            log.debug("Deleting " + fullPath);
+            
+            if (moveToTrash(fullPath) || fs.deleteRecursively(fullPath)) {
+              // delete succeeded, still want to delete
+              removeFlag = true;
+              synchronized (SimpleGarbageCollector.this) {
+                ++status.current.deleted;
+              }
+            } else if (fs.exists(fullPath)) {
+              // leave the entry in the METADATA table; we'll try again
+              // later
+              removeFlag = false;
+              synchronized (SimpleGarbageCollector.this) {
+                ++status.current.errors;
+              }
+              log.warn("File exists, but was not deleted for an unknown reason: " + fullPath);
+            } else {
+              // this failure, we still want to remove the METADATA table
+              // entry
+              removeFlag = true;
+              synchronized (SimpleGarbageCollector.this) {
+                ++status.current.errors;
+              }
+              String parts[] = delete.split("/");
+              if (parts.length > 2) {
+                String tableId = parts[parts.length - 3];
+                String tabletDir = parts[parts.length - 2];
+                TableManager.getInstance().updateTableStateCache(tableId);
+                TableState tableState = TableManager.getInstance().getTableState(tableId);
+                if (tableState != null && tableState != TableState.DELETING) {
+                  // clone directories don't always exist
+                  if (!tabletDir.startsWith("c-"))
+                    log.warn("File doesn't exist: " + fullPath);
+                }
+              } else {
+                log.warn("Very strange path name: " + delete);
+              }
+            }
+            
+            // proceed to clearing out the flags for successful deletes and
+            // non-existent files
+            if (removeFlag && finalWriter != null) {
+              putMarkerDeleteMutation(delete, finalWriter, finalRootWriter);
+            }
+          } catch (Exception e) {
+            log.error(e, e);
+          }
+          
+        }
+        
+      };
+      
+      deleteThreadPool.execute(deleteTask);
+    }
+    
+    deleteThreadPool.shutdown();
+    
+    try {
+      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) {}
+    } catch (InterruptedException e1) {
+      log.error(e1, e1);
+    }
+    
+    if (writer != null) {
+      try {
+        writer.close();
+      } catch (MutationsRejectedException e) {
+        log.error("Problem removing entries from the metadata table: ", e);
+      }
+    }
+    if (rootWriter != null) {
+      try {
+        rootWriter.close();
+      } catch (MutationsRejectedException e) {
+        log.error("Problem removing entries from the metadata table: ", e);
+      }
+    }
+  }
+  
+  private boolean isDir(String delete) {
+    int slashCount = 0;
+    for (int i = 0; i < delete.length(); i++)
+      if (delete.charAt(i) == '/')
+        slashCount++;
+    return slashCount == 2;
+  }
+  
+  @Override
+  public GCStatus getStatus(TInfo info, TCredentials credentials) {
+    return status;
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c67460ee/server/gc/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
deleted file mode 100644
index 867ba93..0000000
--- a/server/gc/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
+++ /dev/null
@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.gc;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.UUID;
-
-import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.gc.thrift.GCStatus;
-import org.apache.accumulo.core.gc.thrift.GcCycleStats;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
-import org.apache.accumulo.core.util.AddressUtil;
-import org.apache.accumulo.core.util.ThriftUtil;
-import org.apache.accumulo.core.zookeeper.ZooUtil;
-import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.security.SystemCredentials;
-import org.apache.accumulo.server.util.MetadataTableUtil;
-import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
-import org.apache.accumulo.trace.instrument.Span;
-import org.apache.accumulo.trace.instrument.Trace;
-import org.apache.accumulo.trace.instrument.Tracer;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Logger;
-import org.apache.thrift.TException;
-import org.apache.zookeeper.KeeperException;
-
-public class GarbageCollectWriteAheadLogs {
-  private static final Logger log = Logger.getLogger(GarbageCollectWriteAheadLogs.class);
-  
-  private final Instance instance;
-  private final VolumeManager fs;
-  
-  private boolean useTrash;
-  
-  GarbageCollectWriteAheadLogs(Instance instance, VolumeManager fs, boolean useTrash) throws IOException {
-    this.instance = instance;
-    this.fs = fs;
-  }
-  
-  public void collect(GCStatus status) {
-    
-    Span span = Trace.start("scanServers");
-    try {
-      
-      Set<Path> sortedWALogs = getSortedWALogs();
-      
-      status.currentLog.started = System.currentTimeMillis();
-      
-      Map<Path,String> fileToServerMap = new HashMap<Path,String>();
-      int count = scanServers(fileToServerMap);
-      long fileScanStop = System.currentTimeMillis();
-      log.info(String.format("Fetched %d files from %d servers in %.2f seconds", fileToServerMap.size(), count,
-          (fileScanStop - status.currentLog.started) / 1000.));
-      status.currentLog.candidates = fileToServerMap.size();
-      span.stop();
-      
-      span = Trace.start("removeMetadataEntries");
-      try {
-        count = removeMetadataEntries(fileToServerMap, sortedWALogs, status);
-      } catch (Exception ex) {
-        log.error("Unable to scan metadata table", ex);
-        return;
-      } finally {
-        span.stop();
-      }
-      
-      long logEntryScanStop = System.currentTimeMillis();
-      log.info(String.format("%d log entries scanned in %.2f seconds", count, (logEntryScanStop - fileScanStop) / 1000.));
-      
-      span = Trace.start("removeFiles");
-      Map<String,ArrayList<Path>> serverToFileMap = mapServersToFiles(fileToServerMap);
-      
-      count = removeFiles(serverToFileMap, sortedWALogs, status);
-      
-      long removeStop = System.currentTimeMillis();
-      log.info(String.format("%d total logs removed from %d servers in %.2f seconds", count, serverToFileMap.size(), (removeStop - logEntryScanStop) / 1000.));
-      status.currentLog.finished = removeStop;
-      status.lastLog = status.currentLog;
-      status.currentLog = new GcCycleStats();
-      span.stop();
-      
-    } catch (Exception e) {
-      log.error("exception occured while garbage collecting write ahead logs", e);
-    } finally {
-      span.stop();
-    }
-  }
-  
-  boolean holdsLock(InetSocketAddress addr) {
-    try {
-      String zpath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + org.apache.accumulo.core.util.AddressUtil.toString(addr);
-      List<String> children = ZooReaderWriter.getInstance().getChildren(zpath);
-      return !(children == null || children.isEmpty());
-    } catch (KeeperException.NoNodeException ex) {
-      return false;
-    } catch (Exception ex) {
-      log.debug(ex, ex);
-      return true;
-    }
-  }
-  
-  private int removeFiles(Map<String,ArrayList<Path>> serverToFileMap, Set<Path> sortedWALogs, final GCStatus status) {
-    AccumuloConfiguration conf = instance.getConfiguration();
-    for (Entry<String,ArrayList<Path>> entry : serverToFileMap.entrySet()) {
-      if (entry.getKey().isEmpty()) {
-        // old-style log entry, just remove it
-        for (Path path : entry.getValue()) {
-          log.debug("Removing old-style WAL " + path);
-          try {
-            if (!useTrash || !fs.moveToTrash(path))
-              fs.deleteRecursively(path);
-            status.currentLog.deleted++;
-          } catch (FileNotFoundException ex) {
-            // ignored
-          } catch (IOException ex) {
-            log.error("Unable to delete wal " + path + ": " + ex);
-          }
-        }
-      } else {
-        InetSocketAddress address = AddressUtil.parseAddress(entry.getKey());
-        if (!holdsLock(address)) {
-          for (Path path : entry.getValue()) {
-            log.debug("Removing WAL for offline server " + path);
-            try {
-              if (!useTrash || !fs.moveToTrash(path))
-                fs.deleteRecursively(path);
-              status.currentLog.deleted++;
-            } catch (FileNotFoundException ex) {
-              // ignored
-            } catch (IOException ex) {
-              log.error("Unable to delete wal " + path + ": " + ex);
-            }
-          }
-          continue;
-        } else {
-          Client tserver = null;
-          try {
-            tserver = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
-            tserver.removeLogs(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), paths2strings(entry.getValue()));
-            log.debug("deleted " + entry.getValue() + " from " + entry.getKey());
-            status.currentLog.deleted += entry.getValue().size();
-          } catch (TException e) {
-            log.warn("Error talking to " + address + ": " + e);
-          } finally {
-            if (tserver != null)
-              ThriftUtil.returnClient(tserver);
-          }
-        }
-      }
-    }
-    
-    for (Path swalog : sortedWALogs) {
-      log.debug("Removing sorted WAL " + swalog);
-      try {
-        if (!useTrash || !fs.moveToTrash(swalog)) {
-          fs.deleteRecursively(swalog);
-        }
-      } catch (FileNotFoundException ex) {
-        // ignored
-      } catch (IOException ioe) {
-        try {
-          if (fs.exists(swalog)) {
-            log.error("Unable to delete sorted walog " + swalog + ": " + ioe);
-          }
-        } catch (IOException ex) {
-          log.error("Unable to check for the existence of " + swalog, ex);
-        }
-      }
-    }
-    
-    return 0;
-  }
-  
-  private List<String> paths2strings(ArrayList<Path> paths) {
-    List<String> result = new ArrayList<String>(paths.size());
-    for (Path path : paths)
-      result.add(path.toString());
-    return result;
-  }
-  
-  private static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap) {
-    Map<String,ArrayList<Path>> result = new HashMap<String,ArrayList<Path>>();
-    for (Entry<Path,String> fileServer : fileToServerMap.entrySet()) {
-      ArrayList<Path> files = result.get(fileServer.getValue());
-      if (files == null) {
-        files = new ArrayList<Path>();
-        result.put(fileServer.getValue(), files);
-      }
-      files.add(fileServer.getKey());
-    }
-    return result;
-  }
-  
-  private static int removeMetadataEntries(Map<Path,String> fileToServerMap, Set<Path> sortedWALogs, GCStatus status) throws IOException, KeeperException,
-      InterruptedException {
-    int count = 0;
-    Iterator<MetadataTableUtil.LogEntry> iterator = MetadataTableUtil.getLogEntries(SystemCredentials.get());
-    while (iterator.hasNext()) {
-      for (String entry : iterator.next().logSet) {
-        String parts[] = entry.split("/", 2);
-        String filename = parts[1];
-        Path path;
-        if (filename.contains(":"))
-          path = new Path(filename);
-        else
-          path = new Path(ServerConstants.getWalDirs()[0] + filename);
-        
-        if (fileToServerMap.remove(path) != null)
-          status.currentLog.inUse++;
-        
-        sortedWALogs.remove(path);
-        
-        count++;
-      }
-    }
-    return count;
-  }
-  
-  private int scanServers(Map<Path,String> fileToServerMap) throws Exception {
-    Set<String> servers = new HashSet<String>();
-    for (String walDir : ServerConstants.getWalDirs()) {
-      Path walRoot = new Path(walDir);
-      FileStatus[] listing = fs.listStatus(walRoot);
-      if (listing == null)
-        continue;
-      for (FileStatus status : listing) {
-        String server = status.getPath().getName();
-        servers.add(server);
-        if (status.isDir()) {
-          for (FileStatus file : fs.listStatus(new Path(walRoot, server))) {
-            if (isUUID(file.getPath().getName()))
-              fileToServerMap.put(file.getPath(), server);
-            else {
-              log.info("Ignoring file " + file.getPath() + " because it doesn't look like a uuid");
-            }
-          }
-        } else if (isUUID(server)) {
-          // old-style WAL are not under a directory
-          fileToServerMap.put(status.getPath(), "");
-        } else {
-          log.info("Ignoring file " + status.getPath() + " because it doesn't look like a uuid");
-        }
-      }
-    }
-    return servers.size();
-  }
-  
-  private Set<Path> getSortedWALogs() throws IOException {
-    Set<Path> result = new HashSet<Path>();
-    
-    for (String dir : ServerConstants.getRecoveryDirs()) {
-      Path recoveryDir = new Path(dir);
-      
-      if (fs.exists(recoveryDir)) {
-        for (FileStatus status : fs.listStatus(recoveryDir)) {
-          if (isUUID(status.getPath().getName())) {
-            result.add(status.getPath());
-          } else {
-            log.debug("Ignoring file " + status.getPath() + " because it doesn't look like a uuid");
-          }
-        }
-      }
-    }
-    return result;
-  }
-  
-  static private boolean isUUID(String name) {
-    try {
-      UUID.fromString(name);
-      return true;
-    } catch (IllegalArgumentException ex) {
-      return false;
-    }
-  }
-  
-}


Mime
View raw message