accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [1/2] accumulo git commit: ACCUMULO-4021 explicitly return a value that marks the end of GC
Date Tue, 20 Oct 2015 20:15:29 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.7 1f28ee760 -> 89d69f60c


ACCUMULO-4021 explicitly return a value that marks the end of GC


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/deef5352
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/deef5352
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/deef5352

Branch: refs/heads/1.7
Commit: deef5352db0bec4b1578568c3da84220c43038a0
Parents: 5ca779a
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Tue Oct 20 16:14:35 2015 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Tue Oct 20 16:14:35 2015 -0400

----------------------------------------------------------------------
 .../accumulo/gc/GarbageCollectionAlgorithm.java     | 16 +++++++++-------
 .../accumulo/gc/GarbageCollectionEnvironment.java   |  6 ++++--
 .../apache/accumulo/gc/SimpleGarbageCollector.java  | 10 ++++------
 .../apache/accumulo/gc/GarbageCollectionTest.java   |  5 ++---
 4 files changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
index 0e016d5..a8cde31 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
@@ -113,7 +113,7 @@ public class GarbageCollectionAlgorithm {
       try {
         relPath = makeRelative(candidate, 0);
       } catch (IllegalArgumentException iae) {
-        log.warn("Ingoring invalid deletion candidate " + candidate);
+        log.warn("Ignoring invalid deletion candidate " + candidate);
         continue;
       }
       ret.put(relPath, candidate);
@@ -230,16 +230,15 @@ public class GarbageCollectionAlgorithm {
 
   }
 
-  private List<String> getCandidates(GarbageCollectionEnvironment gce, String lastCandidate)
throws TableNotFoundException, AccumuloException,
+  private boolean getCandidates(GarbageCollectionEnvironment gce, String lastCandidate, List<String>
candidates)
+      throws TableNotFoundException, AccumuloException,
       AccumuloSecurityException {
     Span candidatesSpan = Trace.start("getCandidates");
-    List<String> candidates;
     try {
-      candidates = gce.getCandidates(lastCandidate);
+      return gce.getCandidates(lastCandidate, candidates);
     } finally {
       candidatesSpan.stop();
     }
-    return candidates;
   }
 
   private void confirmDeletesTrace(GarbageCollectionEnvironment gce, SortedMap<String,String>
candidateMap) throws TableNotFoundException, AccumuloException,
@@ -268,8 +267,11 @@ public class GarbageCollectionAlgorithm {
 
     String lastCandidate = "";
 
-    while (true) {
-      List<String> candidates = getCandidates(gce, lastCandidate);
+    boolean outOfMemory = true;
+    while (outOfMemory) {
+      List<String> candidates = new ArrayList<String>();
+
+      outOfMemory = getCandidates(gce, lastCandidate, candidates);
 
       if (candidates.size() == 0)
         break;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
index cca2411..4eab25a 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
@@ -45,9 +45,11 @@ public interface GarbageCollectionEnvironment {
    * @param continuePoint
    *          A row to resume from if a previous invocation was stopped due to finding an
extremely large number of candidates to remove which would have
    *          exceeded memory limitations
-   * @return A collection of candidates files for deletion, may not be the complete collection
of files for deletion at this point in time
+   * @param candidates
+   *          A collection of candidates files for deletion, may not be the complete collection
of files for deletion at this point in time
+   * @return true if the results are short due to insufficient memory, otherwise false
    */
-  List<String> getCandidates(String continuePoint) throws TableNotFoundException, AccumuloException,
AccumuloSecurityException;
+  boolean getCandidates(String continuePoint, List<String> candidates) throws TableNotFoundException,
AccumuloException, AccumuloSecurityException;
 
   /**
    * Fetch a list of paths for all bulk loads in progress (blip) from a given table, {@link
RootTable#NAME} or {@link MetadataTable#NAME}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 8761480..3ab95a7 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -19,7 +19,6 @@ package org.apache.accumulo.gc;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.UnknownHostException;
-import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -269,7 +268,7 @@ public class SimpleGarbageCollector implements Iface {
     }
 
     @Override
-    public List<String> getCandidates(String continuePoint) throws TableNotFoundException,
AccumuloException, AccumuloSecurityException {
+    public boolean getCandidates(String continuePoint, List<String> result) throws
TableNotFoundException, AccumuloException, AccumuloSecurityException {
       // want to ensure GC makes progress... if the 1st N deletes are stable and we keep
processing them,
       // then will never inspect deletes after N
       Range range = MetadataSchema.DeletesSection.getRange();
@@ -280,19 +279,18 @@ public class SimpleGarbageCollector implements Iface {
 
       Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName,
Authorizations.EMPTY);
       scanner.setRange(range);
-      List<String> result = new ArrayList<String>();
+      result.clear();
       // find candidates for deletion; chop off the prefix
       for (Entry<Key,Value> entry : scanner) {
         String cand = entry.getKey().getRow().toString().substring(MetadataSchema.DeletesSection.getRowPrefix().length());
         result.add(cand);
         if (almostOutOfMemory(Runtime.getRuntime())) {
           log.info("List of delete candidates has exceeded the memory threshold. Attempting
to delete what has been gathered so far.");
-          break;
+          return true;
         }
       }
 
-      return result;
-
+      return false;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 50c3957..2953d07 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@ -51,14 +51,13 @@ public class GarbageCollectionTest {
     ArrayList<String> tablesDirsToDelete = new ArrayList<String>();
 
     @Override
-    public List<String> getCandidates(String continuePoint) {
+    public boolean getCandidates(String continuePoint, List<String> ret) {
       Iterator<String> iter = candidates.tailSet(continuePoint, false).iterator();
-      ArrayList<String> ret = new ArrayList<String>();
       while (iter.hasNext() && ret.size() < 3) {
         ret.add(iter.next());
       }
 
-      return ret;
+      return ret.size() == 3;
     }
 
     @Override


Mime
View raw message