accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject git commit: ACCUMULO-1773 properly handled exceptions in AGC
Date Tue, 29 Oct 2013 13:39:15 GMT
Updated Branches:
  refs/heads/master c2dee4be5 -> cc7b7ef69


ACCUMULO-1773 properly handled exceptions in AGC


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cc7b7ef6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cc7b7ef6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cc7b7ef6

Branch: refs/heads/master
Commit: cc7b7ef69b389010b6700a1723674ee6edf04a18
Parents: c2dee4b
Author: Keith Turner <kturner@apache.org>
Authored: Tue Oct 29 09:38:31 2013 -0400
Committer: Keith Turner <kturner@apache.org>
Committed: Tue Oct 29 09:38:44 2013 -0400

----------------------------------------------------------------------
 .../server/gc/GarbageCollectionAlgorithm.java   | 61 ++++++++++++--------
 .../server/gc/GarbageCollectionEnvironment.java | 35 ++++++++---
 .../server/gc/SimpleGarbageCollector.java       | 21 +------
 3 files changed, 68 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc7b7ef6/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java
b/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java
index 02018bc..325f1d9 100644
--- a/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java
+++ b/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java
@@ -119,7 +119,7 @@ public class GarbageCollectionAlgorithm {
     return ret;
   }
 
-  protected void confirmDeletes(GarbageCollectionEnvironment gce, SortedMap<String,String>
candidateMap) throws TableNotFoundException, AccumuloException,
+  private void confirmDeletes(GarbageCollectionEnvironment gce, SortedMap<String,String>
candidateMap) throws TableNotFoundException, AccumuloException,
       AccumuloSecurityException {
     boolean checkForBulkProcessingFiles = false;
     Iterator<String> relativePaths = candidateMap.keySet().iterator();
@@ -227,18 +227,46 @@ public class GarbageCollectionAlgorithm {
 
   }
 
+  private List<String> getCandidates(GarbageCollectionEnvironment gce, String lastCandidate)
throws TableNotFoundException, AccumuloException,
+      AccumuloSecurityException {
+    Span candidatesSpan = Trace.start("getCandidates");
+    List<String> candidates;
+    try {
+      candidates = gce.getCandidates(lastCandidate);
+    } finally {
+      candidatesSpan.stop();
+    }
+    return candidates;
+  }
+
+  private void confirmDeletesTrace(GarbageCollectionEnvironment gce, SortedMap<String,String>
candidateMap) throws TableNotFoundException, AccumuloException,
+      AccumuloSecurityException {
+    Span confirmDeletesSpan = Trace.start("confirmDeletes");
+    try {
+      confirmDeletes(gce, candidateMap);
+    } finally {
+      confirmDeletesSpan.stop();
+    }
+  }
+
+  private void deleteConfirmed(GarbageCollectionEnvironment gce, SortedMap<String,String>
candidateMap) throws IOException, AccumuloException,
+      AccumuloSecurityException, TableNotFoundException {
+    Span deleteSpan = Trace.start("deleteFiles");
+    try {
+      gce.delete(candidateMap);
+    } finally {
+      deleteSpan.stop();
+    }
+
+    cleanUpDeletedTableDirs(gce, candidateMap);
+  }
+
   public void collect(GarbageCollectionEnvironment gce) throws TableNotFoundException, AccumuloException,
AccumuloSecurityException, IOException {
 
     String lastCandidate = "";
 
     while (true) {
-      Span candidatesSpan = Trace.start("getCandidates");
-      List<String> candidates;
-      try {
-        candidates = gce.getCandidates(lastCandidate);
-      } finally {
-        candidatesSpan.stop();
-      }
+      List<String> candidates = getCandidates(gce, lastCandidate);
 
       if (candidates.size() == 0)
         break;
@@ -250,23 +278,10 @@ public class GarbageCollectionAlgorithm {
 
       SortedMap<String,String> candidateMap = makeRelative(candidates);
 
-      Span confirmDeletesSpan = Trace.start("confirmDeletes");
-      try {
-        confirmDeletes(gce, candidateMap);
-      } finally {
-        confirmDeletesSpan.stop();
-      }
+      confirmDeletesTrace(gce, candidateMap);
       gce.incrementInUseStat(origSize - candidateMap.size());
 
-      Span deleteSpan = Trace.start("deleteFiles");
-      try {
-        gce.delete(candidateMap);
-      } finally {
-        deleteSpan.stop();
-      }
-
-      cleanUpDeletedTableDirs(gce, candidateMap);
+      deleteConfirmed(gce, candidateMap);
     }
   }
-
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc7b7ef6/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java
b/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java
index fa35ed3..1d39254 100644
--- a/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java
+++ b/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.gc;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.SortedMap;
@@ -28,6 +29,8 @@ import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 
@@ -37,9 +40,11 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Sc
 public interface GarbageCollectionEnvironment {
 
   /**
-   * Return a list of paths to files which are candidates for deletion from a given table,
{@link RootTable.NAME} or {@link MetadataTable.NAME}
-   * @param continuePoint A row to resume from if a previous invocation was stopped due to
finding an extremely large number
-   *            of candidates to remove which would have exceeded memory limitations
+   * Return a list of paths to files and dirs which are candidates for deletion from a given
table, {@link RootTable.NAME} or {@link MetadataTable.NAME}
+   * 
+   * @param continuePoint
+   *          A row to resume from if a previous invocation was stopped due to finding an
extremely large number of candidates to remove which would have
+   *          exceeded memory limitations
    * @return A collection of candidates files for deletion, may not be the complete collection
of files for deletion at this point in time
    * @throws TableNotFoundException
    * @throws AccumuloException
@@ -49,6 +54,7 @@ public interface GarbageCollectionEnvironment {
 
   /**
    * Fetch a list of paths for all bulk loads in progress (blip) from a given table, {@link
RootTable.NAME} or {@link MetadataTable.NAME}
+   * 
    * @return The list of files for each bulk load currently in progress.
    * @throws TableNotFoundException
    * @throws AccumuloException
@@ -58,6 +64,7 @@ public interface GarbageCollectionEnvironment {
 
   /**
    * Fetches the references to files, {@link DataFileColumnFamily.NAME} or {@link ScanFileColumnFamily.NAME},
from tablets
+   * 
    * @return An Iterator to the @{link Entry<Key,Value>}s which constitute a reference
to a file.
    * @throws TableNotFoundException
    * @throws AccumuloException
@@ -67,33 +74,45 @@ public interface GarbageCollectionEnvironment {
 
   /**
    * Return the set of tableIDs for the given instance this GarbageCollector is running over
+   * 
    * @return The valueSet for the table name to table id map.
    */
   Set<String> getTableIDs();
 
   /**
    * Delete the given files from the provided {@link Map} of relative path to absolute path
for each file that should be deleted
-   * @param candidateMap A Map from relative path to absolute path for files to be deleted.
+   * 
+   * @param candidateMap
+   *          A Map from relative path to absolute path for files to be deleted.
    * @throws IOException
+   * @throws AccumuloSecurityException
+   * @throws AccumuloException
+   * @throws TableNotFoundException
    */
-  void delete(SortedMap<String,String> candidateMap) throws IOException;
+  void delete(SortedMap<String,String> candidateMap) throws IOException, AccumuloException,
AccumuloSecurityException, TableNotFoundException;
 
   /**
    * Delete a table's directory if it is empty.
-   * @param tableID The id of the table whose directory we are to operate on
+   * 
+   * @param tableID
+   *          The id of the table whose directory we are to operate on
    * @throws IOException
    */
   void deleteTableDirIfEmpty(String tableID) throws IOException;
 
   /**
    * Increment the number of candidates for deletion for the current garbage collection run
-   * @param i Value to increment the deletion candidates by
+   * 
+   * @param i
+   *          Value to increment the deletion candidates by
    */
   void incrementCandidatesStat(long i);
 
   /**
    * Increment the number of files still in use for the current garbage collection run
-   * @param i Value to increment the still-in-use count by.
+   * 
+   * @param i
+   *          Value to increment the still-in-use count by.
    */
   void incrementInUseStat(long i);
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc7b7ef6/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
index 2619a1c..6dd9d2c 100644
--- a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
+++ b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
@@ -238,7 +238,7 @@ public class SimpleGarbageCollector implements Iface {
     }
 
     @Override
-    public void delete(SortedMap<String,String> confirmedDeletes) throws IOException
{
+    public void delete(SortedMap<String,String> confirmedDeletes) throws IOException,
AccumuloException, AccumuloSecurityException, TableNotFoundException {
 
       if (opts.safeMode) {
         if (opts.verbose)
@@ -251,23 +251,8 @@ public class SimpleGarbageCollector implements Iface {
         return;
       }
 
-      // create a batchwriter to remove the delete flags for successful
-      // deletes; Need separate writer for the root tablet.
-      BatchWriter writer = null;
-      Connector c;
-      try {
-        c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
-        writer = c.createBatchWriter(tableName, new BatchWriterConfig());
-      } catch (AccumuloException e) {
-        log.error("Unable to connect to Accumulo to write deletes", e);
-        // TODO Throw exception or return?
-      } catch (AccumuloSecurityException e) {
-        log.error("Unable to connect to Accumulo to write deletes", e);
-        // TODO Throw exception or return?
-      } catch (TableNotFoundException e) {
-        log.error("Unable to create writer to remove file from the " + e.getTableName() +
" table", e);
-        // TODO Throw exception or return?
-      }
+      Connector c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+      BatchWriter writer = c.createBatchWriter(tableName, new BatchWriterConfig());
 
       // when deleting a dir and all files in that dir, only need to delete the dir
       // the dir will sort right before the files... so remove the files in this case


Mime
View raw message