accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1467331 - in /accumulo/trunk: ./ assemble/ core/ core/src/main/java/org/apache/accumulo/core/ examples/ fate/src/main/java/org/apache/accumulo/fate/ fate/src/main/java/org/apache/accumulo/fate/zookeeper/ server/ server/src/main/java/org/ap...
Date Fri, 12 Apr 2013 16:11:26 GMT
Author: ecn
Date: Fri Apr 12 16:11:25 2013
New Revision: 1467331

URL: http://svn.apache.org/r1467331
Log:
ACCUMULO-1264 put delete markers for the METADATA table into the root tablet

Modified:
    accumulo/trunk/   (props changed)
    accumulo/trunk/assemble/   (props changed)
    accumulo/trunk/core/   (props changed)
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
    accumulo/trunk/examples/   (props changed)
    accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java   (props changed)
    accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java 
 (props changed)
    accumulo/trunk/server/   (props changed)
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
    accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
    accumulo/trunk/src/   (props changed)

Propchange: accumulo/trunk/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5:r1467312

Propchange: accumulo/trunk/assemble/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/assemble:r1467312

Propchange: accumulo/trunk/core/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/core:r1467312

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java Fri Apr 12 16:11:25
2013
@@ -97,7 +97,9 @@ public class Constants {
   public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[]
{'~' + 1}));
   public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY,
true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
   public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
+  public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX;
   public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)),
true, new Key(new Text("~dem")), false);
+  public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new
Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false);
   public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in
progress
   public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)),
true, new Key(new Text("~bliq")), false);
   

Propchange: accumulo/trunk/examples/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/examples:r1467312

Propchange: accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:r1467312

Propchange: accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java:r1467312

Propchange: accumulo/trunk/server/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/server:r1467312

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
Fri Apr 12 16:11:25 2013
@@ -101,9 +101,11 @@ public class MetadataConstraints impleme
     
     byte[] row = mutation.getRow();
     
-    // always allow rows that fall within reserved area
+    // always allow rows that fall within reserved areas
     if (row.length > 0 && row[0] == '~')
       return null;
+    if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2]
== '~')
+      return null;
     
     for (byte b : row) {
       if (b == ';') {

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Fri Apr 12 16:11:25 2013
@@ -20,8 +20,12 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -438,23 +442,37 @@ public class SimpleGarbageCollector impl
       }
       return candidates;
     }
+
+    checkForBulkProcessingFiles = false;
+    Range range = Constants.METADATA_DELETES_FOR_METADATA_KEYSPACE;
+    candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX, range));
+    if (candidateMemExceeded)
+      return candidates;
     
-    Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME,
Constants.NO_AUTHS);
+    range = Constants.METADATA_DELETES_KEYSPACE;
+    candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_PREFIX, range));
+    return candidates;
+  }
 
+  private Collection<String> getBatch(String prefix, Range range) throws Exception
{
+    // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing
them, 
+    // then will never inspect deletes after N
     if (continueKey != null) {
-      // want to ensure GC makes progress... if the 1st N deletes are stable and we keep
processing them, then will never inspect deletes after N
-      scanner.setRange(new Range(continueKey, true, Constants.METADATA_DELETES_KEYSPACE.getEndKey(),
Constants.METADATA_DELETES_KEYSPACE.isEndKeyInclusive()));
+      if (!range.contains(continueKey)) {
+        // continue key is for some other range
+        return Collections.emptyList();
+      }
+      range = new Range(continueKey, true, range.getEndKey(), range.isEndKeyInclusive());
       continueKey = null;
-    } else {
-      // scan the reserved keyspace for deletes
-      scanner.setRange(Constants.METADATA_DELETES_KEYSPACE);
     }
     
+    Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME,
Constants.NO_AUTHS);
+    scanner.setRange(range);
+    List<String> result = new ArrayList<String>();
     // find candidates for deletion; chop off the prefix
-    checkForBulkProcessingFiles = false;
     for (Entry<Key,Value> entry : scanner) {
-      String cand = entry.getKey().getRow().toString().substring(Constants.METADATA_DELETE_FLAG_PREFIX.length());
-      candidates.add(cand);
+      String cand = entry.getKey().getRow().toString().substring(prefix.length());
+      result.add(cand);
       checkForBulkProcessingFiles |= cand.toLowerCase(Locale.ENGLISH).contains(Constants.BULK_PREFIX);
       if (almostOutOfMemory()) {
         candidateMemExceeded = true;
@@ -464,7 +482,7 @@ public class SimpleGarbageCollector impl
       }
     }
     
-    return candidates;
+    return result;
   }
   
   static public boolean almostOutOfMemory() {
@@ -567,7 +585,18 @@ public class SimpleGarbageCollector impl
       }
     }
   }
-  
+
+  final static String METADATA_TABLE_DIR = "/" + Constants.METADATA_TABLE_ID;
+  private static Mutation getMarkerDeleteMutation(final String delete) {
+    String prefix = Constants.METADATA_DELETE_FLAG_PREFIX;
+    if (delete.startsWith(METADATA_TABLE_DIR))
+      prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX;
+
+    Mutation m = new Mutation(new Text(prefix + delete));
+    m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
+    return m;
+  }
+
   /**
    * This method attempts to do its best to remove files from the filesystem that have been
confirmed for deletion.
    */
@@ -584,7 +613,6 @@ public class SimpleGarbageCollector impl
         log.error("Unable to create writer to remove file from the !METADATA table", e);
       }
     }
-    
     // when deleting a dir and all files in that dir, only need to delete the dir
     // the dir will sort right before the files... so remove the files in this case
     // to minimize namenode ops
@@ -597,10 +625,8 @@ public class SimpleGarbageCollector impl
       } else if (lastDir != null) {
         if (delete.startsWith(lastDir)) {
           log.debug("Ignoring " + delete + " because " + lastDir + " exist");
-          Mutation m = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + delete));
-          m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
           try {
-            writer.addMutation(m);
+            writer.addMutation(getMarkerDeleteMutation(delete));
           } catch (MutationsRejectedException e) {
             throw new RuntimeException(e);
           }
@@ -623,10 +649,11 @@ public class SimpleGarbageCollector impl
         public void run() {
           boolean removeFlag;
           
-          log.debug("Deleting " + ServerConstants.getTablesDir() + delete);
+          String fullPath = ServerConstants.getTablesDir() + delete;
+          log.debug("Deleting " + fullPath);
           try {
             
-            Path p = new Path(ServerConstants.getTablesDir() + delete);
+            Path p = new Path(fullPath);
             
             if (moveToTrash(p) || fs.delete(p, true)) {
               // delete succeeded, still want to delete
@@ -668,15 +695,14 @@ public class SimpleGarbageCollector impl
             // proceed to clearing out the flags for successful deletes and
             // non-existent files
             if (removeFlag && finalWriter != null) {
-              Mutation m = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX +
delete));
-              m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
-              finalWriter.addMutation(m);
+              finalWriter.addMutation(getMarkerDeleteMutation(delete));
             }
           } catch (Exception e) {
             log.error(e, e);
           }
           
         }
+
       };
       
       deleteThreadPool.execute(deleteTask);

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Fri Apr 12 16:11:25 2013
@@ -479,11 +479,15 @@ public class MetadataTable extends org.a
   
   public static Mutation createDeleteMutation(String tableId, String pathToRemove) {
     Mutation delFlag;
+    String prefix = Constants.METADATA_DELETE_FLAG_PREFIX;
+    if (tableId.equals(Constants.METADATA_TABLE_ID))
+      prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX;
+
     if (pathToRemove.startsWith("../"))
-      delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + pathToRemove.substring(2)));
+      delFlag = new Mutation(new Text(prefix + pathToRemove.substring(2)));
     else
-      delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + "/" + tableId
+ pathToRemove));
-    
+      delFlag = new Mutation(new Text(prefix + "/" + tableId + pathToRemove));
+
     delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
     return delFlag;
   }

Modified: accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
(original)
+++ accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
Fri Apr 12 16:11:25 2013
@@ -21,7 +21,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.TreeMap;
 
-import junit.framework.Assert;
+import org.junit.Assert;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;

Propchange: accumulo/trunk/src/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/src:r1467312



Mime
View raw message