accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bus...@apache.org
Subject [05/11] git commit: ACCUMULO-1264 move !METADATA file delete markers
Date Thu, 31 Jul 2014 16:33:19 GMT
ACCUMULO-1264 move !METADATA file delete markers

Description: Move the !METADATA file delete markers out of the
  !METADATA table and to the root table on upgrade.
Author: Eric Newton
Ref: ACCUMULO-2988

Modified from commit 5416eef


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/53fcb526
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/53fcb526
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/53fcb526

Branch: refs/heads/1.6.1-SNAPSHOT
Commit: 53fcb5262acaac244ef38a1e20c60c2750fef07c
Parents: 3beaa5a
Author: Mike Drob <mdrob@cloudera.com>
Authored: Fri May 2 12:32:20 2014 -0400
Committer: Sean Busbey <busbey@cloudera.com>
Committed: Thu Jul 31 11:30:58 2014 -0500

----------------------------------------------------------------------
 .../accumulo/server/util/MetadataTableUtil.java | 51 +++++++++++++++-----
 .../java/org/apache/accumulo/master/Master.java |  6 ++-
 2 files changed, 43 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fcb526/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 374017d..827eaa9 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -94,6 +94,7 @@ import org.apache.zookeeper.KeeperException;
 public class MetadataTableUtil {
 
   private static final Text EMPTY_TEXT = new Text();
+  private static final byte[] EMPTY_BYTES = new byte[0];
   private static Map<Credentials,Writer> root_tables = new HashMap<Credentials,Writer>();
   private static Map<Credentials,Writer> metadata_tables = new HashMap<Credentials,Writer>();
   private static final Logger log = Logger.getLogger(MetadataTableUtil.class);
@@ -965,30 +966,54 @@ public class MetadataTableUtil {
    * During an upgrade we need to move deletion requests for files under the !METADATA table
to the root tablet.
    */
   public static void moveMetaDeleteMarkers(Instance instance, Credentials creds) {
-    // move old delete markers to new location, to standardize table schema between all metadata
tables
-    byte[] EMPTY_BYTES = new byte[0];
-    Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID, Authorizations.EMPTY);
     String oldDeletesPrefix = "!!~del";
     Range oldDeletesRange = new Range(oldDeletesPrefix, true, "!!~dem", false);
+
+    // move old delete markers to new location, to standardize table schema between all metadata
tables
+    Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID, Authorizations.EMPTY);
     scanner.setRange(oldDeletesRange);
     for (Entry<Key,Value> entry : scanner) {
       String row = entry.getKey().getRow().toString();
       if (row.startsWith(oldDeletesPrefix)) {
-        String filename = row.substring(oldDeletesPrefix.length());
-        // add the new entry first
-        log.info("Moving " + filename + " marker in " + RootTable.NAME);
-        Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename);
-        m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES);
-        update(creds, m, RootTable.EXTENT);
-        // remove the old entry
-        m = new Mutation(entry.getKey().getRow());
-        m.putDelete(EMPTY_BYTES, EMPTY_BYTES);
-        update(creds, m, RootTable.OLD_EXTENT);
+        moveDeleteEntry(creds, RootTable.OLD_EXTENT, entry, row, oldDeletesPrefix);
+      } else {
+        break;
+      }
+    }
+
+  }
+
+
+  public static void moveMetaDeleteMarkersFrom14(Instance instance, Credentials creds) {
+    // new KeyExtent is only added to force update to write to the metadata table, not the
root table
+    KeyExtent notMetadata = new KeyExtent(new Text("anythingNotMetadata"), null, null);
+
+    // move delete markers from the normal delete keyspace to the root tablet delete keyspace
if the files are for the !METADATA table
+    Scanner scanner = new ScannerImpl(instance, creds, MetadataTable.ID, Authorizations.EMPTY);
+    scanner.setRange(MetadataSchema.DeletesSection.getRange());
+    for (Entry<Key,Value> entry : scanner) {
+      String row = entry.getKey().getRow().toString();
+      if (row.startsWith(MetadataSchema.DeletesSection.getRowPrefix() + "/" + MetadataTable.ID))
{
+        moveDeleteEntry(creds, notMetadata, entry, row, MetadataSchema.DeletesSection.getRowPrefix());
       } else {
         break;
       }
     }
+  }
+
+  private static void moveDeleteEntry(Credentials creds, KeyExtent oldExtent, Entry<Key,Value>
entry, String rowID, String prefix) {
+    String filename = rowID.substring(prefix.length());
+
+    // add the new entry first
+    log.info("Moving " + filename + " marker in " + RootTable.NAME);
+    Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename);
+    m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES);
+    update(creds, m, RootTable.EXTENT);
 
+    // then remove the old entry
+    m = new Mutation(entry.getKey().getRow());
+    m.putDelete(EMPTY_BYTES, EMPTY_BYTES);
+    update(creds, m, oldExtent);
   }
 
   public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(SortedMap<Key,Value>
tabletKeyValues, List<ColumnFQ> columns) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fcb526/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index a2048b5..c1644fa 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -378,7 +378,11 @@ public class Master implements LiveTServerSet.Listener, TableObserver,
CurrentSt
           public void run() {
             try {
               log.info("Starting to upgrade !METADATA table.");
-              MetadataTableUtil.moveMetaDeleteMarkers(instance, SystemCredentials.get());
+              if (accumuloPersistentVersion == ServerConstants.TWO_VERSIONS_AGO) {
+                MetadataTableUtil.moveMetaDeleteMarkersFrom14(instance, SystemCredentials.get());
+              } else {
+                MetadataTableUtil.moveMetaDeleteMarkers(instance, SystemCredentials.get());
+              }
               log.info("Updating persistent data version.");
               Accumulo.updateAccumuloVersion(fs, accumuloPersistentVersion);
               log.info("Upgrade complete");


Mime
View raw message