accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject [1/3] git commit: ACCUMULO-1831 ACCUMULO-1888 use uuids to confirm WALog GC; remove WALog entries in the !METADATA table with the correct mutation
Date Wed, 13 Nov 2013 16:46:20 GMT
Updated Branches:
  refs/heads/master 644d51822 -> 4d6925def


ACCUMULO-1831 ACCUMULO-1888 use uuids to confirm WALog GC; remove WALog entries in the !METADATA
table with the correct mutation


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0e63755d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0e63755d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0e63755d

Branch: refs/heads/master
Commit: 0e63755d5e3a9e441bae51d2f28402a3768820dc
Parents: ed11bd3
Author: Eric Newton <eric.newton@gmail.com>
Authored: Wed Nov 13 11:38:34 2013 -0500
Committer: Eric Newton <eric.newton@gmail.com>
Committed: Wed Nov 13 11:38:34 2013 -0500

----------------------------------------------------------------------
 .../server/util/MasterMetadataUtil.java         |  3 +-
 .../accumulo/server/util/MetadataTableUtil.java |  2 +-
 .../gc/GarbageCollectWriteAheadLogs.java        | 34 ++++++++++++--------
 .../apache/accumulo/tserver/log/DfsLogger.java  |  5 ++-
 4 files changed, 26 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/0e63755d/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
index 987eba9..e936b97 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
@@ -36,7 +36,6 @@ import org.apache.accumulo.core.data.PartialKey;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
@@ -243,7 +242,7 @@ public class MasterMetadataUtil {
    */
   public static void updateTabletDataFile(KeyExtent extent, FileRef path, FileRef mergeFile,
DataFileValue dfv, String time, Credentials credentials,
       Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String>
unusedWalLogs, TServerInstance lastLocation, long flushId) {
-    if (extent.equals(RootTable.EXTENT)) {
+    if (extent.isRootTablet()) {
       if (unusedWalLogs != null) {
         IZooReaderWriter zk = ZooReaderWriter.getInstance();
         // unusedWalLogs will contain the location/name of each log in a log set

http://git-wip-us.apache.org/repos/asf/accumulo/blob/0e63755d/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index cbe42b7..76ea3b1 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -686,7 +686,7 @@ public class MetadataTableUtil {
         }
       } else {
         Mutation m = new Mutation(entry.extent.getMetadataEntry());
-        m.putDelete(LogColumnFamily.NAME, new Text(entry.server + "/" + entry.filename));
+        m.putDelete(LogColumnFamily.NAME, new Text(entry.toString()));
         update(SystemCredentials.get(), zooLock, m, entry.extent);
       }
     }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/0e63755d/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 98b7bd6..6eed80d 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -79,7 +79,8 @@ public class GarbageCollectWriteAheadLogs {
       status.currentLog.started = System.currentTimeMillis();
       
       Map<Path,String> fileToServerMap = new HashMap<Path,String>();
-      int count = scanServers(fileToServerMap);
+      Map<String,Path> nameToFileMap = new HashMap<String, Path>();
+      int count = scanServers(fileToServerMap, nameToFileMap);
       long fileScanStop = System.currentTimeMillis();
       log.info(String.format("Fetched %d files from %d servers in %.2f seconds", fileToServerMap.size(),
count,
           (fileScanStop - status.currentLog.started) / 1000.));
@@ -88,7 +89,7 @@ public class GarbageCollectWriteAheadLogs {
       
       span = Trace.start("removeMetadataEntries");
       try {
-        count = removeMetadataEntries(fileToServerMap, sortedWALogs, status);
+        count = removeMetadataEntries(nameToFileMap, sortedWALogs, status);
       } catch (Exception ex) {
         log.error("Unable to scan metadata table", ex);
         return;
@@ -100,9 +101,9 @@ public class GarbageCollectWriteAheadLogs {
       log.info(String.format("%d log entries scanned in %.2f seconds", count, (logEntryScanStop
- fileScanStop) / 1000.));
       
       span = Trace.start("removeFiles");
-      Map<String,ArrayList<Path>> serverToFileMap = mapServersToFiles(fileToServerMap);
+      Map<String,ArrayList<Path>> serverToFileMap = mapServersToFiles(fileToServerMap,
nameToFileMap);
       
-      count = removeFiles(serverToFileMap, sortedWALogs, status);
+      count = removeFiles(nameToFileMap, serverToFileMap, sortedWALogs, status);
       
       long removeStop = System.currentTimeMillis();
       log.info(String.format("%d total logs removed from %d servers in %.2f seconds", count,
serverToFileMap.size(), (removeStop - logEntryScanStop) / 1000.));
@@ -131,7 +132,7 @@ public class GarbageCollectWriteAheadLogs {
     }
   }
   
-  private int removeFiles(Map<String,ArrayList<Path>> serverToFileMap, Set<Path>
sortedWALogs, final GCStatus status) {
+  private int removeFiles(Map<String,Path> nameToFileMap, Map<String,ArrayList<Path>>
serverToFileMap, Set<Path> sortedWALogs, final GCStatus status) {
     AccumuloConfiguration conf = instance.getConfiguration();
     for (Entry<String,ArrayList<Path>> entry : serverToFileMap.entrySet()) {
       if (entry.getKey().isEmpty()) {
@@ -210,9 +211,11 @@ public class GarbageCollectWriteAheadLogs {
     return result;
   }
   
-  private static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String>
fileToServerMap) {
+  private static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String>
fileToServerMap, Map<String,Path> nameToFileMap) {
     Map<String,ArrayList<Path>> result = new HashMap<String,ArrayList<Path>>();
     for (Entry<Path,String> fileServer : fileToServerMap.entrySet()) {
+      if (!nameToFileMap.containsKey(fileServer.getKey().getName()))
+        continue;
       ArrayList<Path> files = result.get(fileServer.getValue());
       if (files == null) {
         files = new ArrayList<Path>();
@@ -223,7 +226,7 @@ public class GarbageCollectWriteAheadLogs {
     return result;
   }
   
-  private int removeMetadataEntries(Map<Path,String> fileToServerMap, Set<Path>
sortedWALogs, GCStatus status) throws IOException, KeeperException,
+  private int removeMetadataEntries(Map<String,Path>  nameToFileMap, Set<Path>
sortedWALogs, GCStatus status) throws IOException, KeeperException,
       InterruptedException {
     int count = 0;
     Iterator<LogEntry> iterator = MetadataTableUtil.getLogEntries(SystemCredentials.get());
@@ -237,11 +240,11 @@ public class GarbageCollectWriteAheadLogs {
         else
           path = fs.getFullPath(FileType.WAL, filename);
         
-        if (fileToServerMap.remove(path) != null)
+        Path pathFromNN = nameToFileMap.remove(path.getName());
+        if (pathFromNN != null) {
           status.currentLog.inUse++;
-        
-        sortedWALogs.remove(path);
-        
+          sortedWALogs.remove(pathFromNN);
+        }
         count++;
       }
     }
@@ -250,7 +253,7 @@ public class GarbageCollectWriteAheadLogs {
 
   //TODO Remove deprecation warning suppression when Hadoop1 support is dropped
   @SuppressWarnings("deprecation")
-  private int scanServers(Map<Path,String> fileToServerMap) throws Exception {
+  private int scanServers(Map<Path,String> fileToServerMap, Map<String,Path>
nameToFileMap) throws Exception {
     Set<String> servers = new HashSet<String>();
     for (String walDir : ServerConstants.getWalDirs()) {
       Path walRoot = new Path(walDir);
@@ -262,9 +265,10 @@ public class GarbageCollectWriteAheadLogs {
         servers.add(server);
         if (status.isDir()) {
           for (FileStatus file : fs.listStatus(new Path(walRoot, server))) {
-            if (isUUID(file.getPath().getName()))
+            if (isUUID(file.getPath().getName())) {
               fileToServerMap.put(file.getPath(), server);
-            else {
+              nameToFileMap.put(file.getPath().getName(), file.getPath());
+            } else {
               log.info("Ignoring file " + file.getPath() + " because it doesn't look like
a uuid");
             }
           }
@@ -276,6 +280,8 @@ public class GarbageCollectWriteAheadLogs {
         }
       }
     }
+    log.debug("fileToServerMap " + fileToServerMap);
+    log.debug("nameToFileMap " + nameToFileMap);
     return servers.size();
   }
   

http://git-wip-us.apache.org/repos/asf/accumulo/blob/0e63755d/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 612678a..eee3f68 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@ -407,7 +407,10 @@ public class DfsLogger {
   
   @Override
   public String toString() {
-    return getLogger() + "/" + getFileName();
+    String fileName = getFileName();
+    if (fileName.contains(":"))
+      return getLogger() + "/" + getFileName();
+    return fileName;
   }
   
   public String getFileName() {


Mime
View raw message