accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1494340 - in /accumulo/branches/ACCUMULO-118/server/src: main/java/org/apache/accumulo/server/fs/ main/java/org/apache/accumulo/server/master/recovery/ main/java/org/apache/accumulo/server/tabletserver/ main/java/org/apache/accumulo/server...
Date Tue, 18 Jun 2013 22:14:06 GMT
Author: ecn
Date: Tue Jun 18 22:14:06 2013
New Revision: 1494340

URL: http://svn.apache.org/r1494340
Log:
ACCUMULO-118 cleanup TODOs simplify VolumeManager interface, add some basic documentation

Modified:
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java
    accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
    accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
Tue Jun 18 22:14:06 2013
@@ -28,7 +28,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
 /**
- * A wrapper around multiple hadoop FileSystem objects, which are assumed to be different
namespaces.
+ * A wrapper around multiple hadoop FileSystem objects, which are assumed to be different
volumes.
+ * This also concentrates a bunch of meta-operations like waiting for SAFE_MODE, and closing
WALs.
  */
 public interface VolumeManager {
   
@@ -68,12 +69,10 @@ public interface VolumeManager {
   // find the appropriate FileSystem object given a path
   FileSystem getFileSystemByPath(Path path);
   
-  FileSystem getFileSystemByPath(String path);
-  
   // get a mapping of volume to FileSystem
   Map<String, ? extends FileSystem> getFileSystems();
   
-  // return the item in options that is in the same namespace as source
+  // return the item in options that is in the same volume as source
   Path matchingFileSystem(Path source, String[] options);
   
   // create a new path in the same volume as the sourceDir
@@ -88,7 +87,7 @@ public interface VolumeManager {
   // forward to the appropriate FileSystem object
   FSDataInputStream open(Path path) throws IOException;
   
-  // forward to the appropriate FileSystem object, throws an exception if the paths are in
different namespaces
+  // forward to the appropriate FileSystem object, throws an exception if the paths are in
different volumes
   boolean rename(Path path, Path newPath) throws IOException;
   
   // forward to the appropriate FileSystem object
@@ -118,5 +117,6 @@ public interface VolumeManager {
   // forward to the appropriate FileSystem object
   ContentSummary getContentSummary(Path dir) throws IOException;
 
+  // decide on which of the given locations to create a new file
   String choose(String[] options);
 }

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
Tue Jun 18 22:14:06 2013
@@ -260,11 +260,6 @@ public class VolumeManagerImpl implement
   }
 
   @Override
-  public FileSystem getFileSystemByPath(String path) {
-    return getFileSystemByPath(new Path(path));
-  }
-
-  @Override
   public Map<String, ? extends FileSystem>  getFileSystems() {
     return volumes;
   }
@@ -399,9 +394,9 @@ public class VolumeManagerImpl implement
       relPath = relPath.substring(2);
     else
       relPath = "/" + new String(tableId) + relPath;
-    String fullPath = ServerConstants.getTablesDirs()[0] + relPath;
+    Path fullPath = new Path(ServerConstants.getTablesDirs()[0] + relPath);
     FileSystem fs = getFileSystemByPath(fullPath);
-    return fs.makeQualified(new Path(fullPath));
+    return fs.makeQualified(fullPath);
   }
 
   @Override
@@ -440,8 +435,8 @@ public class VolumeManagerImpl implement
         fullPath = path + fileName;
       else
         fullPath = path + "/" + fileName;
-      FileSystem ns = getFileSystemByPath(fullPath);
       Path exists = new Path(fullPath);
+      FileSystem ns = getFileSystemByPath(exists);
       if (ns.exists(exists)) {
         Path result = ns.makeQualified(exists);
         log.debug("Found " + exists + " on " + path + " as " + result);

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
Tue Jun 18 22:14:06 2013
@@ -128,7 +128,6 @@ public class RecoveryManager {
         String filename = hostFilename[1];
         String parts[] = filename.split("/");
         String sortId = parts[parts.length - 1];
-        // TODO: ACCUMULO-118: choose recovery directory with extension
         String dest = master.getFileSystem().choose(ServerConstants.getRecoveryDirs()) +
"/" + sortId;
         log.debug("Recovering " + filename + " to " + dest);
         

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
Tue Jun 18 22:14:06 2013
@@ -284,7 +284,7 @@ public class Compactor implements Callab
 
     try {
       FileOperations fileFactory = FileOperations.getInstance();
-      FileSystem ns = this.fs.getFileSystemByPath(outputFile.path().toString());
+      FileSystem ns = this.fs.getFileSystemByPath(outputFile.path());
       mfw = fileFactory.openWriter(outputFile.path().toString(), ns, ns.getConf(), acuTableConf);
       
       Map<String,Set<ByteSequence>> lGroups;
@@ -365,7 +365,7 @@ public class Compactor implements Callab
       try {
         
         FileOperations fileFactory = FileOperations.getInstance();
-        FileSystem fs = this.fs.getFileSystemByPath(mapFile.path().toString());
+        FileSystem fs = this.fs.getFileSystemByPath(mapFile.path());
         FileSKVIterator reader;
         
         reader = fileFactory.openReader(mapFile.path().toString(), false, fs, conf, acuTableConf);

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Tue Jun 18 22:14:06 2013
@@ -650,10 +650,17 @@ public class Tablet {
       
       for (FileRef tpath : paths.keySet()) {
         
-        // TODO ACCUMULO-118
-//        if (!tpath.getParent().getParent().equals(new Path(ServerConstants.getTablesDir()
+ "/" + extent.getTableId()))) {
-//          throw new IOException("Map file " + tpath + " not in table dir " + ServerConstants.getTablesDir()
+ "/" + extent.getTableId());
-//        }
+        boolean inTheRightDirectory = false;
+        Path parent = tpath.path().getParent().getParent();
+        for (String tablesDir : ServerConstants.getTablesDirs()) {
+          if (parent.equals(new Path(tablesDir, extent.getTableId().toString()))) {
+            inTheRightDirectory = true;
+            break;
+          }
+        }
+        if (!inTheRightDirectory) {
+          throw new IOException("Map file " + tpath + " not in table dirs");
+        }
         
         if (bulkDir == null)
           bulkDir = tpath.path().getParent().toString();

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Tue Jun 18 22:14:06 2013
@@ -882,8 +882,9 @@ public class TabletServer extends Abstra
         Map<String,MapFileInfo> fileMap = entry.getValue();
         Map<FileRef, MapFileInfo> fileRefMap = new HashMap<FileRef, MapFileInfo>();
         for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
-          FileSystem ns = fs.getFileSystemByPath(mapping.getKey());
-          Path path = ns.makeQualified(new Path(mapping.getKey()));
+          Path path = new Path(mapping.getKey());
+          FileSystem ns = fs.getFileSystemByPath(path);
+          path = ns.makeQualified(path);
           fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
         }
         

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
Tue Jun 18 22:14:06 2013
@@ -196,9 +196,9 @@ public class LogSorter {
     }
     
     private void writeBuffer(String destPath, ArrayList<Pair<LogFileKey,LogFileValue>>
buffer, int part) throws IOException {
-      String path = destPath + String.format("/part-r-%05d", part++);
+      Path path = new Path(destPath, String.format("part-r-%05d", part++));
       FileSystem ns = fs.getFileSystemByPath(path);
-      MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns, path, LogFileKey.class,
LogFileValue.class);
+      MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns, path.toString(), LogFileKey.class,
LogFileValue.class);
       try {
         Collections.sort(buffer, new Comparator<Pair<LogFileKey,LogFileValue>>()
{
           @Override

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/FileUtil.java
Tue Jun 18 22:14:06 2013
@@ -121,11 +121,11 @@ public class FileUtil {
       
       start = end;
       
-      String newMapFile = String.format("%s/%04d." + RFile.EXTENSION, newDir, count++);
+      FileRef newMapFile = new FileRef(String.format("%s/%04d." + RFile.EXTENSION, newDir,
count++));
       
-      outFiles.add(new FileRef(newMapFile));
-      FileSystem ns = fs.getFileSystemByPath(newMapFile);
-      FileSKVWriter writer = new RFileOperations().openWriter(newMapFile, ns, ns.getConf(),
acuConf);
+      outFiles.add(newMapFile);
+      FileSystem ns = fs.getFileSystemByPath(newMapFile.path());
+      FileSKVWriter writer = new RFileOperations().openWriter(newMapFile.toString(), ns,
ns.getConf(), acuConf);
       writer.startDefaultLocalityGroup();
       List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(inFiles.size());
       
@@ -434,10 +434,10 @@ public class FileUtil {
     for (FileRef mapfile : mapfiles) {
       
       FileSKVIterator reader = null;
-      String path = mapfile.path().toString();
-      FileSystem ns = fs.getFileSystemByPath(path);
+      //String path = mapfile.path().toString();
+      FileSystem ns = fs.getFileSystemByPath(mapfile.path());
       try {
-        reader = FileOperations.getInstance().openReader(path, false, ns, ns.getConf(), acuConf);
+        reader = FileOperations.getInstance().openReader(mapfile.toString(), false, ns, ns.getConf(),
acuConf);
         
         Key firstKey = reader.getFirstKey();
         if (firstKey != null) {

Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
Tue Jun 18 22:14:06 2013
@@ -68,7 +68,7 @@ public class OfflineMetadataScanner exte
   private List<SortedKeyValueIterator<Key,Value>> openMapFiles(Collection<String>
files, VolumeManager fs, AccumuloConfiguration conf) throws IOException {
     List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>();
     for (String file : files) {
-      FileSystem ns = fs.getFileSystemByPath(file);
+      FileSystem ns = fs.getFileSystemByPath(new Path(file));
       FileSKVIterator reader = FileOperations.getInstance().openReader(file, true, ns, ns.getConf(),
conf);
       readers.add(reader);
     }

Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java?rev=1494340&r1=1494339&r2=1494340&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
(original)
+++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
Tue Jun 18 22:14:06 2013
@@ -123,7 +123,7 @@ public class SortedLogRecoveryTest {
     try {
       for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
         String path = workdir + "/" + entry.getKey();
-        FileSystem ns = fs.getFileSystemByPath(path);
+        FileSystem ns = fs.getFileSystemByPath(new Path(path));
         Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class,
LogFileValue.class);
         for (KeyValue lfe : entry.getValue()) {
           map.append(lfe.key, lfe.value);



Mime
View raw message