accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [21/23] git commit: ACCUMULO-2061 Remove implicit assumptions about '/accumulo' directory that all volumes would have.
Date Tue, 11 Mar 2014 18:25:30 GMT
ACCUMULO-2061 Remove implicit assumptions about '/accumulo' directory that all volumes would
have.

Added some extra logging at trace to help if this needs to be revisited.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/039989ad
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/039989ad
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/039989ad

Branch: refs/heads/ACCUMULO-2061
Commit: 039989ad44bb9b46d620a3e302e15c32b3b3840a
Parents: c3214e0
Author: Josh Elser <elserj@apache.org>
Authored: Tue Mar 11 13:57:48 2014 -0400
Committer: Josh Elser <elserj@apache.org>
Committed: Tue Mar 11 13:57:48 2014 -0400

----------------------------------------------------------------------
 .../apache/accumulo/server/ServerConstants.java |  4 +-
 .../accumulo/server/fs/VolumeManager.java       |  3 +-
 .../apache/accumulo/server/fs/VolumeUtil.java   | 20 ++++-
 .../accumulo/server/fs/VolumeUtilTest.java      | 89 ++++++++++++++++++--
 4 files changed, 103 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 6c27897..7dd0a08 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -187,8 +187,8 @@ public class ServerConstants {
 
       HashSet<Path> baseDirs = new HashSet<Path>();
       for (String baseDir : getBaseUris()) {
-        // normalize using path and remove accumulo dir
-        baseDirs.add(new Path(baseDir).getParent());
+        // normalize using path
+        baseDirs.add(new Path(baseDir));
       }
 
       for (Pair<Path,Path> pair : ret)

http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index c345ede..9b8fb98 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -48,9 +48,10 @@ public interface VolumeManager {
     }
 
     private static int endOfVolumeIndex(String path, String dir) {
+      // Strip off the suffix that starts with the FileType (e.g. tables, wal, etc)
       int dirIndex = path.indexOf('/' + dir);
       if (dirIndex != -1) {
-        return path.lastIndexOf('/', dirIndex - 1);
+        return dirIndex;
       }
 
       if (path.contains(":"))

http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 110f1e4..bcfb008 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -81,8 +81,10 @@ public class VolumeUtil {
   }
 
   public static String switchVolume(String path, FileType ft, List<Pair<Path,Path>>
replacements) {
-    if (replacements.size() == 0)
+    if (replacements.size() == 0) {
+      log.trace("Not switching volume because there are no replacements");
       return null;
+    }
 
     Path p = new Path(path);
 
@@ -92,10 +94,15 @@ public class VolumeUtil {
     for (Pair<Path,Path> pair : replacements) {
       Path key = removeTrailingSlash(pair.getFirst());
 
-      if (key.equals(volume))
-        return new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+      if (key.equals(volume)) {
+        String replacement = new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+        log.trace("Replacing " + path + " with " + replacement);
+        return replacement;
+      }
     }
 
+    log.trace("Could not find replacement for " + ft + " at " + path);
+
     return null;
   }
 
@@ -119,13 +126,17 @@ public class VolumeUtil {
 
     }
 
-    if (numSwitched == 0)
+    if (numSwitched == 0) {
+      log.trace("Did not switch " + le);
       return null;
+    }
 
     LogEntry newLogEntry = new LogEntry(le);
     newLogEntry.filename = switchedPath;
     newLogEntry.logSet = switchedLogs;
 
+    log.trace("Switched " + le + " to " + newLogEntry);
+
     return newLogEntry;
   }
 
@@ -165,6 +176,7 @@ public class VolumeUtil {
    */
   public static TabletFiles updateTabletVolumes(ZooLock zooLock, VolumeManager vm, KeyExtent
extent, TabletFiles tabletFiles) throws IOException {
     List<Pair<Path,Path>> replacements = ServerConstants.getVolumeReplacements();
+    log.trace("Using volume replacements: " + replacements);
 
     List<LogEntry> logsToRemove = new ArrayList<LogEntry>();
     List<LogEntry> logsToAdd = new ArrayList<LogEntry>();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
index c85be45..3b905c9 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
@@ -43,9 +43,9 @@ public class VolumeUtilTest {
   @Test
   public void testSwitchVolume() {
     List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
-    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1"), new Path("viewfs:/a")));
-    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/"), new Path("viewfs:/a")));
-    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/"), new Path("viewfs:/b")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a/accumulo")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new
Path("viewfs:/a/accumulo")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b/accumulo")));
 
     Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
         VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
@@ -57,9 +57,9 @@ public class VolumeUtilTest {
     Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf",
FileType.TABLE, replacements));
 
     replacements.clear();
-    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1"), new Path("viewfs:/a")));
-    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1"), new Path("viewfs:/a")));
-    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/"), new Path("viewfs:/b")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a/accumulo")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new
Path("viewfs:/a/accumulo")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b/accumulo")));
 
     Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
         VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
@@ -73,6 +73,70 @@ public class VolumeUtilTest {
   }
 
   @Test
+  public void testSwitchVolumesDifferentSourceDepths() {
+    List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new
Path("viewfs:/a")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b")));
+
+    Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf",
FileType.TABLE, replacements));
+
+    replacements.clear();
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new
Path("viewfs:/a")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b")));
+
+    Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf",
FileType.TABLE, replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf",
FileType.TABLE, replacements));
+  }
+
+  @Test
+  public void testSwitchVolumesDifferentTargetDepths() {
+    List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/path1/path2")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new
Path("viewfs:/path1/path2")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/path3")));
+
+    Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf",
FileType.TABLE, replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf",
FileType.TABLE, replacements));
+
+    replacements.clear();
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/path1/path2")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new
Path("viewfs:/path1/path2")));
+    replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/path3")));
+
+    Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+        VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE,
replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf",
FileType.TABLE, replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf",
FileType.TABLE, replacements));
+    Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf",
FileType.TABLE, replacements));
+  }
+
+  @Test
   public void testSame() throws Exception {
     FileSystem fs = FileSystem.getLocal(new Configuration());
 
@@ -133,6 +197,19 @@ public class VolumeUtilTest {
 
   }
 
+  @Test
+  public void testRootTableReplacement() throws IOException {
+    List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+    replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8")));
+    replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9")));
+    
+    FileType ft = FileType.TABLE;
+
+    Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet",
+        VolumeUtil.switchVolume("file:/foo/v1/tables/+r/root_tablet",
+        ft, replacements));
+  }
+
   private void writeFile(FileSystem fs, Path dir, String filename, String data) throws IOException
{
     FSDataOutputStream out = fs.create(new Path(dir, filename));
     try {


Mime
View raw message