hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r578607 - in /lucene/hadoop/branches/branch-0.14: CHANGES.txt src/java/org/apache/hadoop/fs/s3/S3FileSystem.java src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
Date Sun, 23 Sep 2007 21:47:09 GMT
Author: tomwhite
Date: Sun Sep 23 14:47:07 2007
New Revision: 578607

URL: http://svn.apache.org/viewvc?rev=578607&view=rev
Log:
HADOOP-1783.  Make S3 FileSystem return Paths fully-qualified with scheme and host.

Modified:
    lucene/hadoop/branches/branch-0.14/CHANGES.txt
    lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
    lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
    lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java

Modified: lucene/hadoop/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/CHANGES.txt?rev=578607&r1=578606&r2=578607&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.14/CHANGES.txt (original)
+++ lucene/hadoop/branches/branch-0.14/CHANGES.txt Sun Sep 23 14:47:07 2007
@@ -1,6 +1,13 @@
 Hadoop Change Log
 
 
+Release 0.14.2 (unreleased changes)
+
+  BUG FIXES
+
+    HADOOP-1783.  Make S3 FileSystem return Paths fully-qualified with
+    scheme and host.  (tomwhite)
+
 
 Release 0.14.1 - 2007-09-04
 

Modified: lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=578607&r1=578606&r2=578607&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
(original)
+++ lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
Sun Sep 23 14:47:07 2007
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
@@ -169,8 +170,11 @@
     } else if (inode.isFile()) {
       return new Path[] { absolutePath };
     } else { // directory
-      Set<Path> paths = store.listSubPaths(absolutePath);
-      return paths.toArray(new Path[0]);
+      ArrayList<Path> ret = new ArrayList<Path>();
+      for (Path p : store.listSubPaths(absolutePath)) {
+        ret.add(p.makeQualified(this));
+      }
+      return ret.toArray(new Path[0]);
     }
   }
 

Modified: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java?rev=578607&r1=578606&r2=578607&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
(original)
+++ lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
Sun Sep 23 14:47:07 2007
@@ -56,7 +56,7 @@
   }
 
   public void deleteINode(Path path) throws IOException {
-    inodes.remove(path);
+    inodes.remove(normalize(path));
   }
 
   public void deleteBlock(Block block) throws IOException {
@@ -64,7 +64,7 @@
   }
 
   public boolean inodeExists(Path path) throws IOException {
-    return inodes.containsKey(path);
+    return inodes.containsKey(normalize(path));
   }
 
   public boolean blockExists(long blockId) throws IOException {
@@ -72,7 +72,7 @@
   }
 
   public INode retrieveINode(Path path) throws IOException {
-    return inodes.get(path);
+    return inodes.get(normalize(path));
   }
 
   public File retrieveBlock(Block block, long byteRangeStart) throws IOException {
@@ -101,10 +101,11 @@
   }
 
   public Set<Path> listSubPaths(Path path) throws IOException {
+    Path normalizedPath = normalize(path);
     // This is inefficient but more than adequate for testing purposes.
     Set<Path> subPaths = new LinkedHashSet<Path>();
-    for (Path p : inodes.tailMap(path).keySet()) {
-      if (path.equals(p.getParent())) {
+    for (Path p : inodes.tailMap(normalizedPath).keySet()) {
+      if (normalizedPath.equals(p.getParent())) {
         subPaths.add(p);
       }
     }
@@ -112,13 +113,14 @@
   }
 
   public Set<Path> listDeepSubPaths(Path path) throws IOException {
-    String pathString = path.toUri().getPath();
+    Path normalizedPath = normalize(path);    
+    String pathString = normalizedPath.toUri().getPath();
     if (!pathString.endsWith("/")) {
       pathString += "/";
     }
     // This is inefficient but more than adequate for testing purposes.
     Set<Path> subPaths = new LinkedHashSet<Path>();
-    for (Path p : inodes.tailMap(path).keySet()) {
+    for (Path p : inodes.tailMap(normalizedPath).keySet()) {
       if (p.toUri().getPath().startsWith(pathString)) {
         subPaths.add(p);
       }
@@ -127,7 +129,7 @@
   }
 
   public void storeINode(Path path, INode inode) throws IOException {
-    inodes.put(path, inode);
+    inodes.put(normalize(path), inode);
   }
 
   public void storeBlock(Block block, File file) throws IOException {
@@ -146,6 +148,13 @@
       }
     }
     blocks.put(block.getId(), out.toByteArray());
+  }
+  
+  private Path normalize(Path path) {
+    if (!path.isAbsolute()) {
+      throw new IllegalArgumentException("Path must be absolute: " + path);
+    }
+    return new Path(path.toUri().getPath());
   }
 
   public void purge() throws IOException {

Modified: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java?rev=578607&r1=578606&r2=578607&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
(original)
+++ lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
Sun Sep 23 14:47:07 2007
@@ -111,17 +111,21 @@
     Path[] paths = s3FileSystem.listPaths(new Path("/"));
 
     assertEquals(1, paths.length);
-    assertEquals(new Path("/test"), paths[0]);
+    assertEquals(new Path("/test").makeQualified(s3FileSystem), paths[0]);
 
     paths = s3FileSystem.listPaths(new Path("/test"));
     assertEquals(1, paths.length);
-    assertEquals(new Path("/test/hadoop"), paths[0]);
+    assertEquals(new Path("/test/hadoop").makeQualified(s3FileSystem),
+        paths[0]);
 
     paths = s3FileSystem.listPaths(new Path("/test/hadoop"));
     assertEquals(3, paths.length);
-    assertEquals(new Path("/test/hadoop/a"), paths[0]);
-    assertEquals(new Path("/test/hadoop/b"), paths[1]);
-    assertEquals(new Path("/test/hadoop/c"), paths[2]);
+    assertEquals(new Path("/test/hadoop/a").makeQualified(s3FileSystem),
+        paths[0]);
+    assertEquals(new Path("/test/hadoop/b").makeQualified(s3FileSystem),
+        paths[1]);
+    assertEquals(new Path("/test/hadoop/c").makeQualified(s3FileSystem),
+        paths[2]);
 
     paths = s3FileSystem.listPaths(new Path("/test/hadoop/a"));
     assertEquals(0, paths.length);



Mime
View raw message