hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r693048 - in /hadoop/core/trunk: ./ src/core/org/apache/hadoop/fs/s3native/ src/test/org/apache/hadoop/fs/s3native/
Date Mon, 08 Sep 2008 10:47:42 GMT
Author: tomwhite
Date: Mon Sep  8 03:47:40 2008
New Revision: 693048

URL: http://svn.apache.org/viewvc?rev=693048&view=rev
Log:
HADOOP-3361.  Implement renames for NativeS3FileSystem.  Contributed by Albert Chern.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Sep  8 03:47:40 2008
@@ -279,6 +279,9 @@
     HADOOP-3498. File globbing alternation should be able to span path
     components. (tomwhite)
 
+    HADOOP-3361. Implement renames for NativeS3FileSystem.
+    (Albert Chern via tomwhite)
+
   OPTIMIZATIONS
 
     HADOOP-3556. Removed lock contention in MD5Hash by changing the 

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
Mon Sep  8 03:47:40 2008
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.fs.s3native;
 
+import static org.apache.hadoop.fs.s3native.NativeS3FileSystem.PATH_DELIMITER;
+
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.File;
@@ -171,17 +173,27 @@
     throws IOException {
     return list(prefix, maxListingLength, null);
   }
-
+  
   public PartialListing list(String prefix, int maxListingLength,
       String priorLastKey) throws IOException {
+
+    return list(prefix, PATH_DELIMITER, maxListingLength, priorLastKey);
+  }
+
+  public PartialListing listAll(String prefix, int maxListingLength,
+      String priorLastKey) throws IOException {
+
+    return list(prefix, null, maxListingLength, priorLastKey);
+  }
+
+  private PartialListing list(String prefix, String delimiter,
+      int maxListingLength, String priorLastKey) throws IOException {
     try {
-      if (prefix.length() > 0 &&
-          !prefix.endsWith(NativeS3FileSystem.PATH_DELIMITER)) {
-        prefix += NativeS3FileSystem.PATH_DELIMITER;
+      if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
+        prefix += PATH_DELIMITER;
       }
       S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
-          prefix, NativeS3FileSystem.PATH_DELIMITER, maxListingLength,
-          priorLastKey);
+          prefix, delimiter, maxListingLength, priorLastKey);
       
       FileMetadata[] fileMetadata =
         new FileMetadata[chunk.getObjects().length];
@@ -210,6 +222,18 @@
       throw new S3Exception(e);
     }
   }
+  
+  public void rename(String srcKey, String dstKey) throws IOException {
+    try {
+      s3Service.moveObject(bucket.getName(), srcKey, bucket.getName(),
+          new S3Object(dstKey), false);
+    } catch (S3ServiceException e) {
+      if (e.getCause() instanceof IOException) {
+        throw (IOException) e.getCause();
+      }
+      throw new S3Exception(e);
+    }
+  }
 
   public void purge(String prefix) throws IOException {
     try {

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java Mon
Sep  8 03:47:40 2008
@@ -44,8 +44,12 @@
   PartialListing list(String prefix, int maxListingLength) throws IOException;
   PartialListing list(String prefix, int maxListingLength, String priorLastKey)
     throws IOException;
+  PartialListing listAll(String prefix, int maxListingLength,
+      String priorLastKey) throws IOException;
   
   void delete(String key) throws IOException;
+
+  void rename(String srcKey, String dstKey) throws IOException;
   
   /**
    * Delete all keys with the given prefix. Used for testing.

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Mon Sep
 8 03:47:40 2008
@@ -451,12 +451,116 @@
     return new FSDataInputStream(new BufferedFSInputStream(
         new NativeS3FsInputStream(store.retrieve(key), key), bufferSize));
   }
+  
+  // rename() and delete() use this method to ensure that the parent directory
+  // of the source does not vanish.
+  private void createParent(Path path) throws IOException {
+      Path parent = path.getParent();
+      if (parent != null) {
+          String key = pathToKey(makeAbsolute(parent));
+          if (key.length() > 0) {
+              store.storeEmptyFile(key + FOLDER_SUFFIX);
+          }
+      }
+  }
+  
+  private boolean existsAndIsFile(Path f) throws IOException {
+    
+    Path absolutePath = makeAbsolute(f);
+    String key = pathToKey(absolutePath);
+    
+    if (key.length() == 0) {
+        return false;
+    }
+    
+    FileMetadata meta = store.retrieveMetadata(key);
+    if (meta != null) {
+        // S3 object with given key exists, so this is a file
+        return true;
+    }
+    
+    if (store.retrieveMetadata(key + FOLDER_SUFFIX) != null) {
+        // Signifies empty directory
+        return false;
+    }
+    
+    PartialListing listing = store.list(key, 1, null);
+    if (listing.getFiles().length > 0 ||
+        listing.getCommonPrefixes().length > 0) {
+        // Non-empty directory
+        return false;
+    }
+    
+    throw new FileNotFoundException(absolutePath +
+        ": No such file or directory");
+}
+
 
   @Override
   public boolean rename(Path src, Path dst) throws IOException {
-    throw new IOException("Not supported");
+
+    String srcKey = pathToKey(makeAbsolute(src));
+
+    if (srcKey.length() == 0) {
+      // Cannot rename root of file system
+      return false;
+    }
+
+    // Figure out the final destination
+    String dstKey;
+    try {
+      boolean dstIsFile = existsAndIsFile(dst);
+      if (dstIsFile) {
+        // Attempting to overwrite a file using rename()
+        return false;
+      } else {
+        // Move to within the existent directory
+        dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
+      }
+    } catch (FileNotFoundException e) {
+      // dst doesn't exist, so we can proceed
+      dstKey = pathToKey(makeAbsolute(dst));
+      try {
+        if (!getFileStatus(dst.getParent()).isDir()) {
+          return false; // parent dst is a file
+        }
+      } catch (FileNotFoundException ex) {
+        return false; // parent dst does not exist
+      }
+    }
+
+    try {
+      boolean srcIsFile = existsAndIsFile(src);
+      if (srcIsFile) {
+        store.rename(srcKey, dstKey);
+      } else {
+        // Move the folder object
+        store.delete(srcKey + FOLDER_SUFFIX);
+        store.storeEmptyFile(dstKey + FOLDER_SUFFIX);
+
+        // Move everything inside the folder
+        String priorLastKey = null;
+        do {
+          PartialListing listing = store.listAll(srcKey, S3_MAX_LISTING_LENGTH,
+              priorLastKey);
+          for (FileMetadata file : listing.getFiles()) {
+            store.rename(file.getKey(), dstKey
+                + file.getKey().substring(srcKey.length()));
+          }
+          priorLastKey = listing.getPriorLastKey();
+        } while (priorLastKey != null);
+      }
+
+      createParent(src);
+      return true;
+
+    } catch (FileNotFoundException e) {
+      // Source file does not exist;
+      return false;
+    }
   }
 
+
   /**
    * Set the working directory to the given directory.
    */

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
Mon Sep  8 03:47:40 2008
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.s3native;
 
+import static org.apache.hadoop.fs.s3native.NativeS3FileSystem.PATH_DELIMITER;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.ByteArrayOutputStream;
@@ -127,22 +128,36 @@
   public PartialListing list(String prefix, int maxListingLength,
       String priorLastKey) throws IOException {
 
-    if (prefix.length() > 0 &&
-        !prefix.endsWith(NativeS3FileSystem.PATH_DELIMITER)) {
-      prefix += NativeS3FileSystem.PATH_DELIMITER;
+    return list(prefix, PATH_DELIMITER, maxListingLength, priorLastKey);
+  }
+
+  public PartialListing listAll(String prefix, int maxListingLength,
+      String priorLastKey) throws IOException {
+
+    return list(prefix, null, maxListingLength, priorLastKey);
+  }
+
+  private PartialListing list(String prefix, String delimiter,
+      int maxListingLength, String priorLastKey) throws IOException {
+
+    if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
+      prefix += PATH_DELIMITER;
     }
     
     List<FileMetadata> metadata = new ArrayList<FileMetadata>();
     SortedSet<String> commonPrefixes = new TreeSet<String>();
     for (String key : dataMap.keySet()) {
       if (key.startsWith(prefix)) {
-        int delimIndex = key.indexOf(NativeS3FileSystem.PATH_DELIMITER,
-            prefix.length());
-        if (delimIndex == -1) {
+        if (delimiter == null) {
           metadata.add(retrieveMetadata(key));
         } else {
-          String commonPrefix = key.substring(0, delimIndex);
-          commonPrefixes.add(commonPrefix);
+          int delimIndex = key.indexOf(delimiter, prefix.length());
+          if (delimIndex == -1) {
+            metadata.add(retrieveMetadata(key));
+          } else {
+            String commonPrefix = key.substring(0, delimIndex);
+            commonPrefixes.add(commonPrefix);
+          }
         }
       }
       if (metadata.size() + commonPrefixes.size() == maxListingLength) {
@@ -159,6 +174,11 @@
     dataMap.remove(key);
   }
 
+  public void rename(String srcKey, String dstKey) throws IOException {
+    metadataMap.put(dstKey, metadataMap.remove(srcKey));
+    dataMap.put(dstKey, dataMap.remove(srcKey));
+  }
+  
   public void purge(String prefix) throws IOException {
     Iterator<Entry<String, FileMetadata>> i =
       metadataMap.entrySet().iterator();

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java?rev=693048&r1=693047&r2=693048&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
Mon Sep  8 03:47:40 2008
@@ -47,11 +47,6 @@
     super.tearDown();
   }
   
-  @Override
-  protected boolean renameSupported() {
-    return false;
-  }
-  
   public void testListStatusForRoot() throws Exception {
     Path testDir = path("/test");
     assertTrue(fs.mkdirs(testDir));



Mime
View raw message