hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r980523 - in /hadoop/common/trunk: CHANGES.txt src/java/org/apache/hadoop/fs/FileSystem.java src/test/core/org/apache/hadoop/fs/TestFileSystemCaching.java
Date Thu, 29 Jul 2010 17:58:55 GMT
Author: szetszwo
Date: Thu Jul 29 17:58:55 2010
New Revision: 980523

URL: http://svn.apache.org/viewvc?rev=980523&view=rev
Log:
HADOOP-6888. Add a new FileSystem API closeAllForUGI(..) for closing all file systems associated
with a particular UGI.  Contributed by Kan Zhang

Modified:
    hadoop/common/trunk/CHANGES.txt
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFileSystemCaching.java

Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=980523&r1=980522&r2=980523&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Thu Jul 29 17:58:55 2010
@@ -26,6 +26,9 @@ Trunk (unreleased changes)
     input path if recursive is true. Block locations are returned together
     with each file's status. (hairong)
 
+    HADOOP-6888. Add a new FileSystem API closeAllForUGI(..) for closing all
+    file systems associated with a particular UGI.  (Kan Zhang via szetszwo)
+
   IMPROVEMENTS
 
     HADOOP-6644. util.Shell getGROUPS_FOR_USER_COMMAND method name 

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=980523&r1=980522&r2=980523&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Thu Jul 29 17:58:55
2010
@@ -311,6 +311,17 @@ public abstract class FileSystem extends
     CACHE.closeAll();
   }
 
+  /**
+   * Close all cached filesystems for a given UGI. Be sure those filesystems 
+   * are not used anymore.
+   * @param ugi
+   * @throws IOException
+   */
+  public static void closeAllForUGI(UserGroupInformation ugi) 
+  throws IOException {
+    CACHE.closeAll(ugi);
+  }
+
   /** Make sure that a path specifies a FileSystem. */
   public Path makeQualified(Path path) {
     checkPath(path);
@@ -1816,6 +1827,32 @@ public abstract class FileSystem extends
       }
     }
 
+    synchronized void closeAll(UserGroupInformation ugi) throws IOException {
+      List<FileSystem> targetFSList = new ArrayList<FileSystem>();
+      //Make a pass over the list and collect the filesystems to close
+      //we cannot close inline since close() removes the entry from the Map
+      for (Map.Entry<Key, FileSystem> entry : map.entrySet()) {
+        final Key key = entry.getKey();
+        final FileSystem fs = entry.getValue();
+        if (ugi.equals(key.ugi) && fs != null) {
+          targetFSList.add(fs);   
+        }
+      }
+      List<IOException> exceptions = new ArrayList<IOException>();
+      //now make a pass over the target list and close each
+      for (FileSystem fs : targetFSList) {
+        try {
+          fs.close();
+        }
+        catch(IOException ioe) {
+          exceptions.add(ioe);
+        }
+      }
+      if (!exceptions.isEmpty()) {
+        throw MultipleIOException.createIOException(exceptions);
+      }
+    }
+
     /** FileSystem.Cache.Key */
     static class Key {
       final String scheme;

Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFileSystemCaching.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFileSystemCaching.java?rev=980523&r1=980522&r2=980523&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFileSystemCaching.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFileSystemCaching.java Thu
Jul 29 17:58:55 2010
@@ -179,4 +179,33 @@ public class TestFileSystemCaching {
     fs1.close();
     fs2.close();
   }
+  
+  @Test
+  public void testCloseAllForUGI() throws Exception {
+    final Configuration conf = new Configuration();
+    conf.set("fs.cachedfile.impl", conf.get("fs.file.impl"));
+    UserGroupInformation ugiA = UserGroupInformation.createRemoteUser("foo");
+    FileSystem fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
+      public FileSystem run() throws Exception {
+        return FileSystem.get(new URI("cachedfile://a"), conf);
+      }
+    });
+    //Now we should get the cached filesystem
+    FileSystem fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
+      public FileSystem run() throws Exception {
+        return FileSystem.get(new URI("cachedfile://a"), conf);
+      }
+    });
+    assertSame(fsA, fsA1);
+    
+    FileSystem.closeAllForUGI(ugiA);
+    
+    //Now we should get a different (newly created) filesystem
+    fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
+      public FileSystem run() throws Exception {
+        return FileSystem.get(new URI("cachedfile://a"), conf);
+      }
+    });
+    assertNotSame(fsA, fsA1);
+  }
 }



Mime
View raw message