hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r639052 - in /hadoop/core/trunk: ./ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/ src/test/org/apache/hadoop/dfs/ src/test/org/apache/hadoop/fs/
Date Wed, 19 Mar 2008 22:59:07 GMT
Author: dhruba
Date: Wed Mar 19 15:58:52 2008
New Revision: 639052

URL: http://svn.apache.org/viewvc?rev=639052&view=rev
Log:
HADOOP-3003. FileSystem cache key is updated after a
FileSystem object is created. (Tsz Wo (Nicholas), SZE via dhruba)


Added:
    hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedFileSystem.java   (with
props)
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=639052&r1=639051&r2=639052&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Mar 19 15:58:52 2008
@@ -306,6 +306,9 @@
     Also makes the _temporary a constant in MRConstants.java.
     (Amareshwari Sriramadasu via ddas)
 
+    HADOOP-3003. FileSystem cache key is updated after a 
+    FileSystem object is created. (Tsz Wo (Nicholas), SZE via dhruba)
+
 Release 0.16.1 - 2008-03-13
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?rev=639052&r1=639051&r2=639052&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Wed Mar 19
15:58:52 2008
@@ -221,9 +221,13 @@
     return dfs.mkdirs(getPathName(f), permission);
   }
 
-  public void close() throws IOException {
-    super.close();
-    dfs.close();
+  /** {@inheritDoc} */
+  public synchronized void close() throws IOException {
+    try {
+      dfs.close();
+    } finally {
+      super.close();
+    }
   }
 
   public String toString() {

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=639052&r1=639051&r2=639052&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Wed Mar 19 15:58:52 2008
@@ -1157,7 +1157,7 @@
    * release any held locks.
    */
   public void close() throws IOException {
-    CACHE.remove(new Cache.Key(getUri(), getConf()));
+    CACHE.remove(new Cache.Key(this), this);
   }
 
   /** Return the total size of all files in the filesystem.*/
@@ -1258,39 +1258,51 @@
 
   /** Caching FileSystem objects */
   private static class Cache {
-    final Map<Key, FsRef> map = new HashMap<Key, FsRef>();
+    private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>();
 
     synchronized FileSystem get(URI uri, Configuration conf) throws IOException{
-      Key key = new Key(uri, conf);
-      FsRef ref = map.get(key);
-      FileSystem fs = ref == null? null: ref.get();
+      FileSystem fs = map.get(new Key(uri, conf));
       if (fs == null) {
         if (map.isEmpty() && !clientFinalizer.isAlive()) {
           Runtime.getRuntime().addShutdownHook(clientFinalizer);
         }
 
         fs = createFileSystem(uri, conf);
-        map.put(key, new FsRef(fs, key));
+        map.put(new Key(fs), fs);
       }
       return fs;
     }
 
-    synchronized FsRef remove(Key key) {
-      FsRef ref = map.remove(key);
-      if (map.isEmpty() && !clientFinalizer.isAlive()) {
-        if (!Runtime.getRuntime().removeShutdownHook(clientFinalizer)) {
-          LOG.info("Could not cancel cleanup thread, though no " +
-                   "FileSystems are open");
+    synchronized void remove(Key key, FileSystem fs) {
+      if (map.containsKey(key) && fs == map.get(key)) {
+        map.remove(key);
+        if (map.isEmpty() && !clientFinalizer.isAlive()) {
+          if (!Runtime.getRuntime().removeShutdownHook(clientFinalizer)) {
+            LOG.info("Could not cancel cleanup thread, though no " +
+                     "FileSystems are open");
+          }
         }
       }
-      return ref;
     }
 
     synchronized void closeAll() throws IOException {
       List<IOException> exceptions = new ArrayList<IOException>();
-      for(FsRef ref : new ArrayList<FsRef>(map.values())) {
-        FileSystem fs = ref.get();
+      for(; !map.isEmpty(); ) {
+        Map.Entry<Key, FileSystem> e = map.entrySet().iterator().next();
+        final Key key = e.getKey();
+        final FileSystem fs = e.getValue();
+
+        //remove from cache
+        remove(key, fs);
+
         if (fs != null) {
+          //check consistency
+          if (!new Key(fs).equals(key)) {
+            exceptions.add(new IOException(fs.getClass().getSimpleName()
+                + "(=" + fs + ") and " + key.getClass().getSimpleName()
+                + "(=" + key + ") do not match."));
+          }
+
           try {
             fs.close();
           }
@@ -1298,9 +1310,6 @@
             exceptions.add(ioe);
           }
         }
-        else {
-          remove(ref.key);
-        }        
       }
 
       if (!exceptions.isEmpty()) {
@@ -1308,25 +1317,16 @@
       }
     }
 
-    /** Reference of FileSystem which contains the cache key */
-    private static class FsRef {
-      final FileSystem fs;
-      final Key key;
-      
-      FsRef(FileSystem fs, Key key) {
-        this.fs = fs;
-        this.key = key;
-      }
-
-      FileSystem get() {return fs;}
-    }
-
     /** FileSystem.Cache.Key */
     private static class Key {
       final String scheme;
       final String authority;
       final String username;
 
+      Key(FileSystem fs) throws IOException {
+        this(fs.getUri(), fs.getConf());
+      }
+
       Key(URI uri, Configuration conf) throws IOException {
         scheme = uri.getScheme();
         authority = uri.getAuthority();
@@ -1355,6 +1355,11 @@
                  && isEqual(this.username, that.username);
         }
         return false;        
+      }
+
+      /** {@inheritDoc} */
+      public String toString() {
+        return username + "@" + scheme + "://" + authority;        
       }
     }
   }

Added: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedFileSystem.java?rev=639052&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedFileSystem.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedFileSystem.java Wed Mar
19 15:58:52 2008
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.dfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+public class TestDistributedFileSystem extends junit.framework.TestCase {
+  public void testFileSystemCloseAll() throws Exception {
+    Configuration conf = new Configuration();
+    MiniDFSCluster cluster = new MiniDFSCluster(conf, 0, true, null);
+    String address = conf.get("fs.default.name");
+
+    try {
+      FileSystem.closeAll();
+
+      conf = new Configuration();
+      conf.set("fs.default.name", address);
+      FileSystem.get(conf);
+      FileSystem.get(conf);
+      FileSystem.closeAll();
+    }
+    finally {
+      if (cluster != null) {cluster.shutdown();}
+    }
+  }
+}
\ No newline at end of file

Propchange: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedFileSystem.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedFileSystem.java
------------------------------------------------------------------------------
    svn:keywords = Id Revision HeadURL

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java?rev=639052&r1=639051&r2=639052&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java Wed Mar 19 15:58:52
2008
@@ -42,7 +42,6 @@
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.lib.LongSumReducer;
 import org.apache.hadoop.security.UnixUserGroupInformation;
-import org.apache.hadoop.security.UserGroupInformation;
 
 public class TestFileSystem extends TestCase {
   private static final Log LOG = FileSystem.LOG;
@@ -469,8 +468,18 @@
   }
 
   public void testFsCache() throws Exception {
-    Configuration c1 = createConf4Testing("foo");
-    Configuration c2 = createConf4Testing("bar");
-    assertFalse(FileSystem.get(c1) == FileSystem.get(c2));
+    long now = System.currentTimeMillis();
+    Configuration[] conf = {new Configuration(),
+        createConf4Testing("foo" + now), createConf4Testing("bar" + now)};
+    FileSystem[] fs = new FileSystem[conf.length];
+
+    for(int i = 0; i < conf.length; i++) {
+      fs[i] = FileSystem.get(conf[i]);
+      assertEquals(fs[i], FileSystem.get(conf[i]));
+      for(int j = 0; j < i; j++) {
+        assertFalse(fs[j] == fs[i]);
+      }
+    }
+    FileSystem.closeAll();
   }
 }



Mime
View raw message