hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r594452 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/DFSClient.java src/java/org/apache/hadoop/fs/FileSystem.java
Date Tue, 13 Nov 2007 08:17:18 GMT
Author: dhruba
Date: Tue Nov 13 00:17:17 2007
New Revision: 594452

URL: http://svn.apache.org/viewvc?rev=594452&view=rev
Log:
HADOOP-1130. The FileSystem.closeAll() method closes all existing
DFSClients.  (Chris Douglas via dhruba)


Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=594452&r1=594451&r2=594452&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue Nov 13 00:17:17 2007
@@ -94,6 +94,9 @@
     HADOOP-2121. Cleanup DFSOutputStream when the stream encountered errors
     when Datanodes became full.  (Raghu Angadi via dhruba)
 
+    HADOOP-1130. The FileSystem.closeAll() method closes all existing
+    DFSClients.  (Chris Douglas via dhruba)
+
 Release 0.15.1 -
 
   BUG FIXES

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?rev=594452&r1=594451&r2=594452&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Tue Nov 13 00:17:17
2007
@@ -72,38 +72,6 @@
   private TreeMap<String, OutputStream> pendingCreates =
     new TreeMap<String, OutputStream>();
     
-  /**
-   * A class to track the list of DFS clients, so that they can be closed
-   * on exit.
-   */
-  private static class ClientFinalizer extends Thread {
-    private List<DFSClient> clients = new ArrayList<DFSClient>();
-
-    public synchronized void addClient(DFSClient client) {
-      clients.add(client);
-    }
-
-    @Override
-    public synchronized void run() {
-      for (DFSClient client : clients) {
-        if (client.running) {
-          try {
-            client.close();
-          } catch (IOException ie) {
-            System.err.println("Error closing client");
-            ie.printStackTrace();
-          }
-        }
-      }
-    }
-  }
-
-  // add a cleanup thread
-  private static ClientFinalizer clientFinalizer = new ClientFinalizer();
-  static {
-    Runtime.getRuntime().addShutdownHook(clientFinalizer);
-  }
-
   private static ClientProtocol createNamenode(
       InetSocketAddress nameNodeAddr, Configuration conf)
     throws IOException {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=594452&r1=594451&r2=594452&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Tue Nov 13 00:17:17
2007
@@ -145,6 +145,9 @@
 
     Map<String,FileSystem> authorityToFs = CACHE.get(scheme);
     if (authorityToFs == null) {
+      if (CACHE.isEmpty()) {
+        Runtime.getRuntime().addShutdownHook(clientFinalizer);
+      }
       authorityToFs = new HashMap<String,FileSystem>();
       CACHE.put(scheme, authorityToFs);
     }
@@ -163,16 +166,29 @@
     return fs;
   }
 
+  private static class ClientFinalizer extends Thread {
+    public synchronized void run() {
+      try {
+        FileSystem.closeAll();
+      } catch (IOException e) {
+        LOG.info("FileSystem.closeAll() threw an exception:\n" + e);
+      }
+    }
+  }
+  private static final ClientFinalizer clientFinalizer = new ClientFinalizer();
+
   /**
    * Close all cached filesystems. Be sure those filesystems are not
    * used anymore.
    * 
    * @throws IOException
    */
-  public static synchronized void closeAll() throws IOException{
-    for(Map<String, FileSystem>  fss : CACHE.values()){
-      for(FileSystem fs : fss.values()){
-        fs.close();
+  public static synchronized void closeAll() throws IOException {
+    Set<String> scheme = new HashSet<String>(CACHE.keySet());
+    for (String s : scheme) {
+      Set<String> authority = new HashSet<String>(CACHE.get(s).keySet());
+      for (String a : authority) {
+        CACHE.get(s).get(a).close();
       }
     }
   }
@@ -895,6 +911,15 @@
       Map<String,FileSystem> authorityToFs = CACHE.get(uri.getScheme());
       if (authorityToFs != null) {
         authorityToFs.remove(uri.getAuthority());
+        if (authorityToFs.isEmpty()) {
+          CACHE.remove(uri.getScheme());
+          if (CACHE.isEmpty() && !clientFinalizer.isAlive()) {
+            if (!Runtime.getRuntime().removeShutdownHook(clientFinalizer)) {
+              LOG.info("Could not cancel cleanup thread, though no " +
+                       "FileSystems are open");
+            }
+          }
+        }
       }
     }
   }



Mime
View raw message