hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r659005 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/DistributedFileSystem.java src/java/org/apache/hadoop/fs/FileSystem.java src/test/org/apache/hadoop/dfs/TestFileCreation.java
Date Thu, 22 May 2008 05:03:11 GMT
Author: dhruba
Date: Wed May 21 22:03:10 2008
New Revision: 659005

URL: http://svn.apache.org/viewvc?rev=659005&view=rev
Log:
HADOOP-3400. A new API FileSystem.deleteOnExit() that facilitates
handling of temporary files in HDFS. (dhruba)


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
    hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=659005&r1=659004&r2=659005&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed May 21 22:03:10 2008
@@ -94,6 +94,9 @@
     HADOOP-3336. Direct a subset of annotated FSNamesystem calls for audit
     logging. (cdouglas)
 
+    HADOOP-3400. A new API FileSystem.deleteOnExit() that facilitates
+    handling of temporary files in HDFS. (dhruba)
+
   IMPROVEMENTS
    
     HADOOP-2928. Remove deprecated FileSystem.getContentLength().

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?rev=659005&r1=659004&r2=659005&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Wed May 21
22:03:10 2008
@@ -194,6 +194,7 @@
   /** {@inheritDoc} */
   public void close() throws IOException {
     try {
+      super.processDeleteOnExit();
       dfs.close();
     } finally {
       super.close();

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=659005&r1=659004&r2=659005&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Wed May 21 22:03:10 2008
@@ -69,6 +69,12 @@
    * The statistics for this file system.
    */
   protected final Statistics statistics;
+
+  /**
+   * A cache of files that should be deleted when filsystem is closed
+   * or the JVM is exited.
+   */
+  private Set<Path> deleteOnExit = new TreeSet<Path>();
   
   /**
    * Parse the cmd-line args, starting at i.  Remove consumed args
@@ -591,6 +597,41 @@
    * @throws IOException
    */
   public abstract boolean delete(Path f, boolean recursive) throws IOException;
+
+  /** Mark a path to be deleted when FileSystem is closed or JVM exits.
+   *  The path has to exist in the file system.
+   *
+   * @param f the path to delete.
+   * @return  true if deleteOnExit is successful, otherwise false.
+   * @throws IOException
+   */
+  public boolean deleteOnExit(Path f) throws IOException {
+    if (!exists(f)) {
+      return false;
+    }
+    synchronized (deleteOnExit) {
+      deleteOnExit.add(f);
+    }
+    return true;
+  }
+
+  // Delete all files that were marked as delete-on-exit. This recursively
+  // deletes all files in the specified paths.
+  //
+  protected void processDeleteOnExit() {
+    synchronized (deleteOnExit) {
+      for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
+        Path path = iter.next();
+        try {
+          delete(path, true);
+        }
+        catch (IOException e) {
+          LOG.info("Ignoring failure to deleteOnExit for path " + path);
+        }
+        iter.remove();
+      }
+    }
+  }
   
   /** Check if exists.
    * @param f source file
@@ -1172,6 +1213,8 @@
    * release any held locks.
    */
   public void close() throws IOException {
+    // delete all files that were marked as delete-on-exit.
+    processDeleteOnExit();
     CACHE.remove(new Cache.Key(this), this);
   }
 

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java?rev=659005&r1=659004&r2=659005&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java Wed May 21 22:03:10
2008
@@ -203,6 +203,71 @@
   }
 
   /**
+   * Test deleteOnExit
+   */
+  public void testDeleteOnExit() throws IOException {
+    Configuration conf = new Configuration();
+    if (simulatedStorage) {
+      conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
+    }
+    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
+    FileSystem fs = cluster.getFileSystem();
+    FileSystem localfs = FileSystem.getLocal(conf);
+
+    try {
+
+      // Creates files in HDFS and local file system.
+      //
+      Path file1 = new Path("filestatus.dat");
+      Path file2 = new Path("filestatus2.dat");
+      Path file3 = new Path("filestatus3.dat");
+      FSDataOutputStream stm1 = createFile(fs, file1, 1);
+      FSDataOutputStream stm2 = createFile(fs, file2, 1);
+      FSDataOutputStream stm3 = createFile(localfs, file3, 1);
+      System.out.println("DeleteOnExit: Created files.");
+
+      // write to files and close. Purposely, do not close file2.
+      writeFile(stm1);
+      writeFile(stm3);
+      stm1.close();
+      stm3.close();
+
+      // set delete on exit flag on files.
+      fs.deleteOnExit(file1);
+      fs.deleteOnExit(file2);
+      localfs.deleteOnExit(file3);
+
+      // close the file system. This should make the above files
+      // disappear.
+      fs.close();
+      localfs.close();
+      fs = null;
+      localfs = null;
+
+      // reopen file system and verify that file does not exist.
+      fs = cluster.getFileSystem();
+      localfs = FileSystem.getLocal(conf);
+
+      assertTrue(file1 + " still exists inspite of deletOnExit set.",
+                 !fs.exists(file1));
+      assertTrue(file2 + " still exists inspite of deletOnExit set.",
+                 !fs.exists(file2));
+      assertTrue(file3 + " still exists inspite of deletOnExit set.",
+                 !localfs.exists(file3));
+      System.out.println("DeleteOnExit successful.");
+
+    } finally {
+      if (fs != null) {
+        fs.close();
+      }
+      if (localfs != null) {
+        localfs.close();
+      }
+      cluster.shutdown();
+    }
+  }
+
+  /**
    * Test that file data does not become corrupted even in the face of errors.
    */
   public void testFileCreationError1() throws IOException {



Mime
View raw message