hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r548494 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/DFSClient.java src/java/org/apache/hadoop/fs/LocalDirAllocator.java
Date Mon, 18 Jun 2007 21:45:10 GMT
Author: cutting
Date: Mon Jun 18 14:45:08 2007
New Revision: 548494

URL: http://svn.apache.org/viewvc?view=rev&rev=548494
Log:
HADOOP-1372.  Use LocalDirAllocator for HDFS temporary block files.  Contributed by Dhruba.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=548494&r1=548493&r2=548494
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Jun 18 14:45:08 2007
@@ -160,6 +160,10 @@
      property in "mapred.child.java.opts" configuration property.
      (Enis Soztutar via cutting)
 
+ 51. HADOOP-1372.  Use LocalDirAllocator for HDFS temporary block
+     files, so that disk space, writability, etc. is considered.
+     (Dhruba Borthakur via cutting)
+
 
 Release 0.13.0 - 2007-06-08
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?view=diff&rev=548494&r1=548493&r2=548494
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Mon Jun 18 14:45:08
2007
@@ -58,6 +58,7 @@
   private Configuration conf;
   private long defaultBlockSize;
   private short defaultReplication;
+  private LocalDirAllocator dirAllocator;
     
   /**
    * A map from name -> DFSOutputStream of files that are currently being
@@ -159,6 +160,7 @@
     }
     defaultBlockSize = conf.getLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
     defaultReplication = (short) conf.getInt("dfs.replication", 3);
+    dirAllocator = new LocalDirAllocator("dfs.client.buffer.dir");
     this.leaseChecker = new Daemon(new LeaseChecker());
     this.leaseChecker.start();
   }
@@ -1167,12 +1169,11 @@
     }
         
     private File newBackupFile() throws IOException {
-      File file = conf.getFile("dfs.client.buffer.dir", "tmp");
-      File dir = file.getParentFile();
-      String prefix = "client-" + Math.abs(r.nextLong());
-      String suffix = ".tmp";
-      File result = File.createTempFile(prefix, suffix, dir);
-      result.deleteOnExit();
+      String name = "tmp" + File.separator +
+                     "client-" + Math.abs(r.nextLong());
+      File result = dirAllocator.createTmpFileForWrite(name, 
+                                                       2 * blockSize, 
+                                                       conf);
       return result;
     }
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java?view=diff&rev=548494&r1=548493&r2=548494
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java Mon Jun 18 14:45:08
2007
@@ -137,6 +137,23 @@
     AllocatorPerContext context = obtainContext(contextCfgItemName);
     return context.getLocalPathToRead(pathStr, conf);
   }
+
+  /** Creates a temporary file in the local FS. Pass size as -1 if not known 
+   *  apriori. We round-robin over the set of disks (via the configured dirs) 
+   *  and select the first complete path which has enough space. A file is
+   *  created on this directory. The file is guaranteed to go away when the
+   *  JVM exits.
+   *  @param pathStr prefix for the temporary file
+   *  @param size the size of the file that is going to be written
+   *  @param conf the Configuration object
+   *  @return a unique temporary file
+   *  @throws IOException
+   */
+  public File createTmpFileForWrite(String pathStr, long size, 
+      Configuration conf) throws IOException {
+    AllocatorPerContext context = obtainContext(contextCfgItemName);
+    return context.createTmpFileForWrite(pathStr, size, conf);
+  }
   
   /** Method to check whether a context is valid
    * @param contextCfgItemName
@@ -245,6 +262,25 @@
       //no path found
       throw new DiskErrorException("Could not find any valid local " +
           "directory for " + pathStr);
+    }
+
+    /** Creates a file on the local FS. Pass size as -1 if not known apriori. We
+     *  round-robin over the set of disks (via the configured dirs) and return
+     *  a file on the first path which has enough space. The file is guaranteed
+     *  to go away when the JVM exits.
+     */
+    public File createTmpFileForWrite(String pathStr, long size, 
+        Configuration conf) throws IOException {
+
+      // find an appropriate directory
+      Path path = getLocalPathForWrite(pathStr, size, conf);
+      File dir = new File(path.getParent().toUri().getPath());
+      String prefix = path.getName();
+
+      // create a temp file on this directory
+      File result = File.createTempFile(prefix, null, dir);
+      result.deleteOnExit();
+      return result;
     }
 
     /** Get a path from the local FS for reading. We search through all the



Mime
View raw message