hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r392451 - in /lucene/hadoop/trunk/src: java/org/apache/hadoop/fs/LocalFileSystem.java java/org/apache/hadoop/mapred/LocalJobRunner.java test/org/apache/hadoop/fs/TestLocalFileSystem.java
Date Fri, 07 Apr 2006 23:27:44 GMT
Author: cutting
Date: Fri Apr  7 16:27:42 2006
New Revision: 392451

URL: http://svn.apache.org/viewcvs?rev=392451&view=rev
Log:
Fix for HADOOP-125.  Absolute paths are tricky on Windows.  For Hadoop's purposes, consider
things that start with a slash to be absolute.  Also, Hadoop should not change the JVM's CWD.
 All files are now correctly cleaned up for a Nutch crawl, in either local or psuedo-distributed
mode.

Modified:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java?rev=392451&r1=392450&r2=392451&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java Fri Apr  7 16:27:42
2006
@@ -29,7 +29,8 @@
  * @author Mike Cafarella
  *****************************************************************/
 public class LocalFileSystem extends FileSystem {
-    private File workingDir = new File(System.getProperty("user.dir"));
+    private File workingDir
+      = new File(System.getProperty("user.dir")).getAbsoluteFile();
     TreeMap sharedLockDataSet = new TreeMap();
     TreeMap nonsharedLockDataSet = new TreeMap();
     TreeMap lockObjSet = new TreeMap();
@@ -156,7 +157,7 @@
       if (isAbsolute(f)) {
         return f;
       } else {
-        return new File(workingDir, f.toString());
+        return new File(workingDir, f.toString()).getAbsoluteFile();
       }
     }
     
@@ -200,7 +201,9 @@
     }
 
     public boolean isAbsolute(File f) {
-      return f.isAbsolute();
+      return f.isAbsolute() ||
+        f.getPath().startsWith("/") ||
+        f.getPath().startsWith("\\");
     }
 
     public long getLength(File f) throws IOException {
@@ -226,7 +229,6 @@
      */
     public void setWorkingDirectory(File new_dir) {
       workingDir = makeAbsolute(new_dir);
-      System.setProperty("user.dir", workingDir.toString());
     }
     
     public File getWorkingDirectory() {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=392451&r1=392450&r2=392451&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Fri Apr  7 16:27:42
2006
@@ -92,7 +92,6 @@
         job.setNumReduceTasks(1);                 // force a single reduce task
         for (int i = 0; i < splits.length; i++) {
           mapIds.add("map_" + newId());
-          setWorkingDirectory(job, fs);
           MapTask map = new MapTask(file, (String)mapIds.get(i), splits[i]);
           map.setConf(job);
           map_tasks += 1;

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=392451&r1=392450&r2=392451&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java Fri Apr  7
16:27:42 2006
@@ -42,7 +42,7 @@
       
       // create a directory and check for it
       File dir1 = new File("dir1");
-      File dir1Absolute = dir1.getAbsoluteFile();
+      File dir1Absolute = new File(subdirAbsolute, dir1.getPath());
       fileSys.mkdirs(dir1);
       assertTrue(fileSys.isDirectory(dir1));
       assertTrue(fileSys.isDirectory(dir1Absolute));
@@ -55,8 +55,7 @@
       // create files and manipulate them.
       File file1 = new File("file1");
       File file2 = new File("sub/file2");
-      File file2_abs = file2.getAbsoluteFile();
-      assertEquals(file2_abs, new File(subdirAbsolute, file2.getPath()));
+      File file2_abs = new File(subdirAbsolute, file2.getPath());
       writeFile(fileSys, file1);
       fileSys.copyFromLocalFile(file1, file2);
       assertTrue(fileSys.exists(file1));



Mime
View raw message