hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r390734 - in /lucene/hadoop/trunk/src/java/org/apache/hadoop: conf/Configuration.java dfs/DFSFile.java dfs/DistributedFileSystem.java fs/FileSystem.java fs/LocalFileSystem.java mapred/InputFormatBase.java mapred/JobConf.java
Date Sat, 01 Apr 2006 19:19:40 GMT
Author: cutting
Date: Sat Apr  1 11:19:38 2006
New Revision: 390734

URL: http://svn.apache.org/viewcvs?rev=390734&view=rev
Log:
More fixes to get working directory to work on Windows.  Long-term we should stop using java.io.File
for our abstract file paths.  On Windows, 'new File(/foo/bar).isAbsolute()' returns false,
which caused lots of problems.  So I have added a FileSystem.isAbsolute() method that we use
internally.  I also added a few more .getAbsoluteFile() calls to convert paths into absolute
paths so that LocalFileSystem works correctly.  Finally I took advantage of file status information
cached in DFSFile, eliminating some namenode RPCs.

Modified:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java Sat Apr  1 11:19:38
2006
@@ -273,10 +273,10 @@
     int hashCode = path.hashCode();
     for (int i = 0; i < dirs.length; i++) {  // try each local dir
       int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
-      File file = new File(dirs[index], path);
+      File file = new File(dirs[index], path).getAbsoluteFile();
       File dir = file.getParentFile();
       if (dir.exists() || dir.mkdirs()) {
-        return file.getAbsoluteFile();
+        return file;
       }
     }
     throw new IOException("No valid local directories in property: "+dirsProp);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSFile.java Sat Apr  1 11:19:38 2006
@@ -57,6 +57,9 @@
     public boolean isHidden() {
         return false;
     }
+    public boolean isAbsolute() {
+        return true;
+    }
 
     /**
      * We need to reimplement some of them

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Sat Apr
 1 11:19:38 2006
@@ -54,7 +54,7 @@
     }
     
     private File makeAbsolute(File f) {
-      if (f.isAbsolute()) {
+      if (isAbsolute(f)) {
         return f;
       } else {
         return new File(workingDir, f.getPath());
@@ -102,10 +102,23 @@
     }
 
     public boolean isDirectory(File f) throws IOException {
+        if (f instanceof DFSFile) {
+          return ((DFSFile)f).isDirectory();
+        }
         return dfs.isDirectory(getPath(f));
     }
 
+    public boolean isAbsolute(File f) {
+      return f.isAbsolute() ||
+        f.getPath().startsWith("/") ||
+        f.getPath().startsWith("\\");
+    }
+
     public long getLength(File f) throws IOException {
+        if (f instanceof DFSFile) {
+          return ((DFSFile)f).length();
+        }
+
         DFSFileInfo info[] = dfs.listFiles(getPath(f));
         return info[0].getLen();
     }
@@ -282,7 +295,7 @@
         path.append(DFSFile.DFS_FILE_SEPARATOR);
         path.append(l.get(i));
       }
-      if (f.isAbsolute() && path.length() == 0) {
+      if (isAbsolute(f) && path.length() == 0) {
         path.append(DFSFile.DFS_FILE_SEPARATOR);
       }
       return path.toString();

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Sat Apr  1 11:19:38
2006
@@ -265,6 +265,9 @@
         }
     }
     
+    /** True iff the named path is absolute. */
+    public abstract boolean isAbsolute(File f);
+
     /** The number of bytes in a file. */
     public abstract long getLength(File f) throws IOException;
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalFileSystem.java Sat Apr  1 11:19:38
2006
@@ -153,7 +153,7 @@
     }
 
     private File makeAbsolute(File f) {
-      if (f.isAbsolute()) {
+      if (isAbsolute(f)) {
         return f;
       } else {
         return new File(workingDir, f.toString());
@@ -197,6 +197,10 @@
     public boolean isDirectory(File f) throws IOException {
         f = makeAbsolute(f);
         return f.isDirectory();
+    }
+
+    public boolean isAbsolute(File f) {
+      return f.isAbsolute();
     }
 
     public long getLength(File f) throws IOException {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InputFormatBase.java Sat Apr  1
11:19:38 2006
@@ -65,7 +65,7 @@
     for (int i = 0; i < dirs.length; i++) {
       // if it is relative, make it absolute using the directory from the 
       // JobConf
-      if (workDir != null && !dirs[i].isAbsolute()) {
+      if (workDir != null && !fs.isAbsolute(dirs[i])) {
         dirs[i] = new File(workDir, dirs[i].toString());
       }
       File[] dir = fs.listFiles(dirs[i]);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=390734&r1=390733&r2=390734&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Sat Apr  1 11:19:38
2006
@@ -99,8 +99,8 @@
   public void setJar(String jar) { set("mapred.jar", jar); }
 
   public File getSystemDir() {
-    return new File(get("mapred.system.dir",
-                                        "/tmp/hadoop/mapred/system"));
+    return new File(get("mapred.system.dir", "/tmp/hadoop/mapred/system"))
+      .getAbsoluteFile();
   }
 
   public String[] getLocalDirs() throws IOException {



Mime
View raw message