hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r541070 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/mapred/
Date Wed, 23 May 2007 20:53:02 GMT
Author: cutting
Date: Wed May 23 13:53:01 2007
New Revision: 541070

URL: http://svn.apache.org/viewvc?view=rev&rev=541070
Log:
HADOOP-1226.  Change HDFS so that paths it returns are always fully qualified.  Contributed
by Dhruba.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed May 23 13:53:01 2007
@@ -60,6 +60,9 @@
  19. HADOOP-1413.  Add example program that uses Knuth's dancing links
      algorithm to solve pentomino problems.  (omalley via cutting)
 
+ 20. HADOOP-1226.  Change HDFS so that paths it returns are always
+     fully qualified.  (Dhruba Borthakur via cutting)
+
 
 Branch 0.13 (unreleased changes)
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DfsPath.java Wed May 23 13:53:01 2007
@@ -18,7 +18,7 @@
 package org.apache.hadoop.dfs;
 
 import org.apache.hadoop.fs.Path;
-
+import org.apache.hadoop.fs.FileSystem;
 
 /** 
  * DfsPath is a Path that's been annotated with some extra information.
@@ -28,8 +28,11 @@
 class DfsPath extends Path {
   DFSFileInfo info;
 
-  public DfsPath(DFSFileInfo info) {
-    super(info.getPath());
+  /**
+   * DfsPaths are fully qualified with scheme and authority.
+   */
+  public DfsPath(DFSFileInfo info, FileSystem fs) {
+    super((new Path(info.getPath())).makeQualified(fs).toString());
     this.info = info;
   }
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Wed May
23 13:53:01 2007
@@ -218,7 +218,7 @@
       } else {
         Path results[] = new DfsPath[info.length];
         for (int i = 0; i < info.length; i++) {
-          results[i] = new DfsPath(info[i]);
+          results[i] = new DfsPath(info[i], this);
         }
         return results;
       }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Wed May 23 13:53:01
2007
@@ -181,32 +181,7 @@
   /** Make sure that a path specifies a FileSystem. */
   public Path makeQualified(Path path) {
     checkPath(path);
-
-    if (!path.isAbsolute())
-      path = new Path(getWorkingDirectory(), path);
-
-    URI pathUri = path.toUri();
-    URI fsUri = getUri();
-      
-    String scheme = pathUri.getScheme();
-    String authority = pathUri.getAuthority();
-
-    if (scheme != null &&
-        (authority != null || fsUri.getAuthority() == null))
-      return path;
-
-    if (scheme == null) {
-      scheme = fsUri.getScheme();
-    }
-
-    if (authority == null) {
-      authority = fsUri.getAuthority();
-      if (authority == null) {
-        authority = "";
-      }
-    }
-
-    return new Path(scheme+":"+"//"+authority + pathUri.getPath());
+    return path.makeQualified(this);
   }
     
   ///////////////////////////////////////////////////////////////

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FsShell.java Wed May 23 13:53:01 2007
@@ -329,7 +329,7 @@
       }
       for (int i = 0; i < items.length; i++) {
         Path cur = items[i];
-        System.out.println(cur + "\t" 
+        System.out.println(cur.toUri().getPath() + "\t" 
                            + (fs.isDirectory(cur) ? 
                               "<dir>" : 
                               ("<r " + fs.getReplication(cur) 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java Wed May 23 13:53:01 2007
@@ -264,5 +264,35 @@
     }
     return depth;
   }
-  
+
+  /** Returns a qualified path object. */
+  public Path makeQualified(FileSystem fs) {
+    Path path = this;
+    if (!isAbsolute()) {
+      path = new Path(fs.getWorkingDirectory(), this);
+    }
+
+    URI pathUri = path.toUri();
+    URI fsUri = fs.getUri();
+      
+    String scheme = pathUri.getScheme();
+    String authority = pathUri.getAuthority();
+
+    if (scheme != null &&
+        (authority != null || fsUri.getAuthority() == null))
+      return path;
+
+    if (scheme == null) {
+      scheme = fsUri.getScheme();
+    }
+
+    if (authority == null) {
+      authority = fsUri.getAuthority();
+      if (authority == null) {
+        authority = "";
+      }
+    }
+
+    return new Path(scheme+":"+"//"+authority + pathUri.getPath());
+  }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java?view=diff&rev=541070&r1=541069&r2=541070
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java Wed May 23 13:53:01 2007
@@ -159,7 +159,13 @@
   public String toString() { return taskId; }
 
   private Path getTaskOutputPath(JobConf conf) {
-    return new Path(conf.getOutputPath(), ("_" + taskId));
+    Path p = new Path(conf.getOutputPath(), ("_" + taskId));
+    try {
+      return p.makeQualified(FileSystem.get(conf));
+    } catch (IOException ie) {
+      LOG.warn(StringUtils.stringifyException(ie));
+      return p;
+    }
   }
   
   /**



Mime
View raw message