hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hair...@apache.org
Subject svn commit: r723838 - in /hadoop/core/branches/branch-0.19: ./ CHANGES.txt src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
Date Fri, 05 Dec 2008 19:57:59 GMT
Author: hairong
Date: Fri Dec  5 11:57:58 2008
New Revision: 723838

URL: http://svn.apache.org/viewvc?rev=723838&view=rev
Log:
Merge -r 723830:723831 from trunk to move the change log of HADOOP-4717 into branch 0.19

Modified:
    hadoop/core/branches/branch-0.19/   (props changed)
    hadoop/core/branches/branch-0.19/CHANGES.txt   (contents, props changed)
    hadoop/core/branches/branch-0.19/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
    hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java
    hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java

Propchange: hadoop/core/branches/branch-0.19/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec  5 11:57:58 2008
@@ -1 +1 @@
-/hadoop/core/trunk:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,709040,709303,712881,713888,720602,723013,723460
+/hadoop/core/trunk:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,709040,709303,712881,713888,720602,723013,723460,723831

Modified: hadoop/core/branches/branch-0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/CHANGES.txt?rev=723838&r1=723837&r2=723838&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.19/CHANGES.txt Fri Dec  5 11:57:58 2008
@@ -1062,6 +1062,9 @@
 
     HADOOP-4746. Job output directory should be normalized. (hairong)
 
+    HADOOP-4717. Removal of default port# in NameNode.getUri() causes a
+    map/reduce job failed to prompt temporary output. (hairong)
+
 Release 0.18.2 - 2008-11-03
 
   BUG FIXES

Propchange: hadoop/core/branches/branch-0.19/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec  5 11:57:58 2008
@@ -1 +1 @@
-/hadoop/core/trunk/CHANGES.txt:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,708723,709040,709303,711717,712881,713888,720602,723013,723460
+/hadoop/core/trunk/CHANGES.txt:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,708723,709040,709303,711717,712881,713888,720602,723013,723460,723831

Modified: hadoop/core/branches/branch-0.19/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=723838&r1=723837&r2=723838&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
(original)
+++ hadoop/core/branches/branch-0.19/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
Fri Dec  5 11:57:58 2008
@@ -92,6 +92,24 @@
     super.checkPath(path);
   }
 
+  /** Normalize paths that explicitly specify the default port. */
+  public Path makeQualified(Path path) {
+    URI thisUri = this.getUri();
+    URI thatUri = path.toUri();
+    String thatAuthority = thatUri.getAuthority();
+    if (thatUri.getScheme() != null
+        && thatUri.getScheme().equalsIgnoreCase(thisUri.getScheme())
+        && thatUri.getPort() == NameNode.DEFAULT_PORT
+        && thisUri.getPort() == -1
+        && thatAuthority.substring(0,thatAuthority.indexOf(":"))
+        .equalsIgnoreCase(thisUri.getAuthority())) {
+      path = new Path(thisUri.getScheme(), thisUri.getAuthority(),
+                      thatUri.getPath());
+    }
+    return super.makeQualified(path);
+  }
+
+
   public Path getWorkingDirectory() {
     return workingDir;
   }

Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java?rev=723838&r1=723837&r2=723838&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java
(original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputCommitter.java
Fri Dec  5 11:57:58 2008
@@ -141,8 +141,13 @@
   }
 
   private Path getFinalPath(Path jobOutputDir, Path taskOutput, 
-                            Path taskOutputPath) {
-    URI relativePath = taskOutputPath.toUri().relativize(taskOutput.toUri());
+                            Path taskOutputPath) throws IOException {
+    URI taskOutputUri = taskOutput.toUri();
+    URI relativePath = taskOutputPath.toUri().relativize(taskOutputUri);
+    if (taskOutputUri == relativePath) {//taskOutputPath is not a parent of taskOutput
+      throw new IOException("Can not get the relative path: base = " + 
+          taskOutputPath + " child = " + taskOutput);
+    }
     if (relativePath.getPath().length() > 0) {
       return new Path(jobOutputDir, relativePath.getPath());
     } else {

Modified: hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java?rev=723838&r1=723837&r2=723838&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
(original)
+++ hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
Fri Dec  5 11:57:58 2008
@@ -33,6 +33,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.examples.WordCount;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
@@ -249,4 +250,37 @@
     }
   }
   
+  public void testWithDFSWithDefaultPort() throws IOException {
+    MiniDFSCluster dfs = null;
+    MiniMRCluster mr = null;
+    FileSystem fileSys = null;
+    try {
+      final int taskTrackers = 4;
+
+      Configuration conf = new Configuration();
+      // start a dfs with the default port number
+      dfs = new MiniDFSCluster(
+          NameNode.DEFAULT_PORT, conf, 4, true, true, null, null);
+      fileSys = dfs.getFileSystem();
+      mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1);
+
+      JobConf jobConf = mr.createJobConf();
+      TestResult result;
+      final Path inDir = new Path("./wc/input");
+      final Path outDir = new Path("hdfs://" +
+          dfs.getNameNode().getNameNodeAddress().getHostName() +
+          ":" + NameNode.DEFAULT_PORT +"/./wc/output");
+      String input = "The quick brown fox\nhas many silly\nred fox sox\n";
+      result = launchWordCount(jobConf, inDir, outDir, input, 3, 1);
+      assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
+                   "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result.output);
+    } catch (java.net.BindException be) {
+      LOG.info("Skip the test this time because can not start namenode on port "
+          + NameNode.DEFAULT_PORT, be);
+    } finally {
+      if (dfs != null) { dfs.shutdown(); }
+      if (mr != null) { mr.shutdown();
+      }
+    }
+  }
 }



Mime
View raw message