hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject svn commit: r1329403 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/tes...
Date Mon, 23 Apr 2012 19:44:31 GMT
Author: bobby
Date: Mon Apr 23 19:44:30 2012
New Revision: 1329403

URL: http://svn.apache.org/viewvc?rev=1329403&view=rev
Log:
MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)

Modified:
    hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
    hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1329403&r1=1329402&r2=1329403&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Apr 23 19:44:30 2012
@@ -395,6 +395,7 @@ Release 0.23.3 - UNRELEASED
     MAPREDUCE-4190. Improve web UI for task attempts userlog link (Tom Graves
     via bobby)
 
+    MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)
 
 Release 0.23.2 - UNRELEASED
 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1329403&r1=1329402&r2=1329403&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
Mon Apr 23 19:44:30 2012
@@ -827,7 +827,7 @@ public class JobImpl implements org.apac
         || (sysMaxReduces == 1);
 
     long sysMaxBytes = conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES,
-        fs.getDefaultBlockSize()); // FIXME: this is wrong; get FS from
+        fs.getDefaultBlockSize(this.remoteJobSubmitDir)); // FIXME: this is wrong; get FS
from
                                    // [File?]InputFormat and default block size
                                    // from that
 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java?rev=1329403&r1=1329402&r2=1329403&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
Mon Apr 23 19:44:30 2012
@@ -128,7 +128,7 @@ public class DistributedFSCheck extends 
     if (rootStatus.isFile()) {
       nrFiles++;
       // For a regular file generate <fName,offset> pairs
-      long blockSize = fs.getDefaultBlockSize();
+      long blockSize = fs.getDefaultBlockSize(rootFile);
       long fileLength = rootStatus.getLen();
       for(long offset = 0; offset < fileLength; offset += blockSize)
         writer.append(new Text(rootFile.toString()), new LongWritable(offset));
@@ -160,15 +160,16 @@ public class DistributedFSCheck extends 
                        ) throws IOException {
       // open file
       FSDataInputStream in = null;
+      Path p = new Path(name);
       try {
-        in = fs.open(new Path(name));
+        in = fs.open(p);
       } catch(IOException e) {
         return name + "@(missing)";
       }
       in.seek(offset);
       long actualSize = 0;
       try {
-        long blockSize = fs.getDefaultBlockSize();
+        long blockSize = fs.getDefaultBlockSize(p);
         reporter.setStatus("reading " + name + "@" + 
                            offset + "/" + blockSize);
         for( int curSize = bufferSize; 

Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=1329403&r1=1329402&r2=1329403&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
Mon Apr 23 19:44:30 2012
@@ -287,7 +287,7 @@ public class JobHistory {
     FSDataOutputStream out = logDirFs.create(logFile, 
         new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
         true, defaultBufferSize, 
-        logDirFs.getDefaultReplication(), 
+        logDirFs.getDefaultReplication(logFile), 
         jobHistoryBlockSize, null);
   
     EventWriter writer = new EventWriter(out);
@@ -306,8 +306,8 @@ public class JobHistory {
           jobFileOut = logDirFs.create(logDirConfPath,
               new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
               true, defaultBufferSize,
-              logDirFs.getDefaultReplication(),
-              logDirFs.getDefaultBlockSize(), null);
+              logDirFs.getDefaultReplication(logDirConfPath),
+              logDirFs.getDefaultBlockSize(logDirConfPath), null);
           jobConf.writeXml(jobFileOut);
           jobFileOut.close();
         }



Mime
View raw message