Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B25B096E8 for ; Mon, 23 Apr 2012 19:44:56 +0000 (UTC) Received: (qmail 59672 invoked by uid 500); 23 Apr 2012 19:44:56 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 59554 invoked by uid 500); 23 Apr 2012 19:44:56 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 59485 invoked by uid 99); 23 Apr 2012 19:44:55 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Apr 2012 19:44:55 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Apr 2012 19:44:52 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 6F3F02388962; Mon, 23 Apr 2012 19:44:31 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1329403 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/tes... Date: Mon, 23 Apr 2012 19:44:31 -0000 To: mapreduce-commits@hadoop.apache.org From: bobby@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120423194431.6F3F02388962@eris.apache.org> Author: bobby Date: Mon Apr 23 19:44:30 2012 New Revision: 1329403 URL: http://svn.apache.org/viewvc?rev=1329403&view=rev Log: MAPREDUCE-4133. MR over viewfs is broken (John George via bobby) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1329403&r1=1329402&r2=1329403&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Apr 23 19:44:30 2012 @@ -395,6 +395,7 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4190. Improve web UI for task attempts userlog link (Tom Graves via bobby) + MAPREDUCE-4133. MR over viewfs is broken (John George via bobby) Release 0.23.2 - UNRELEASED Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1329403&r1=1329402&r2=1329403&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Mon Apr 23 19:44:30 2012 @@ -827,7 +827,7 @@ public class JobImpl implements org.apac || (sysMaxReduces == 1); long sysMaxBytes = conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES, - fs.getDefaultBlockSize()); // FIXME: this is wrong; get FS from + fs.getDefaultBlockSize(this.remoteJobSubmitDir)); // FIXME: this is wrong; get FS from // [File?]InputFormat and default block size // from that Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java?rev=1329403&r1=1329402&r2=1329403&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java Mon Apr 23 19:44:30 2012 @@ -128,7 +128,7 @@ public class DistributedFSCheck extends if (rootStatus.isFile()) { nrFiles++; // For a regular file generate pairs - long blockSize = fs.getDefaultBlockSize(); + long blockSize = fs.getDefaultBlockSize(rootFile); long fileLength = rootStatus.getLen(); for(long offset = 0; offset < fileLength; offset += blockSize) writer.append(new Text(rootFile.toString()), new LongWritable(offset)); @@ -160,15 +160,16 @@ public class DistributedFSCheck extends ) throws IOException { // open file FSDataInputStream in = null; + Path p = new Path(name); try { - in = fs.open(new Path(name)); + in = fs.open(p); } catch(IOException e) { return name + "@(missing)"; } in.seek(offset); long actualSize = 0; try { - long blockSize = fs.getDefaultBlockSize(); + long blockSize = fs.getDefaultBlockSize(p); reporter.setStatus("reading " + name + "@" + offset + "/" + blockSize); for( int curSize = bufferSize; Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=1329403&r1=1329402&r2=1329403&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java Mon Apr 23 19:44:30 2012 @@ -287,7 +287,7 @@ public class JobHistory { FSDataOutputStream out = logDirFs.create(logFile, new FsPermission(JobHistory.HISTORY_FILE_PERMISSION), true, defaultBufferSize, - logDirFs.getDefaultReplication(), + logDirFs.getDefaultReplication(logFile), jobHistoryBlockSize, null); EventWriter writer = new EventWriter(out); @@ -306,8 +306,8 @@ public class JobHistory { jobFileOut = logDirFs.create(logDirConfPath, new FsPermission(JobHistory.HISTORY_FILE_PERMISSION), true, defaultBufferSize, - logDirFs.getDefaultReplication(), - logDirFs.getDefaultBlockSize(), null); + logDirFs.getDefaultReplication(logDirConfPath), + logDirFs.getDefaultBlockSize(logDirConfPath), null); jobConf.writeXml(jobFileOut); jobFileOut.close(); }