Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id EF42BD2CC for ; Tue, 31 Jul 2012 21:49:50 +0000 (UTC) Received: (qmail 95901 invoked by uid 500); 31 Jul 2012 21:49:50 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 95856 invoked by uid 500); 31 Jul 2012 21:49:50 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 95847 invoked by uid 99); 31 Jul 2012 21:49:50 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Jul 2012 21:49:50 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Jul 2012 21:49:47 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E381923888CD; Tue, 31 Jul 2012 21:49:02 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1367795 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/mapred/org/apache/hadoop/mapred/ Date: Tue, 31 Jul 2012 21:49:02 -0000 To: mapreduce-commits@hadoop.apache.org From: jeagles@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120731214902.E381923888CD@eris.apache.org> Author: jeagles Date: Tue Jul 31 21:49:02 2012 New Revision: 1367795 URL: http://svn.apache.org/viewvc?rev=1367795&view=rev Log: svn merge -c 1367789 FIXES: MAPREDUCE-4234. SortValidator.java is incompatible with multi-user or parallel use (due to a /tmp file with static name) (Robert Evans via jeagles) Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367795&r1=1367794&r2=1367795&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 21:49:02 2012 @@ -375,6 +375,10 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at FAILED (Robert Evans via tgraves) + MAPREDUCE-4234. SortValidator.java is incompatible with multi-user or + parallel use (due to a /tmp file with static name) (Robert Evans via + jeagles) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1367795&r1=1367794&r2=1367795&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java Tue Jul 31 21:49:02 2012 @@ -33,7 +33,6 @@ import org.apache.hadoop.io.WritableComp import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapred.lib.HashPartitioner; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.fs.*; @@ -345,7 +344,8 @@ public class SortValidator extends Confi FileInputFormat.setInputPaths(jobConf, sortInput); FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path("/tmp/sortvalidate/recordstatschecker"); + Path outputPath = new Path(new Path(jobConf.get("hadoop.tmp.dir", "/tmp"), + "sortvalidate"), UUID.randomUUID().toString()); if (defaultfs.exists(outputPath)) { defaultfs.delete(outputPath, true); } @@ -365,31 +365,44 @@ public class SortValidator extends Confi Date startTime = new Date(); System.out.println("Job started: " + startTime); JobClient.runJob(jobConf); - Date end_time = new Date(); - System.out.println("Job ended: " + end_time); - System.out.println("The job took " + - (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); - - // Check to ensure that the statistics of the - // framework's sort-input and sort-output match - SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs, - new Path(outputPath, "part-00000"), defaults); - IntWritable k1 = new IntWritable(); - IntWritable k2 = new IntWritable(); - RecordStatsWritable v1 = new RecordStatsWritable(); - RecordStatsWritable v2 = new RecordStatsWritable(); - if (!stats.next(k1, v1)) { - throw new IOException("Failed to read record #1 from reduce's output"); - } - if (!stats.next(k2, v2)) { - throw new IOException("Failed to read record #2 from reduce's output"); - } - - if ((v1.getBytes() != v2.getBytes()) || (v1.getRecords() != v2.getRecords()) || - v1.getChecksum() != v2.getChecksum()) { - throw new IOException("(" + - v1.getBytes() + ", " + v1.getRecords() + ", " + v1.getChecksum() + ") v/s (" + - v2.getBytes() + ", " + v2.getRecords() + ", " + v2.getChecksum() + ")"); + try { + Date end_time = new Date(); + System.out.println("Job ended: " + end_time); + System.out.println("The job took " + + (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); + + // Check to ensure that the statistics of the + // framework's sort-input and sort-output match + SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs, + new Path(outputPath, "part-00000"), defaults); + try { + IntWritable k1 = new IntWritable(); + IntWritable k2 = new IntWritable(); + RecordStatsWritable v1 = new RecordStatsWritable(); + RecordStatsWritable v2 = new RecordStatsWritable(); + if (!stats.next(k1, v1)) { + throw new IOException( + "Failed to read record #1 from reduce's output"); + } + if (!stats.next(k2, v2)) { + throw new IOException( + "Failed to read record #2 from reduce's output"); + } + + if ((v1.getBytes() != v2.getBytes()) || + (v1.getRecords() != v2.getRecords()) || + v1.getChecksum() != v2.getChecksum()) { + throw new IOException("(" + + v1.getBytes() + ", " + v1.getRecords() + ", " + v1.getChecksum() + + ") v/s (" + + v2.getBytes() + ", " + v2.getRecords() + ", " + v2.getChecksum() + + ")"); + } + } finally { + stats.close(); + } + } finally { + defaultfs.delete(outputPath, true); } } Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java?rev=1367795&r1=1367794&r2=1367795&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java Tue Jul 31 21:49:02 2012 @@ -33,7 +33,6 @@ import org.apache.hadoop.io.WritableComp import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapred.lib.HashPartitioner; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.fs.*; @@ -345,7 +344,8 @@ public class SortValidator extends Confi FileInputFormat.setInputPaths(jobConf, sortInput); FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path("/tmp/sortvalidate/recordstatschecker"); + Path outputPath = new Path(new Path(jobConf.get("hadoop.tmp.dir", "/tmp"), + "sortvalidate"), UUID.randomUUID().toString()); if (defaultfs.exists(outputPath)) { defaultfs.delete(outputPath, true); } @@ -365,31 +365,44 @@ public class SortValidator extends Confi Date startTime = new Date(); System.out.println("Job started: " + startTime); JobClient.runJob(jobConf); - Date end_time = new Date(); - System.out.println("Job ended: " + end_time); - System.out.println("The job took " + - (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); - - // Check to ensure that the statistics of the - // framework's sort-input and sort-output match - SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs, - new Path(outputPath, "part-00000"), defaults); - IntWritable k1 = new IntWritable(); - IntWritable k2 = new IntWritable(); - RecordStatsWritable v1 = new RecordStatsWritable(); - RecordStatsWritable v2 = new RecordStatsWritable(); - if (!stats.next(k1, v1)) { - throw new IOException("Failed to read record #1 from reduce's output"); - } - if (!stats.next(k2, v2)) { - throw new IOException("Failed to read record #2 from reduce's output"); - } - - if ((v1.getBytes() != v2.getBytes()) || (v1.getRecords() != v2.getRecords()) || - v1.getChecksum() != v2.getChecksum()) { - throw new IOException("(" + - v1.getBytes() + ", " + v1.getRecords() + ", " + v1.getChecksum() + ") v/s (" + - v2.getBytes() + ", " + v2.getRecords() + ", " + v2.getChecksum() + ")"); + try { + Date end_time = new Date(); + System.out.println("Job ended: " + end_time); + System.out.println("The job took " + + (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); + + // Check to ensure that the statistics of the + // framework's sort-input and sort-output match + SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs, + new Path(outputPath, "part-00000"), defaults); + try { + IntWritable k1 = new IntWritable(); + IntWritable k2 = new IntWritable(); + RecordStatsWritable v1 = new RecordStatsWritable(); + RecordStatsWritable v2 = new RecordStatsWritable(); + if (!stats.next(k1, v1)) { + throw new IOException( + "Failed to read record #1 from reduce's output"); + } + if (!stats.next(k2, v2)) { + throw new IOException( + "Failed to read record #2 from reduce's output"); + } + + if ((v1.getBytes() != v2.getBytes()) || + (v1.getRecords() != v2.getRecords()) || + v1.getChecksum() != v2.getChecksum()) { + throw new IOException("(" + + v1.getBytes() + ", " + v1.getRecords() + ", " + v1.getChecksum() + + ") v/s (" + + v2.getBytes() + ", " + v2.getRecords() + ", " + v2.getChecksum() + + ")"); + } + } finally { + stats.close(); + } + } finally { + defaultfs.delete(outputPath, true); } }