From mapreduce-commits-return-4649-apmail-hadoop-mapreduce-commits-archive=hadoop.apache.org@hadoop.apache.org Thu Aug 9 22:32:00 2012 Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id AF81CD1ED for ; Thu, 9 Aug 2012 22:32:00 +0000 (UTC) Received: (qmail 8003 invoked by uid 500); 9 Aug 2012 22:32:00 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 7934 invoked by uid 500); 9 Aug 2012 22:32:00 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 7926 invoked by uid 99); 9 Aug 2012 22:32:00 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 09 Aug 2012 22:32:00 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 09 Aug 2012 22:31:50 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 04A572388B56; Thu, 9 Aug 2012 22:30:44 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1371518 [1/2] - in /hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project: ./ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapreduce-client/hadoop-mapreduce... Date: Thu, 09 Aug 2012 22:30:41 -0000 To: mapreduce-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120809223044.04A572388B56@eris.apache.org> Author: todd Date: Thu Aug 9 22:29:36 2012 New Revision: 1371518 URL: http://svn.apache.org/viewvc?rev=1371518&view=rev Log: Merge trunk into branch. Branch will not build after this commit: need to implement new JournalManager interfaces in QuorumJournalManager in a follow-up. Added: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java - copied unchanged from r1371513, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java - copied unchanged from r1371513, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java - copied unchanged from r1371513, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java Removed: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/conf/container-executor.cfg hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-yarn/ Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/conf/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/pom.xml hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/c++/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/build.xml (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/data_join/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/index/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/examples/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/java/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed) hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/webapps/job/ (props changed) Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1367365-1371513 Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt Thu Aug 9 22:29:36 2012 @@ -159,6 +159,14 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4342. Distributed Cache gives inconsistent result if cache files get deleted from tasktracker. (mayank_bansal via tucu) + MAPREDUCE-4498. Remove hsqldb jar from Hadoop runtime classpath. (rkanter via tucu) + + MAPREDUCE-4494. TestFifoScheduler failing with Metrics source QueueMetrics,q0=default + already exists!. (ahmed.radwan via tucu) + + MAPREDUCE-4484. Incorrect IS_MINI_YARN_CLUSTER property name in YarnConfiguration. + (ahmed.radwan via tucu) + Release 2.1.0-alpha - Unreleased INCOMPATIBLE CHANGES @@ -206,6 +214,9 @@ Release 2.1.0-alpha - Unreleased MAPREDUCE-4447. Remove aop from cruft from the ant build. (eli) + MAPREDUCE-3289. Make use of fadvise in the NM's shuffle handler. + (Todd Lipcon and Siddharth Seth via sseth) + OPTIMIZATIONS BUG FIXES @@ -461,6 +472,11 @@ Release 2.0.0-alpha - 05-23-2012 MAPREDUCE-4231. Update RAID to use the new BlockCollection interface. (szetszwo) + MAPREDUCE-4483. 2.0 build does not work (John George via bobby) + + MAPREDUCE-4444. nodemanager fails to start when one of the local-dirs is + bad (Jason Lowe via bobby) + Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES @@ -505,6 +521,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4267. mavenize pipes (tgraves via bobby) + MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby + via tgraves) + OPTIMIZATIONS MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn @@ -763,6 +782,33 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans via tgraves) + MAPREDUCE-4456. LocalDistributedCacheManager can get an + ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves) + + MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby) + + MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans + via tgraves) + + MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at + perticular level is sucessfull (Mayank Bansal via bobby) + + MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at + FAILED (Robert Evans via tgraves) + + MAPREDUCE-4234. SortValidator.java is incompatible with multi-user or + parallel use (due to a /tmp file with static name) (Robert Evans via + jeagles) + + MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans + via tgraves) + + MAPREDUCE-4503. Should throw InvalidJobConfException if duplicates found in + cacheArchives or cacheFiles (Robert Evans via jeagles) + + MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason + Lowe via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1367365-1371513 Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/conf/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1367365-1371513 Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Thu Aug 9 22:29:36 2012 @@ -1370,7 +1370,8 @@ public class JobImpl implements org.apac } } - float failureRate = (float) fetchFailures / runningReduceTasks; + float failureRate = runningReduceTasks == 0 ? 1.0f : + (float) fetchFailures / runningReduceTasks; // declare faulty if fetch-failures >= max-allowed-failures boolean isMapFaulty = (failureRate >= MAX_ALLOWED_FETCH_FAILURES_FRACTION); @@ -1561,7 +1562,7 @@ public class JobImpl implements org.apac Path confPath = getConfFile(); FileContext fc = FileContext.getFileContext(confPath.toUri(), conf); Configuration jobConf = new Configuration(false); - jobConf.addResource(fc.open(confPath)); + jobConf.addResource(fc.open(confPath), confPath.toString()); return jobConf; } } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Thu Aug 9 22:29:36 2012 @@ -435,7 +435,8 @@ public abstract class TaskAttemptImpl im TaskAttemptEventType.TA_CONTAINER_CLEANED, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, - TaskAttemptEventType.TA_FAILMSG)) + TaskAttemptEventType.TA_FAILMSG, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)) // Transitions from KILLED state .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED, Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java Thu Aug 9 22:29:36 2012 @@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock tr(). th(_TH, "key"). th(_TH, "value"). + th(_TH, "source chain"). _(). _(). tbody(); for (ConfEntryInfo entry : info.getProperties()) { + StringBuffer buffer = new StringBuffer(); + String[] sources = entry.getSource(); + //Skip the last entry, because it is always the same HDFS file, and + // output them in reverse order so most recent is output first + boolean first = true; + for(int i = (sources.length - 2); i >= 0; i--) { + if(!first) { + // \u2B05 is an arrow <-- + buffer.append(" \u2B05 "); + } + first = false; + buffer.append(sources[i]); + } tbody. tr(). td(entry.getName()). td(entry.getValue()). + td(buffer.toString()). _(); } tbody._(). @@ -93,6 +108,7 @@ public class ConfBlock extends HtmlBlock tr(). th().input("search_init").$type(InputType.text).$name("key").$value("key")._()._(). th().input("search_init").$type(InputType.text).$name("value").$value("value")._()._(). + th().input("search_init").$type(InputType.text).$name("source chain").$value("source chain")._()._(). _(). _(). _(); Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java Thu Aug 9 22:29:36 2012 @@ -64,7 +64,7 @@ public class AMAttemptInfo { if (containerId != null) { this.containerId = containerId.toString(); this.logsLink = join("http://" + nodeHttpAddress, - ujoin("node", "containerlogs", this.containerId)); + ujoin("node", "containerlogs", this.containerId, user)); } } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java Thu Aug 9 22:29:36 2012 @@ -27,13 +27,19 @@ public class ConfEntryInfo { protected String name; protected String value; + protected String[] source; public ConfEntryInfo() { } public ConfEntryInfo(String key, String value) { + this(key, value, null); + } + + public ConfEntryInfo(String key, String value, String[] source) { this.name = key; this.value = value; + this.source = source; } public String getName() { @@ -43,4 +49,8 @@ public class ConfEntryInfo { public String getValue() { return this.value; } + + public String[] getSource() { + return source; + } } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java Thu Aug 9 22:29:36 2012 @@ -46,7 +46,8 @@ public class ConfInfo { Configuration jobConf = job.loadConfFile(); this.path = job.getConfFile().toString(); for (Map.Entry entry : jobConf) { - this.property.add(new ConfEntryInfo(entry.getKey(), entry.getValue())); + this.property.add(new ConfEntryInfo(entry.getKey(), entry.getValue(), + jobConf.getPropertySources(entry.getKey()))); } } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java Thu Aug 9 22:29:36 2012 @@ -603,7 +603,7 @@ public class MockJobs extends MockApps { public Configuration loadConfFile() throws IOException { FileContext fc = FileContext.getFileContext(configFile.toUri(), conf); Configuration jobConf = new Configuration(false); - jobConf.addResource(fc.open(configFile)); + jobConf.addResource(fc.open(configFile), configFile.toString()); return jobConf; } }; Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java Thu Aug 9 22:29:36 2012 @@ -565,6 +565,73 @@ public class TestTaskAttempt{ assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED", eventHandler.internalError); } + + @Test + public void testDoubleTooManyFetchFailure() throws Exception { + ApplicationId appId = BuilderUtils.newApplicationId(1, 2); + ApplicationAttemptId appAttemptId = + BuilderUtils.newApplicationAttemptId(appId, 0); + JobId jobId = MRBuilderUtils.newJobId(appId, 1); + TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP); + TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); + Path jobFile = mock(Path.class); + + MockEventHandler eventHandler = new MockEventHandler(); + TaskAttemptListener taListener = mock(TaskAttemptListener.class); + when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0)); + + JobConf jobConf = new JobConf(); + jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); + jobConf.setBoolean("fs.file.impl.disable.cache", true); + jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, ""); + jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10"); + + TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class); + when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"}); + + AppContext appCtx = mock(AppContext.class); + ClusterInfo clusterInfo = mock(ClusterInfo.class); + Resource resource = mock(Resource.class); + when(appCtx.getClusterInfo()).thenReturn(clusterInfo); + when(clusterInfo.getMinContainerCapability()).thenReturn(resource); + when(resource.getMemory()).thenReturn(1024); + + TaskAttemptImpl taImpl = + new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, + splits, jobConf, taListener, + mock(OutputCommitter.class), mock(Token.class), new Credentials(), + new SystemClock(), appCtx); + + NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0); + ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3); + Container container = mock(Container.class); + when(container.getId()).thenReturn(contId); + when(container.getNodeId()).thenReturn(nid); + when(container.getNodeHttpAddress()).thenReturn("localhost:0"); + + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_SCHEDULE)); + taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId, + container, mock(Map.class))); + taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0)); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_DONE)); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_CONTAINER_CLEANED)); + + assertEquals("Task attempt is not in succeeded state", taImpl.getState(), + TaskAttemptState.SUCCEEDED); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)); + assertEquals("Task attempt is not in FAILED state", taImpl.getState(), + TaskAttemptState.FAILED); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)); + assertEquals("Task attempt is not in FAILED state, still", taImpl.getState(), + TaskAttemptState.FAILED); + assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED", + eventHandler.internalError); + } public static class MockEventHandler implements EventHandler { public boolean internalError; Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java Thu Aug 9 22:29:36 2012 @@ -972,7 +972,8 @@ public class TestAMWebServicesJobs exten WebServicesTestUtils.checkStringMatch("containerId", amInfo .getContainerId().toString(), containerId); - String localLogsLink = ujoin("node", "containerlogs", containerId); + String localLogsLink =ujoin("node", "containerlogs", containerId, + job.getUserName()); assertTrue("logsLink", logsLink.contains(localLogsLink)); } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml Thu Aug 9 22:29:36 2012 @@ -77,7 +77,7 @@ protoc - -I../../hadoop-yarn/hadoop-yarn-api/src/main/proto/ + -I../../../hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/ -Isrc/main/proto/ --java_out=target/generated-sources/proto src/main/proto/mr_protos.proto Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java Thu Aug 9 22:29:36 2012 @@ -18,12 +18,9 @@ package org.apache.hadoop.mapred; -import com.google.common.collect.Maps; - import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLClassLoader; @@ -34,6 +31,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.FSDownload; +import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; /** @@ -85,6 +84,8 @@ class LocalDistributedCacheManager { * @throws IOException */ public void setup(JobConf conf) throws IOException { + File workDir = new File(System.getProperty("user.dir")); + // Generate YARN local resources objects corresponding to the distributed // cache configuration Map localResources = @@ -132,7 +133,8 @@ class LocalDistributedCacheManager { Future future = exec.submit(download); resourcesToPaths.put(resource, future); } - for (LocalResource resource : localResources.values()) { + for (Entry entry : localResources.entrySet()) { + LocalResource resource = entry.getValue(); Path path; try { path = resourcesToPaths.get(resource).get(); @@ -142,6 +144,10 @@ class LocalDistributedCacheManager { throw new IOException(e); } String pathString = path.toUri().toString(); + String link = entry.getKey(); + String target = new File(path.toUri()).getPath(); + symlink(workDir, target, link); + if (resource.getType() == LocalResourceType.ARCHIVE) { localArchives.add(pathString); } else if (resource.getType() == LocalResourceType.FILE) { @@ -175,27 +181,6 @@ class LocalDistributedCacheManager { .arrayToString(localFiles.toArray(new String[localArchives .size()]))); } - if (DistributedCache.getSymlink(conf)) { - File workDir = new File(System.getProperty("user.dir")); - URI[] archives = DistributedCache.getCacheArchives(conf); - URI[] files = DistributedCache.getCacheFiles(conf); - Path[] localArchives = DistributedCache.getLocalCacheArchives(conf); - Path[] localFiles = DistributedCache.getLocalCacheFiles(conf); - if (archives != null) { - for (int i = 0; i < archives.length; i++) { - String link = archives[i].getFragment(); - String target = new File(localArchives[i].toUri()).getPath(); - symlink(workDir, target, link); - } - } - if (files != null) { - for (int i = 0; i < files.length; i++) { - String link = files[i].getFragment(); - String target = new File(localFiles[i].toUri()).getPath(); - symlink(workDir, target, link); - } - } - } setupCalled = true; } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Thu Aug 9 22:29:36 2012 @@ -35,6 +35,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.InvalidJobConfException; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskAttemptID; @@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; /** * Helper class for MR applications @@ -264,6 +266,13 @@ public class MRApps extends Apps { DistributedCache.getFileClassPaths(conf)); } + private static String getResourceDescription(LocalResourceType type) { + if(type == LocalResourceType.ARCHIVE) { + return "cache archive (" + MRJobConfig.CACHE_ARCHIVES + ") "; + } + return "cache file (" + MRJobConfig.CACHE_FILES + ") "; + } + // TODO - Move this to MR! // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], // long[], boolean[], Path[], FileType) @@ -309,6 +318,13 @@ public class MRApps extends Apps { throw new IllegalArgumentException("Resource name must be relative"); } String linkName = name.toUri().getPath(); + LocalResource orig = localResources.get(linkName); + if(orig != null && !orig.getResource().equals( + ConverterUtils.getYarnUrlFromURI(p.toUri()))) { + throw new InvalidJobConfException( + getResourceDescription(orig.getType()) + orig.getResource() + + " conflicts with " + getResourceDescription(type) + u); + } localResources.put( linkName, BuilderUtils.newLocalResource( Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java Thu Aug 9 22:29:36 2012 @@ -117,7 +117,8 @@ public class TestMRWithDistributedCache TestCase.assertEquals("symlink distributed.first.symlink length not 1", 1, symlinkFile.length()); - TestCase.assertFalse("second file should not be symlinked", + //This last one is a difference between MRv2 and MRv1 + TestCase.assertTrue("second file should be symlinked too", expectedAbsentSymlinkFile.exists()); } } @@ -145,7 +146,6 @@ public class TestMRWithDistributedCache job.addFileToClassPath(second); job.addArchiveToClassPath(third); job.addCacheArchive(fourth.toUri()); - job.createSymlink(); job.setMaxMapAttempts(1); // speed up failures job.submit(); Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Thu Aug 9 22:29:36 2012 @@ -19,25 +19,33 @@ package org.apache.hadoop.mapreduce.v2.util; import java.io.IOException; +import java.net.URI; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.InvalidJobConfException; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.util.MRApps; -import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.junit.Test; import static org.junit.Assert.*; +import static org.mockito.Mockito.*; public class TestMRApps { @@ -166,5 +174,122 @@ public class TestMRApps { assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!", env_str.indexOf("$PWD:job.jar"), 0); } - + + @Test + public void testSetupDistributedCacheEmpty() throws IOException { + Configuration conf = new Configuration(); + Map localResources = new HashMap(); + MRApps.setupDistributedCache(conf, localResources); + assertTrue("Empty Config did not produce an empty list of resources", + localResources.isEmpty()); + } + + @SuppressWarnings("deprecation") + @Test(expected = InvalidJobConfException.class) + public void testSetupDistributedCacheConflicts() throws Exception { + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI mockUri = URI.create("mockfs://mock/"); + FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf)) + .getRawFileSystem(); + + URI archive = new URI("mockfs://mock/tmp/something.zip#something"); + Path archivePath = new Path(archive); + URI file = new URI("mockfs://mock/tmp/something.txt#something"); + Path filePath = new Path(file); + + when(mockFs.resolvePath(archivePath)).thenReturn(archivePath); + when(mockFs.resolvePath(filePath)).thenReturn(filePath); + + DistributedCache.addCacheArchive(archive, conf); + conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10"); + conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10"); + conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true"); + DistributedCache.addCacheFile(file, conf); + conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11"); + conf.set(MRJobConfig.CACHE_FILES_SIZES, "11"); + conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true"); + Map localResources = + new HashMap(); + MRApps.setupDistributedCache(conf, localResources); + } + + @SuppressWarnings("deprecation") + @Test(expected = InvalidJobConfException.class) + public void testSetupDistributedCacheConflictsFiles() throws Exception { + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI mockUri = URI.create("mockfs://mock/"); + FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf)) + .getRawFileSystem(); + + URI file = new URI("mockfs://mock/tmp/something.zip#something"); + Path filePath = new Path(file); + URI file2 = new URI("mockfs://mock/tmp/something.txt#something"); + Path file2Path = new Path(file); + + when(mockFs.resolvePath(filePath)).thenReturn(filePath); + when(mockFs.resolvePath(file2Path)).thenReturn(file2Path); + + DistributedCache.addCacheFile(file, conf); + DistributedCache.addCacheFile(file2, conf); + conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11"); + conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11"); + conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true"); + Map localResources = + new HashMap(); + MRApps.setupDistributedCache(conf, localResources); + } + + @SuppressWarnings("deprecation") + @Test + public void testSetupDistributedCache() throws Exception { + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI mockUri = URI.create("mockfs://mock/"); + FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf)) + .getRawFileSystem(); + + URI archive = new URI("mockfs://mock/tmp/something.zip"); + Path archivePath = new Path(archive); + URI file = new URI("mockfs://mock/tmp/something.txt#something"); + Path filePath = new Path(file); + + when(mockFs.resolvePath(archivePath)).thenReturn(archivePath); + when(mockFs.resolvePath(filePath)).thenReturn(filePath); + + DistributedCache.addCacheArchive(archive, conf); + conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10"); + conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10"); + conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true"); + DistributedCache.addCacheFile(file, conf); + conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11"); + conf.set(MRJobConfig.CACHE_FILES_SIZES, "11"); + conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true"); + Map localResources = + new HashMap(); + MRApps.setupDistributedCache(conf, localResources); + assertEquals(2, localResources.size()); + LocalResource lr = localResources.get("something.zip"); + assertNotNull(lr); + assertEquals(10l, lr.getSize()); + assertEquals(10l, lr.getTimestamp()); + assertEquals(LocalResourceType.ARCHIVE, lr.getType()); + lr = localResources.get("something"); + assertNotNull(lr); + assertEquals(11l, lr.getSize()); + assertEquals(11l, lr.getTimestamp()); + assertEquals(LocalResourceType.FILE, lr.getType()); + } + + static class MockFileSystem extends FilterFileSystem { + MockFileSystem() { + super(mock(FileSystem.class)); + } + public void initialize(URI name, Configuration conf) throws IOException {} + } + } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java Thu Aug 9 22:29:36 2012 @@ -48,8 +48,12 @@ import org.apache.hadoop.mapreduce.Job; * Archives (zip, tar and tgz/tar.gz files) are un-archived at the slave nodes. * Jars may be optionally added to the classpath of the tasks, a rudimentary * software distribution mechanism. Files have execution permissions. - * Optionally users can also direct it to symlink the distributed cache file(s) - * into the working directory of the task.

+ * In older version of Hadoop Map/Reduce users could optionally ask for symlinks + * to be created in the working directory of the child task. In the current + * version symlinks are always created. If the URL does not have a fragment + * the name of the file or directory will be used. If multiple files or + * directories map to the same link name, the last one added, will be used. All + * others will not even be downloaded.

* *

DistributedCache tracks modification timestamps of the cache * files. Clearly the cache files should not be modified by the application @@ -91,8 +95,7 @@ import org.apache.hadoop.mapreduce.Job; * * public void configure(JobConf job) { * // Get the cached archives/files - * localArchives = DistributedCache.getLocalCacheArchives(job); - * localFiles = DistributedCache.getLocalCacheFiles(job); + * File f = new File("./map.zip/some/file/in/zip.txt"); * } * * public void map(K key, V value, Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java Thu Aug 9 22:29:36 2012 @@ -313,7 +313,6 @@ public class Submitter extends Configure // add default debug script only when executable is expressed as // # if (exec.contains("#")) { - DistributedCache.createSymlink(conf); // set default gdb commands for map and reduce task String defScript = "$HADOOP_PREFIX/src/c++/pipes/debug/pipes-default-script"; setIfUnset(conf, MRJobConfig.MAP_DEBUG_SCRIPT,defScript); Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java Thu Aug 9 22:29:36 2012 @@ -1049,9 +1049,10 @@ public class Job extends JobContextImpl } /** - * This method allows you to create symlinks in the current working directory - * of the task to all the cache files/archives + * Originally intended to enable symlinks, but currently symlinks cannot be + * disabled. */ + @Deprecated public void createSymlink() { ensureState(JobState.DEFINE); DistributedCache.createSymlink(conf); Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java Thu Aug 9 22:29:36 2012 @@ -221,10 +221,11 @@ public interface JobContext extends MRJo public String getUser(); /** - * This method checks to see if symlinks are to be create for the - * localized cache files in the current working directory - * @return true if symlinks are to be created- else return false + * Originally intended to check if symlinks should be used, but currently + * symlinks cannot be disabled. + * @return true */ + @Deprecated public boolean getSymlink(); /** @@ -251,14 +252,22 @@ public interface JobContext extends MRJo * Return the path array of the localized caches * @return A path array of localized caches * @throws IOException + * @deprecated the array returned only includes the items the were + * downloaded. There is no way to map this to what is returned by + * {@link #getCacheArchives()}. */ + @Deprecated public Path[] getLocalCacheArchives() throws IOException; /** * Return the path array of the localized files * @return A path array of localized files * @throws IOException + * @deprecated the array returned only includes the items the were + * downloaded. There is no way to map this to what is returned by + * {@link #getCacheFiles()}. */ + @Deprecated public Path[] getLocalCacheFiles() throws IOException; /** Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java Thu Aug 9 22:29:36 2012 @@ -190,7 +190,6 @@ class JobSubmitter { //should not throw a uri exception throw new IOException("Failed to create uri for " + tmpFile, ue); } - DistributedCache.createSymlink(conf); } } @@ -225,7 +224,6 @@ class JobSubmitter { //should not throw an uri excpetion throw new IOException("Failed to create uri for " + tmpArchives, ue); } - DistributedCache.createSymlink(conf); } } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Thu Aug 9 22:29:36 2012 @@ -114,6 +114,10 @@ public interface MRJobConfig { public static final String CACHE_ARCHIVES_VISIBILITIES = "mapreduce.job.cache.archives.visibilities"; + /** + * @deprecated Symlinks are always on and cannot be disabled. + */ + @Deprecated public static final String CACHE_SYMLINK = "mapreduce.job.cache.symlink.create"; public static final String USER_LOG_RETAIN_HOURS = "mapreduce.job.userlog.retain.hours"; Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java Thu Aug 9 22:29:36 2012 @@ -55,8 +55,12 @@ import java.net.URI; * Archives (zip, tar and tgz/tar.gz files) are un-archived at the slave nodes. * Jars may be optionally added to the classpath of the tasks, a rudimentary * software distribution mechanism. Files have execution permissions. - * Optionally users can also direct it to symlink the distributed cache file(s) - * into the working directory of the task.

+ * In older version of Hadoop Map/Reduce users could optionally ask for symlinks + * to be created in the working directory of the child task. In the current + * version symlinks are always created. If the URL does not have a fragment + * the name of the file or directory will be used. If multiple files or + * directories map to the same link name, the last one added, will be used. All + * others will not even be downloaded.

* *

DistributedCache tracks modification timestamps of the cache * files. Clearly the cache files should not be modified by the application @@ -98,8 +102,7 @@ import java.net.URI; * * public void configure(JobConf job) { * // Get the cached archives/files - * localArchives = DistributedCache.getLocalCacheArchives(job); - * localFiles = DistributedCache.getLocalCacheFiles(job); + * File f = new File("./map.zip/some/file/in/zip.txt"); * } * * public void map(K key, V value, @@ -375,32 +378,26 @@ public class DistributedCache { } /** - * This method allows you to create symlinks in the current working directory - * of the task to all the cache files/archives. - * Intended to be used by user code. + * Originally intended to enable symlinks, but currently symlinks cannot be + * disabled. This is a NO-OP. * @param conf the jobconf - * @deprecated Use {@link Job#createSymlink()} instead + * @deprecated This is a NO-OP. */ @Deprecated public static void createSymlink(Configuration conf){ - conf.set(MRJobConfig.CACHE_SYMLINK, "yes"); + //NOOP } /** - * This method checks to see if symlinks are to be create for the - * localized cache files in the current working directory - * Used by internal DistributedCache code. + * Originally intended to check if symlinks should be used, but currently + * symlinks cannot be disabled. * @param conf the jobconf - * @return true if symlinks are to be created- else return false - * @deprecated Use {@link JobContext#getSymlink()} instead + * @return true + * @deprecated symlinks are always created. */ @Deprecated public static boolean getSymlink(Configuration conf){ - String result = conf.get(MRJobConfig.CACHE_SYMLINK); - if ("yes".equals(result)){ - return true; - } - return false; + return true; } private static boolean[] parseBooleans(String[] strs) { Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java Thu Aug 9 22:29:36 2012 @@ -246,8 +246,6 @@ public class ConfigUtil { new String[] {MRJobConfig.CACHE_FILE_TIMESTAMPS}); Configuration.addDeprecation("mapred.cache.archives.timestamps", new String[] {MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS}); - Configuration.addDeprecation("mapred.create.symlink", - new String[] {MRJobConfig.CACHE_SYMLINK}); Configuration.addDeprecation("mapred.working.dir", new String[] {MRJobConfig.WORKING_DIR}); Configuration.addDeprecation("user.name", Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1367365-1371513 Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java Thu Aug 9 22:29:36 2012 @@ -336,7 +336,7 @@ public class HistoryFileManager extends public synchronized Configuration loadConfFile() throws IOException { FileContext fc = FileContext.getFileContext(confFile.toUri(), conf); Configuration jobConf = new Configuration(false); - jobConf.addResource(fc.open(confFile)); + jobConf.addResource(fc.open(confFile), confFile.toString()); return jobConf; } } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml Thu Aug 9 22:29:36 2012 @@ -94,7 +94,6 @@ org.hsqldb hsqldb - 2.0.0 test Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java Thu Aug 9 22:29:36 2012 @@ -210,19 +210,10 @@ public class MRCaching { fs.copyFromLocalFile(tarPath1, cachePath); fs.copyFromLocalFile(tarPath2, cachePath); } - - public static TestResult launchMRCache(String indir, - String outdir, String cacheDir, - JobConf conf, String input) - throws IOException { - setupCache(cacheDir, FileSystem.get(conf)); - return launchMRCache(indir,outdir, cacheDir, conf, input, false); - } public static TestResult launchMRCache(String indir, String outdir, String cacheDir, - JobConf conf, String input, - boolean withSymlink) + JobConf conf, String input) throws IOException { String TEST_ROOT_DIR = new Path(System.getProperty("test.build.data","/tmp")) .toString().replace(' ', '+'); @@ -256,24 +247,13 @@ public class MRCaching { conf.setNumReduceTasks(1); conf.setSpeculativeExecution(false); URI[] uris = new URI[6]; - if (!withSymlink) { - conf.setMapperClass(MRCaching.MapClass.class); - uris[0] = fs.getUri().resolve(cacheDir + "/test.txt"); - uris[1] = fs.getUri().resolve(cacheDir + "/test.jar"); - uris[2] = fs.getUri().resolve(cacheDir + "/test.zip"); - uris[3] = fs.getUri().resolve(cacheDir + "/test.tgz"); - uris[4] = fs.getUri().resolve(cacheDir + "/test.tar.gz"); - uris[5] = fs.getUri().resolve(cacheDir + "/test.tar"); - } else { - DistributedCache.createSymlink(conf); - conf.setMapperClass(MRCaching.MapClass2.class); - uris[0] = fs.getUri().resolve(cacheDir + "/test.txt#" + "test.txt"); - uris[1] = fs.getUri().resolve(cacheDir + "/test.jar#" + "testjar"); - uris[2] = fs.getUri().resolve(cacheDir + "/test.zip#" + "testzip"); - uris[3] = fs.getUri().resolve(cacheDir + "/test.tgz#" + "testtgz"); - uris[4] = fs.getUri().resolve(cacheDir + "/test.tar.gz#" + "testtargz"); - uris[5] = fs.getUri().resolve(cacheDir + "/test.tar#" + "testtar"); - } + conf.setMapperClass(MRCaching.MapClass2.class); + uris[0] = fs.getUri().resolve(cacheDir + "/test.txt"); + uris[1] = fs.getUri().resolve(cacheDir + "/test.jar"); + uris[2] = fs.getUri().resolve(cacheDir + "/test.zip"); + uris[3] = fs.getUri().resolve(cacheDir + "/test.tgz"); + uris[4] = fs.getUri().resolve(cacheDir + "/test.tar.gz"); + uris[5] = fs.getUri().resolve(cacheDir + "/test.tar"); DistributedCache.addCacheFile(uris[0], conf); // Save expected file sizes Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java Thu Aug 9 22:29:36 2012 @@ -33,7 +33,6 @@ import org.apache.hadoop.io.WritableComp import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapred.lib.HashPartitioner; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.fs.*; @@ -345,7 +344,8 @@ public class SortValidator extends Confi FileInputFormat.setInputPaths(jobConf, sortInput); FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path("/tmp/sortvalidate/recordstatschecker"); + Path outputPath = new Path(new Path("/tmp", + "sortvalidate"), UUID.randomUUID().toString()); if (defaultfs.exists(outputPath)) { defaultfs.delete(outputPath, true); } @@ -365,31 +365,44 @@ public class SortValidator extends Confi Date startTime = new Date(); System.out.println("Job started: " + startTime); JobClient.runJob(jobConf); - Date end_time = new Date(); - System.out.println("Job ended: " + end_time); - System.out.println("The job took " + - (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); - - // Check to ensure that the statistics of the - // framework's sort-input and sort-output match - SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs, - new Path(outputPath, "part-00000"), defaults); - IntWritable k1 = new IntWritable(); - IntWritable k2 = new IntWritable(); - RecordStatsWritable v1 = new RecordStatsWritable(); - RecordStatsWritable v2 = new RecordStatsWritable(); - if (!stats.next(k1, v1)) { - throw new IOException("Failed to read record #1 from reduce's output"); - } - if (!stats.next(k2, v2)) { - throw new IOException("Failed to read record #2 from reduce's output"); - } - - if ((v1.getBytes() != v2.getBytes()) || (v1.getRecords() != v2.getRecords()) || - v1.getChecksum() != v2.getChecksum()) { - throw new IOException("(" + - v1.getBytes() + ", " + v1.getRecords() + ", " + v1.getChecksum() + ") v/s (" + - v2.getBytes() + ", " + v2.getRecords() + ", " + v2.getChecksum() + ")"); + try { + Date end_time = new Date(); + System.out.println("Job ended: " + end_time); + System.out.println("The job took " + + (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); + + // Check to ensure that the statistics of the + // framework's sort-input and sort-output match + SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs, + new Path(outputPath, "part-00000"), defaults); + try { + IntWritable k1 = new IntWritable(); + IntWritable k2 = new IntWritable(); + RecordStatsWritable v1 = new RecordStatsWritable(); + RecordStatsWritable v2 = new RecordStatsWritable(); + if (!stats.next(k1, v1)) { + throw new IOException( + "Failed to read record #1 from reduce's output"); + } + if (!stats.next(k2, v2)) { + throw new IOException( + "Failed to read record #2 from reduce's output"); + } + + if ((v1.getBytes() != v2.getBytes()) || + (v1.getRecords() != v2.getRecords()) || + v1.getChecksum() != v2.getChecksum()) { + throw new IOException("(" + + v1.getBytes() + ", " + v1.getRecords() + ", " + v1.getChecksum() + + ") v/s (" + + v2.getBytes() + ", " + v2.getRecords() + ", " + v2.getChecksum() + + ")"); + } + } finally { + stats.close(); + } + } finally { + defaultfs.delete(outputPath, true); } } Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java Thu Aug 9 22:29:36 2012 @@ -48,7 +48,7 @@ public class TestMiniMRDFSCaching extend "/cachedir", mr.createJobConf(), "The quick brown fox\nhas many silly\n" - + "red fox sox\n", false); + + "red fox sox\n"); assertTrue("Archives not matching", ret.isOutputOk); // launch MR cache with symlinks ret = MRCaching.launchMRCache("/testing/wc/input", @@ -56,7 +56,7 @@ public class TestMiniMRDFSCaching extend "/cachedir", mr.createJobConf(), "The quick brown fox\nhas many silly\n" - + "red fox sox\n", true); + + "red fox sox\n"); assertTrue("Archives not matching", ret.isOutputOk); } finally { if (fileSys != null) { Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Thu Aug 9 22:29:36 2012 @@ -211,6 +211,7 @@ public class TestMRJobs { Path outputDir = new Path(mrCluster.getTestWorkDir().getAbsolutePath(), "random-output"); FileOutputFormat.setOutputPath(job, outputDir); + job.setSpeculativeExecution(false); job.addFileToClassPath(APP_JAR); // The AppMaster jar itself. job.setJarByClass(RandomTextWriterJob.class); job.setMaxMapAttempts(1); // speed up failures @@ -462,7 +463,6 @@ public class TestMRJobs { job.addFileToClassPath(APP_JAR); // The AppMaster jar itself. job.addArchiveToClassPath(third); job.addCacheArchive(fourth.toUri()); - job.createSymlink(); job.setMaxMapAttempts(1); // speed up failures job.submit(); Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java Thu Aug 9 22:29:36 2012 @@ -301,7 +301,6 @@ public class TestSpeculativeExecution { // Creates the Job Configuration job.addFileToClassPath(APP_JAR); // The AppMaster jar itself. - job.createSymlink(); job.setMaxMapAttempts(2); job.submit(); Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Thu Aug 9 22:29:36 2012 @@ -55,6 +55,7 @@ import org.apache.hadoop.fs.LocalDirAllo import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; import org.apache.hadoop.security.ssl.SSLFactory; @@ -86,9 +87,7 @@ import org.jboss.netty.channel.ChannelHa import org.jboss.netty.channel.ChannelPipeline; import org.jboss.netty.channel.ChannelPipelineFactory; import org.jboss.netty.channel.Channels; -import org.jboss.netty.channel.DefaultFileRegion; import org.jboss.netty.channel.ExceptionEvent; -import org.jboss.netty.channel.FileRegion; import org.jboss.netty.channel.MessageEvent; import org.jboss.netty.channel.SimpleChannelUpstreamHandler; import org.jboss.netty.channel.group.ChannelGroup; @@ -104,7 +103,6 @@ import org.jboss.netty.handler.codec.htt import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.jboss.netty.handler.codec.http.QueryStringDecoder; import org.jboss.netty.handler.ssl.SslHandler; -import org.jboss.netty.handler.stream.ChunkedFile; import org.jboss.netty.handler.stream.ChunkedWriteHandler; import org.jboss.netty.util.CharsetUtil; @@ -114,6 +112,12 @@ public class ShuffleHandler extends Abst implements AuxServices.AuxiliaryService { private static final Log LOG = LogFactory.getLog(ShuffleHandler.class); + + public static final String SHUFFLE_MANAGE_OS_CACHE = "mapreduce.shuffle.manage.os.cache"; + public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true; + + public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes"; + public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024; private int port; private ChannelFactory selector; @@ -121,6 +125,15 @@ public class ShuffleHandler extends Abst private HttpPipelineFactory pipelineFact; private int sslFileBufferSize; + /** + * Should the shuffle use posix_fadvise calls to manage the OS cache during + * sendfile + */ + private boolean manageOsCache; + private int readaheadLength; + private ReadaheadPool readaheadPool = ReadaheadPool.getInstance(); + + public static final String MAPREDUCE_SHUFFLE_SERVICEID = "mapreduce.shuffle"; @@ -242,6 +255,12 @@ public class ShuffleHandler extends Abst @Override public synchronized void init(Configuration conf) { + manageOsCache = conf.getBoolean(SHUFFLE_MANAGE_OS_CACHE, + DEFAULT_SHUFFLE_MANAGE_OS_CACHE); + + readaheadLength = conf.getInt(SHUFFLE_READAHEAD_BYTES, + DEFAULT_SHUFFLE_READAHEAD_BYTES); + ThreadFactory bossFactory = new ThreadFactoryBuilder() .setNameFormat("ShuffleHandler Netty Boss #%d") .build(); @@ -503,14 +522,14 @@ public class ShuffleHandler extends Abst base + "/file.out", conf); LOG.debug("DEBUG1 " + base + " : " + mapOutputFileName + " : " + indexFileName); - IndexRecord info = + final IndexRecord info = indexCache.getIndexInformation(mapId, reduce, indexFileName, user); final ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce); final DataOutputBuffer dob = new DataOutputBuffer(); header.write(dob); ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); - File spillfile = new File(mapOutputFileName.toString()); + final File spillfile = new File(mapOutputFileName.toString()); RandomAccessFile spill; try { spill = new RandomAccessFile(spillfile, "r"); @@ -520,22 +539,25 @@ public class ShuffleHandler extends Abst } ChannelFuture writeFuture; if (ch.getPipeline().get(SslHandler.class) == null) { - final FileRegion partition = new DefaultFileRegion( - spill.getChannel(), info.startOffset, info.partLength); + final FadvisedFileRegion partition = new FadvisedFileRegion(spill, + info.startOffset, info.partLength, manageOsCache, readaheadLength, + readaheadPool, spillfile.getAbsolutePath()); writeFuture = ch.write(partition); writeFuture.addListener(new ChannelFutureListener() { // TODO error handling; distinguish IO/connection failures, // attribute to appropriate spill output - @Override - public void operationComplete(ChannelFuture future) { - partition.releaseExternalResources(); - } - }); + @Override + public void operationComplete(ChannelFuture future) { + partition.releaseExternalResources(); + } + }); } else { // HTTPS cannot be done with zero copy. - writeFuture = ch.write(new ChunkedFile(spill, info.startOffset, - info.partLength, - sslFileBufferSize)); + final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill, + info.startOffset, info.partLength, sslFileBufferSize, + manageOsCache, readaheadLength, readaheadPool, + spillfile.getAbsolutePath()); + writeFuture = ch.write(chunk); } metrics.shuffleConnections.incr(); metrics.shuffleOutputBytes.incr(info.partLength); // optimistic Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml Thu Aug 9 22:29:36 2012 @@ -98,7 +98,7 @@ org.hsqldb hsqldb - 2.0.0 + provided Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java Thu Aug 9 22:29:36 2012 @@ -65,6 +65,16 @@ import org.hsqldb.server.Server; * * When called with no arguments the program starts a local HSQLDB server, and * uses this database for storing/retrieving the data. + *
+ * This program requires some additional configuration relating to HSQLDB. + * The the hsqldb jar should be added to the classpath: + *
+ * export HADOOP_CLASSPATH=share/hadoop/mapreduce/lib-examples/hsqldb-2.0.0.jar + *
+ * And the hsqldb jar should be included with the -libjars + * argument when executing it with hadoop: + *
+ * -libjars share/hadoop/mapreduce/lib-examples/hsqldb-2.0.0.jar */ public class DBCountPageView extends Configured implements Tool { Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java?rev=1371518&r1=1371517&r2=1371518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java Thu Aug 9 22:29:36 2012 @@ -167,7 +167,6 @@ public class Sort extends Configure URI partitionUri = new URI(partitionFile.toString() + "#" + "_sortPartitioning"); DistributedCache.addCacheFile(partitionUri, conf); - DistributedCache.createSymlink(conf); } System.out.println("Running on " +