hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject svn commit: r783055 [1/6] - in /hadoop/core/branches/HADOOP-3628-2: ./ .eclipse.templates/ ivy/ lib/ lib/jsp-2.1/ src/contrib/ src/contrib/capacity-scheduler/ src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/ src/contrib/capacity-schedu...
Date Tue, 09 Jun 2009 16:11:23 GMT
Author: stevel
Date: Tue Jun  9 16:11:19 2009
New Revision: 783055

URL: http://svn.apache.org/viewvc?rev=783055&view=rev
Log:
sync up with SVN HEAD

Added:
    hadoop/core/branches/HADOOP-3628-2/lib/jsp-2.1/
      - copied from r779668, hadoop/core/trunk/lib/jsp-2.1/
    hadoop/core/branches/HADOOP-3628-2/lib/jsp-2.1/jsp-2.1.jar
      - copied unchanged from r779668, hadoop/core/trunk/lib/jsp-2.1/jsp-2.1.jar
    hadoop/core/branches/HADOOP-3628-2/lib/jsp-2.1/jsp-api-2.1.jar
      - copied unchanged from r779668, hadoop/core/trunk/lib/jsp-2.1/jsp-api-2.1.jar
    hadoop/core/branches/HADOOP-3628-2/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/LocalMySQLManager.java
      - copied unchanged from r782987, hadoop/core/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/LocalMySQLManager.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/LocalMySQLTest.java
      - copied unchanged from r782987, hadoop/core/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/LocalMySQLTest.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingStatus.java
      - copied unchanged from r779668, hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingStatus.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
      - copied unchanged from r780635, hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
      - copied unchanged from r780635, hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapreduce/lib/input/CombineFileSplit.java
      - copied unchanged from r780635, hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/CombineFileSplit.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java
      - copied unchanged from r782987, hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRChildTask.java
      - copied unchanged from r782987, hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRChildTask.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestRunningTaskLimits.java
      - copied unchanged from r782987, hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRunningTaskLimits.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapreduce/lib/input/
      - copied from r780635, hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/lib/input/
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
      - copied unchanged from r780635, hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
Removed:
    hadoop/core/branches/HADOOP-3628-2/lib/commons-cli-2.0-SNAPSHOT.jar
    hadoop/core/branches/HADOOP-3628-2/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestINode.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRTaskTempDir.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java
Modified:
    hadoop/core/branches/HADOOP-3628-2/   (props changed)
    hadoop/core/branches/HADOOP-3628-2/.eclipse.templates/.classpath
    hadoop/core/branches/HADOOP-3628-2/CHANGES.txt   (contents, props changed)
    hadoop/core/branches/HADOOP-3628-2/ivy.xml
    hadoop/core/branches/HADOOP-3628-2/ivy/libraries.properties
    hadoop/core/branches/HADOOP-3628-2/src/contrib/build-contrib.xml
    hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml
    hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/ivy.xml
    hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/MemoryMatcher.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/eclipse-plugin/build.xml
    hadoop/core/branches/HADOOP-3628-2/src/contrib/hdfsproxy/bin/hdfsproxy
    hadoop/core/branches/HADOOP-3628-2/src/contrib/hdfsproxy/build.xml
    hadoop/core/branches/HADOOP-3628-2/src/contrib/hdfsproxy/ivy.xml
    hadoop/core/branches/HADOOP-3628-2/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/mapred/ImportJob.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/orm/CompilationManager.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/AllTests.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/streaming/ivy.xml
    hadoop/core/branches/HADOOP-3628-2/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
    hadoop/core/branches/HADOOP-3628-2/src/contrib/streaming/src/test/org/apache/hadoop/streaming/StderrApp.java
    hadoop/core/branches/HADOOP-3628-2/src/core/core-default.xml
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/filecache/DistributedCache.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/fs/FsShell.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/io/IOUtils.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/metrics/ContextFactory.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/util/GenericOptionsParser.java
    hadoop/core/branches/HADOOP-3628-2/src/core/org/apache/hadoop/util/ProcfsBasedProcessTree.java
    hadoop/core/branches/HADOOP-3628-2/src/docs/src/documentation/content/xdocs/hdfs_design.xml
    hadoop/core/branches/HADOOP-3628-2/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
    hadoop/core/branches/HADOOP-3628-2/src/examples/org/apache/hadoop/examples/MultiFileWordCount.java
    hadoop/core/branches/HADOOP-3628-2/src/examples/org/apache/hadoop/examples/Sort.java
    hadoop/core/branches/HADOOP-3628-2/src/examples/org/apache/hadoop/examples/dancing/DistributedPentomino.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/common/StorageInfo.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DatanodeBlockInfo.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INode.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
    hadoop/core/branches/HADOOP-3628-2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/mapred-default.xml
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/JobClient.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/JobConf.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/JobHistory.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/JobTracker.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/MRConstants.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/MapTask.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/Merger.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskMemoryManagerThread.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
    hadoop/core/branches/HADOOP-3628-2/src/mapred/org/apache/hadoop/mapred/lib/CombineFileSplit.java
    hadoop/core/branches/HADOOP-3628-2/src/test/bin/test-patch.sh
    hadoop/core/branches/HADOOP-3628-2/src/test/core/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
    hadoop/core/branches/HADOOP-3628-2/src/test/core/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
    hadoop/core/branches/HADOOP-3628-2/src/test/core/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
    hadoop/core/branches/HADOOP-3628-2/src/test/core/org/apache/hadoop/util/TestProcfsBasedProcessTree.java
    hadoop/core/branches/HADOOP-3628-2/src/test/findbugsExcludeFile.xml
    hadoop/core/branches/HADOOP-3628-2/src/test/hdfs-with-mr/org/apache/hadoop/fs/TestCopyFiles.java
    hadoop/core/branches/HADOOP-3628-2/src/test/hdfs-with-mr/org/apache/hadoop/hdfs/NNBench.java
    hadoop/core/branches/HADOOP-3628-2/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSShellGenericOptions.java
    hadoop/core/branches/HADOOP-3628-2/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java
    hadoop/core/branches/HADOOP-3628-2/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java
    hadoop/core/branches/HADOOP-3628-2/src/test/hdfs/org/apache/hadoop/hdfs/TestFileStatus.java
    hadoop/core/branches/HADOOP-3628-2/src/test/hdfs/org/apache/hadoop/hdfs/TestSeekBug.java
    hadoop/core/branches/HADOOP-3628-2/src/test/hdfs/org/apache/hadoop/hdfs/TestSmallBlock.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerRestart.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
    hadoop/core/branches/HADOOP-3628-2/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
    hadoop/core/branches/HADOOP-3628-2/src/tools/org/apache/hadoop/tools/DistCp.java
    hadoop/core/branches/HADOOP-3628-2/src/webapps/job/jobdetails.jsp

Propchange: hadoop/core/branches/HADOOP-3628-2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  9 16:11:19 2009
@@ -1 +1,2 @@
 /hadoop/core/branches/branch-0.19:713112
+/hadoop/core/trunk:779103-783032

Modified: hadoop/core/branches/HADOOP-3628-2/.eclipse.templates/.classpath
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/.eclipse.templates/.classpath?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/.eclipse.templates/.classpath (original)
+++ hadoop/core/branches/HADOOP-3628-2/.eclipse.templates/.classpath Tue Jun  9 16:11:19 2009
@@ -14,11 +14,10 @@
 	<classpathentry kind="src" path="src/contrib/streaming/src/test"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="var" path="ANT_HOME/lib/ant.jar"/>
-	<classpathentry kind="lib" path="lib/commons-cli-2.0-SNAPSHOT.jar"/>
 	<classpathentry kind="lib" path="lib/hsqldb-1.8.0.10.jar"/>
 	<classpathentry kind="lib" path="lib/kfs-0.2.2.jar"/>
-	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/jsp-2.1-6.1.14.jar"/>
-	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/jsp-api-2.1-6.1.14.jar"/>
+	<classpathentry kind="lib" path="lib/jsp-2.1/jsp-2.1.jar"/>
+	<classpathentry kind="lib" path="lib/jsp-2.1/jsp-api-2.1.jar"/>
 	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/commons-codec-1.3.jar"/>
 	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/commons-httpclient-3.0.1.jar"/>
 	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/commons-el-1.0.jar"/>
@@ -38,6 +37,8 @@
 	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/slf4j-api-1.4.3.jar"/>
 	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/slf4j-log4j12-1.4.3.jar"/>
 	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/xmlenc-0.52.jar"/>
+	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/commons-cli-1.2.jar"/>
+	<classpathentry kind="lib" path="build/ivy/lib/Hadoop/common/commons-cli-2.0-mahout.jar"/>
 	<classpathentry kind="lib" path="src/test/lib/ftplet-api-1.0.0-SNAPSHOT.jar"/>
 	<classpathentry kind="lib" path="src/test/lib/ftpserver-core-1.0.0-SNAPSHOT.jar"/>
 	<classpathentry kind="lib" path="src/test/lib/ftpserver-server-1.0.0-SNAPSHOT.jar"/>

Modified: hadoop/core/branches/HADOOP-3628-2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/CHANGES.txt?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/CHANGES.txt (original)
+++ hadoop/core/branches/HADOOP-3628-2/CHANGES.txt Tue Jun  9 16:11:19 2009
@@ -131,6 +131,12 @@
     HADOOP-4861. Add disk usage with human-readable size (-duh).
     (Todd Lipcon via tomwhite)
 
+    HADOOP-5844. Use mysqldump when connecting to local mysql instance in Sqoop.
+    (Aaron Kimball via tomwhite)
+
+    HADOOP-5170. Allows jobs to set max maps/reduces per-node and per-cluster.
+    (Matei Zaharia via ddas)
+
   IMPROVEMENTS
 
     HADOOP-4565. Added CombineFileInputFormat to use data locality information
@@ -349,8 +355,6 @@
 
     HADOOP-5780. Improve per block message prited by "-metaSave" in HDFS.
     (Raghu Angadi)
-   
-    HADOOP-5792. To resolve jsp-2.1 jars through ivy (Giridharan Kesavan) 
 
     HADOOP-5823. Added a new class DeprecatedUTF8 to help with removing
     UTF8 related javac warnings. These warnings are removed in 
@@ -392,12 +396,48 @@
     TestNameEditsConfig, TestStartup and TestStorageRestore.
     (Jakob Homan via shv)
 
-    HADOOP-5438. Provide a single FileSystem method to create or open-for-append 
+    HADOOP-5438. Provide a single FileSystem method to create or open-for-append
     to a file.  (He Yongqiang via dhruba)
 
     HADOOP-5472. Change DistCp to support globbing of input paths.  (Dhruba
     Borthakur and Rodrigo Schmidt via szetszwo)
 
+    HADOOP-5175. Don't unpack libjars on classpath. (Todd Lipcon via tomwhite)
+
+    HADOOP-5620. Add an option to DistCp for preserving modification and access
+    times.  (Rodrigo Schmidt via szetszwo)
+
+    HADOOP-5664. Change map serialization so a lock is obtained only where
+    contention is possible, rather than for each write. (cdouglas)
+
+    HADOOP-5896. Remove the dependency of GenericOptionsParser on 
+    Option.withArgPattern. (Giridharan Kesavan and Sharad Agarwal via 
+    sharad)
+
+    HADOOP-5698. Change org.apache.hadoop.examples.MultiFileWordCount to 
+    use new mapreduce api. (Amareshwari Sriramadasu via sharad)
+
+    HADOOP-5696. Change org.apache.hadoop.examples.Sort to use new 
+    mapreduce api. (Amareshwari Sriramadasu via sharad)
+
+    HADOOP-5784. Makes the number of heartbeats that should arrive a second
+    at the JobTracker configurable. (Amareshwari Sriramadasu via ddas)
+
+    HADOOP-5955. Changes TestFileOuputFormat so that is uses LOCAL_MR
+    instead of CLUSTER_MR. (Jothi Padmanabhan via das)
+
+    HADOOP-5948. Changes TestJavaSerialization to use LocalJobRunner 
+    instead of MiniMR/DFS cluster. (Jothi Padmanabhan via das)
+
+    HADOOP-2838. Add mapred.child.env to pass environment variables to 
+    tasktracker's child processes. (Amar Kamat via sharad)
+
+    HADOOP-5961. DataNode process understand generic hadoop command line
+    options (like -Ddfs.property=value). (Raghu Angadi)
+
+    HADOOP-5694. Change org.apache.hadoop.examples.dancing to use new 
+    mapreduce api. (Amareshwari Sriramadasu via sharad)
+
   OPTIMIZATIONS
 
     HADOOP-5595. NameNode does not need to run a replicator to choose a
@@ -695,8 +735,68 @@
     HADOOP-5635. Change distributed cache to work with other distributed file
     systems. (Andrew Hitchcock via tomwhite)
 
-    HADOOP-5700. INode.getPathComponents throws NPE when given a non-absolute
-    path. (Todd Lipcon via tomwhite)
+    HADOOP-5856. Fix "unsafe multithreaded use of DateFormat" findbugs warning
+    in DataBlockScanner.  (Kan Zhang via szetszwo)
+
+    HADOOP-4864. Fixes a problem to do with -libjars with multiple jars when
+    client and cluster reside on different OSs. (Amareshwari Sriramadasu via ddas)
+
+    HADOOP-5623. Fixes a problem to do with status messages getting overwritten
+    in streaming jobs. (Rick Cox and Jothi Padmanabhan via ddas)
+
+    HADOOP-5895. Fixes computation of count of merged bytes for logging.
+    (Ravi Gummadi via ddas)
+
+    HADOOP-5805. problem using top level s3 buckets as input/output directories.
+    (Ian Nowland via tomwhite)
+   
+    HADOOP-5940. trunk eclipse-plugin build fails while trying to copy 
+    commons-cli jar from the lib dir (Giridharan Kesavan via gkesavan)
+
+    HADOOP-5864. Fix DMI and OBL findbugs in packages hdfs and metrics.
+    (hairong)
+
+    HADOOP-5935. Fix Hudson's release audit warnings link is broken. 
+    (Giridharan Kesavan via gkesavan)
+
+    HADOOP-5947. Delete empty TestCombineFileInputFormat.java
+
+    HADOOP-5899. Move a log message in FSEditLog to the right place for
+    avoiding unnecessary log.  (Suresh Srinivas via szetszwo)
+
+    HADOOP-5944. Add Apache license header to BlockManager.java.  (Suresh
+    Srinivas via szetszwo)
+
+    HADOOP-5891. SecondaryNamenode is able to converse with the NameNode 
+    even when the default value of dfs.http.address is not overridden.
+    (Todd Lipcon via dhruba)
+
+    HADOOP-5953. The isDirectory(..) and isFile(..) methods in KosmosFileSystem
+    should not be deprecated.  (szetszwo)
+
+    HADOOP-5954. Fix javac warnings in TestFileCreation, TestSmallBlock,
+    TestFileStatus, TestDFSShellGenericOptions, TestSeekBug and
+    TestDFSStartupVersions.  (szetszwo)
+
+    HADOOP-5956. Fix ivy dependency in hdfsproxy and capacity-scheduler.
+    (Giridharan Kesavan via szetszwo)
+
+    HADOOP-5836. Bug in S3N handling of directory markers using an object with
+    a trailing "/" causes jobs to fail. (Ian Nowland via tomwhite)
+
+    HADOOP-5861. s3n files are not getting split by default. (tomwhite)
+
+    HADOOP-5762. Fix a problem that DistCp does not copy empty directory.
+    (Rodrigo Schmidt via szetszwo)
+
+    HADOOP-5859. Fix "wait() or sleep() with locks held" findbugs warnings in
+    DFSClient.  (Kan Zhang via szetszwo)
+   
+    HADOOP-5457. Fix to continue to run builds even if contrib test fails
+    (Giridharan Kesavan via gkesavan)
+
+    HADOOP-5963. Remove an unnecessary exception catch in NNBench.  (Boris
+    Shkolnik via szetszwo)
 
 Release 0.20.1 - Unreleased
 
@@ -780,6 +880,42 @@
     HADOOP-5850. Fixes a problem to do with not being able to jobs with
     0 maps/reduces. (Vinod K V via ddas)
 
+    HADOOP-4626. Correct the API links in hdfs forrest doc so that they
+    point to the same version of hadoop.  (szetszwo)
+
+    HADOOP-5883. Fixed tasktracker memory monitoring to account for
+    momentary spurts in memory usage due to java's fork() model.
+    (yhemanth)
+
+    HADOOP-5539. Fixes a problem to do with not preserving intermediate
+    output compression for merged data.
+    (Jothi Padmanabhan and Billy Pearson via ddas)
+
+    HADOOP-5932. Fixes a problem in capacity scheduler in computing
+    available memory on a tasktracker.
+    (Vinod Kumar Vavilapalli via yhemanth)
+
+    HADOOP-5937. Correct a safemode message in FSNamesystem.  (Ravi Phulari
+    via szetszwo)
+
+    HADOOP-5908. Fixes a problem to do with ArithmeticException in the 
+    JobTracker when there are jobs with 0 maps. (Amar Kamat via ddas)
+
+    HADOOP-5924. Fixes a corner case problem to do with job recovery with
+    empty history files. Also, after a JT restart, sends KillTaskAction to 
+    tasks that report back but the corresponding job hasn't been initialized
+    yet. (Amar Kamat via ddas)
+
+    HADOOP-5882. Fixes a reducer progress update problem for new mapreduce
+    api. (Amareshwari Sriramadasu via sharad)
+
+    HADOOP-5746. Fixes a corner case problem in Streaming, where if an exception
+    happens in MROutputThread after the last call to the map/reduce method, the 
+    exception goes undetected. (Amar Kamat via ddas)
+
+    HADOOP-5884. Fixes accounting in capacity scheduler so that high RAM jobs
+    take more slots. (Vinod Kumar Vavilapalli via yhemanth)
+
 Release 0.20.0 - 2009-04-15
 
   INCOMPATIBLE CHANGES
@@ -1544,7 +1680,7 @@
     (Rahul Kumar Singh via yhemanth)
 
     HADOOP-5328. Fixes a problem in the renaming of job history files during 
-    job recovery. Amar Kamat via ddas)
+    job recovery. (Amar Kamat via ddas)
 
     HADOOP-5417. Don't ignore InterruptedExceptions that happen when calling 
     into rpc. (omalley)
@@ -1657,7 +1793,7 @@
     (ddas)
 
     HADOOP-5392. Fixes a problem to do with JT crashing during recovery when
-    the job files are garbled. (Amar Kamat vi ddas)
+    the job files are garbled. (Amar Kamat via ddas)
 
     HADOOP-5332. Appending to files is not allowed (by default) unless
     dfs.support.append is set to true. (dhruba)
@@ -1718,6 +1854,9 @@
     HADOOP-5816. Fixes a problem in the KeyFieldBasedComparator to do with
     ArrayIndexOutOfBounds exception. (He Yongqiang via ddas)
 
+    HADOOP-5951. Add Apache license header to StorageInfo.java.  (Suresh
+    Srinivas via szetszwo)
+
 Release 0.19.1 - 2009-02-23 
 
   IMPROVEMENTS
@@ -2361,7 +2500,7 @@
     (Lohit Vijayarenu via mahadev)
 
     HADOOP-3778. DFSInputStream.seek() did not retry in case of some errors.
-    (LN via rangadi)
+    (Luo Ning via rangadi)
 
     HADOOP-3661. The handling of moving files deleted through fuse-dfs to
     Trash made similar to the behaviour from dfs shell.

Propchange: hadoop/core/branches/HADOOP-3628-2/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jun  9 16:11:19 2009
@@ -1,2 +1,3 @@
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
+/hadoop/core/trunk/CHANGES.txt:779103-783032

Modified: hadoop/core/branches/HADOOP-3628-2/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/ivy.xml?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/ivy.xml (original)
+++ hadoop/core/branches/HADOOP-3628-2/ivy.xml Tue Jun  9 16:11:19 2009
@@ -78,13 +78,14 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
-
- <!--used client side-->
-<!--  <dependency org="commons-cli"
-    name="commons-cli"
-    rev="${commons-cli.version}"
-    conf="client->default"/> -->
-
+    <dependency org="org.apache.mahout.commons"
+      name="commons-cli"
+      rev="${commons-cli2.version}"
+      conf="common->default"/>
+    <dependency org="commons-cli"
+      name="commons-cli"
+      rev="${commons-cli.version}"
+      conf="common->default"/>
     <dependency org="checkstyle"
       name="checkstyle"
       rev="${checkstyle.version}"
@@ -98,7 +99,6 @@
       rev="${xerces.version}"
       conf="jdiff->default">
     </dependency>
-
     <dependency org="xmlenc"
       name="xmlenc"
       rev="${xmlenc.version}"
@@ -149,14 +149,11 @@
       name="jasper-compiler"
       rev="${jasper.version}"
       conf="jetty->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jsp-api-2.1"
-      rev="${jetty.version}"
-      conf="jetty->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jsp-2.1"
-      rev="${jetty.version}"
-      conf="jetty->master"/>
+<!-- this is resolved locally from the lib folder 
+   <dependency org="tomcat"
+      name="jsp-api"
+      rev="${jsp-api.version}"
+      conf="jetty->master"/> -->
     <dependency org="commons-el"
       name="commons-el"
       rev="${commons-el.version}"

Modified: hadoop/core/branches/HADOOP-3628-2/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/ivy/libraries.properties?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/ivy/libraries.properties (original)
+++ hadoop/core/branches/HADOOP-3628-2/ivy/libraries.properties Tue Jun  9 16:11:19 2009
@@ -18,7 +18,8 @@
 
 checkstyle.version=4.2
 
-commons-cli.version=2.0-SNAPSHOT
+commons-cli.version=1.2
+commons-cli2.version=2.0-mahout
 commons-codec.version=1.3
 commons-collections.version=3.1
 commons-httpclient.version=3.0.1

Modified: hadoop/core/branches/HADOOP-3628-2/src/contrib/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/contrib/build-contrib.xml?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/src/contrib/build-contrib.xml (original)
+++ hadoop/core/branches/HADOOP-3628-2/src/contrib/build-contrib.xml Tue Jun  9 16:11:19 2009
@@ -41,6 +41,7 @@
   <property name="test.junit.output.format" value="plain"/>
   <property name="test.output" value="no"/>
   <property name="test.timeout" value="900000"/>
+  <property name="build.contrib.dir" location="${hadoop.root}/build/contrib"/>
   <property name="build.dir" location="${hadoop.root}/build/contrib/${name}"/>
   <property name="build.classes" location="${build.dir}/classes"/>
   <property name="build.test" location="${build.dir}/test"/>
@@ -247,7 +248,12 @@
         <fileset dir="${src.test}" includes="**/${testcase}.java"/>
       </batchtest>
     </junit>
-    <fail if="tests.failed">Tests failed!</fail>
+    <antcall target="checkfailure"/>
+  </target>
+
+  <target name="checkfailure" if="tests.failed">
+    <touch file="${build.contrib.dir}/testsfailed"/>
+    <fail unless="continueOnFailure">Contrib Tests failed!</fail>
   </target>
 
   <!-- ================================================================== -->

Modified: hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml (original)
+++ hadoop/core/branches/HADOOP-3628-2/src/contrib/build.xml Tue Jun  9 16:11:19 2009
@@ -45,16 +45,21 @@
   <!-- Test all the contribs.                               -->
   <!-- ====================================================== -->
   <target name="test">
+    <property name="hadoop.root" location="${root}/../../../"/>
+    <property name="build.contrib.dir" location="${hadoop.root}/build/contrib"/>
+    <delete file="${build.contrib.dir}/testsfailed"/>
     <subant target="test">
+      <property name="continueOnFailure" value="true"/>
       <fileset dir="." includes="hdfsproxy/build.xml"/>
       <fileset dir="." includes="streaming/build.xml"/>
       <fileset dir="." includes="fairscheduler/build.xml"/>
       <fileset dir="." includes="capacity-scheduler/build.xml"/>
       <fileset dir="." includes="mrunit/build.xml"/>
     </subant>
+    <available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
+    <fail if="testsfailed">Tests failed!</fail>
   </target>
   
-  
   <!-- ====================================================== -->
   <!-- Clean all the contribs.                              -->
   <!-- ====================================================== -->

Modified: hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/ivy.xml?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/ivy.xml (original)
+++ hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/ivy.xml Tue Jun  9 16:11:19 2009
@@ -24,6 +24,10 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+    <dependency org="commons-cli"
+      name="commons-cli"
+      rev="${commons-cli.version}"
+      conf="common->default"/>
     <dependency org="commons-logging"
       name="commons-logging"
       rev="${commons-logging.version}"

Modified: hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java (original)
+++ hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java Tue Jun  9 16:11:19 2009
@@ -164,10 +164,10 @@
    * Sets the capacity of the given queue.
    * 
    * @param queue name of the queue
-   * @param gc percent of the cluster for the queue.
+   * @param capacity percent of the cluster for the queue.
    */
-  public void setCapacity(String queue,float gc) {
-    rmConf.setFloat(toFullPropertyName(queue, "capacity"),gc);
+  public void setCapacity(String queue,float capacity) {
+    rmConf.setFloat(toFullPropertyName(queue, "capacity"),capacity);
   }
   
   /**

Modified: hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java (original)
+++ hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Tue Jun  9 16:11:19 2009
@@ -24,8 +24,6 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -34,8 +32,6 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.JobTracker.IllegalStateException;
-import org.apache.hadoop.util.StringUtils;
-
 
 /**
  * A {@link TaskScheduler} that implements the requirements in HADOOP-3421
@@ -78,14 +74,20 @@
 
   private static class TaskSchedulingInfo {
     /** 
-     * the actual gc, which depends on how many slots are available
+     * the actual capacity, which depends on how many slots are available
      * in the cluster at any given time. 
      */
     int capacity = 0;
     // number of running tasks
     int numRunningTasks = 0;
-    /** for each user, we need to keep track of number of running tasks */
-    Map<String, Integer> numRunningTasksByUser = 
+    // number of slots occupied by running tasks
+    int numSlotsOccupied = 0;
+
+    /**
+     * for each user, we need to keep track of number of slots occupied by
+     * running tasks
+     */
+    Map<String, Integer> numSlotsOccupiedByUser = 
       new HashMap<String, Integer>();
     
     /**
@@ -93,32 +95,41 @@
      */
     void resetTaskVars() {
       numRunningTasks = 0;
-      for (String s: numRunningTasksByUser.keySet()) {
-        numRunningTasksByUser.put(s, 0);
+      numSlotsOccupied = 0;
+      for (String s: numSlotsOccupiedByUser.keySet()) {
+        numSlotsOccupiedByUser.put(s, Integer.valueOf(0));
       }
     }
 
     /**
      * return information about the tasks
      */
-    public String toString(){
-      float runningTasksAsPercent = capacity!= 0 ?
-          ((float)numRunningTasks * 100/capacity):0;
+    @Override
+    public String toString() {
+      float occupiedSlotsAsPercent =
+          capacity != 0 ? ((float) numSlotsOccupied * 100 / capacity) : 0;
       StringBuffer sb = new StringBuffer();
-      sb.append("Capacity: " + capacity + "\n");
-      sb.append(String.format("Running tasks: %.1f%% of Capacity\n",
-          runningTasksAsPercent));
+      sb.append("Capacity: " + capacity + " slots\n");
+      sb.append(String.format("Used capacity: %d (%.1f%% of Capacity)\n",
+          Integer.valueOf(numSlotsOccupied), Float
+              .valueOf(occupiedSlotsAsPercent)));
+      sb.append(String.format("Running tasks: %d\n", Integer
+          .valueOf(numRunningTasks)));
       // include info on active users
-      if (numRunningTasks != 0) {
+      if (numSlotsOccupied != 0) {
         sb.append("Active users:\n");
-        for (Map.Entry<String, Integer> entry: numRunningTasksByUser.entrySet()) {
+        for (Map.Entry<String, Integer> entry : numSlotsOccupiedByUser
+            .entrySet()) {
           if ((entry.getValue() == null) || (entry.getValue().intValue() <= 0)) {
             // user has no tasks running
             continue;
           }
-          sb.append("User '" + entry.getKey()+ "': ");
-          float p = (float)entry.getValue().intValue()*100/numRunningTasks;
-          sb.append(String.format("%.1f%% of running tasks\n", p));
+          sb.append("User '" + entry.getKey() + "': ");
+          int numSlotsOccupiedByThisUser = entry.getValue().intValue();
+          float p =
+              (float) numSlotsOccupiedByThisUser * 100 / numSlotsOccupied;
+          sb.append(String.format("%d (%.1f%% of used capacity)\n", Long
+              .valueOf(numSlotsOccupiedByThisUser), Float.valueOf(p)));
         }
       }
       return sb.toString();
@@ -152,10 +163,10 @@
     TaskSchedulingInfo mapTSI;
     TaskSchedulingInfo reduceTSI;
     
-    public QueueSchedulingInfo(String queueName, float gcPercent, 
+    public QueueSchedulingInfo(String queueName, float capacityPercent, 
         int ulMin, JobQueuesManager jobQueuesManager) {
       this.queueName = new String(queueName);
-      this.capacityPercent = gcPercent;
+      this.capacityPercent = capacityPercent;
       this.ulMin = ulMin;
       this.jobQueuesManager = jobQueuesManager;
       this.mapTSI = new TaskSchedulingInfo();
@@ -164,13 +175,14 @@
     
     /**
      * return information about the queue
+     * @return a String representing the information about the queue.
      */
+    @Override
     public String toString(){
       // We print out the queue information first, followed by info
       // on map and reduce tasks and job info
       StringBuffer sb = new StringBuffer();
       sb.append("Queue configuration\n");
-      //sb.append("Name: " + queueName + "\n");
       sb.append("Capacity Percentage: ");
       sb.append(capacityPercent);
       sb.append("%\n");
@@ -281,7 +293,15 @@
     protected CapacityTaskScheduler.TYPE type = null;
 
     abstract Task obtainNewTask(TaskTrackerStatus taskTracker, 
-        JobInProgress job) throws IOException; 
+        JobInProgress job) throws IOException;
+
+    int getSlotsOccupied(JobInProgress job) {
+      return getRunningTasks(job) * getSlotsPerTask(job);
+    }
+
+    abstract int getClusterCapacity();
+    abstract int getSlotsPerTask(JobInProgress job);
+    abstract int getRunningTasks(JobInProgress job);
     abstract int getPendingTasks(JobInProgress job);
     abstract TaskSchedulingInfo getTSI(QueueSchedulingInfo qsi);
     /**
@@ -313,12 +333,12 @@
       public int compare(QueueSchedulingInfo q1, QueueSchedulingInfo q2) {
         TaskSchedulingInfo t1 = getTSI(q1);
         TaskSchedulingInfo t2 = getTSI(q2);
-        // look at how much capacity they've filled. Treat a queue with gc=0 
-        // equivalent to a queue running at capacity
+        // look at how much capacity they've filled. Treat a queue with
+        // capacity=0 equivalent to a queue running at capacity
         double r1 = (0 == t1.capacity)? 1.0f:
-          (double)t1.numRunningTasks/(double)t1.capacity;
+          (double)t1.numSlotsOccupied/(double)t1.capacity;
         double r2 = (0 == t2.capacity)? 1.0f:
-          (double)t2.numRunningTasks/(double)t2.capacity;
+          (double)t2.numSlotsOccupied/(double)t2.capacity;
         if (r1<r2) return -1;
         else if (r1>r2) return 1;
         else return 0;
@@ -340,7 +360,17 @@
     protected final static ReduceQueueComparator reduceComparator = new ReduceQueueComparator();
     // and this is the comparator to use
     protected QueueComparator queueComparator;
-   
+
+    // Returns queues sorted according to the QueueComparator.
+    // Mainly for testing purposes.
+    String[] getOrderedQueues() {
+      List<String> queues = new ArrayList<String>(qsiForAssigningTasks.size());
+      for (QueueSchedulingInfo qsi : qsiForAssigningTasks) {
+        queues.add(qsi.queueName);
+      }
+      return queues.toArray(new String[queues.size()]);
+    }
+
     TaskSchedulingMgr(CapacityTaskScheduler sched) {
       scheduler = sched;
     }
@@ -357,24 +387,26 @@
     }
 
 
-    private boolean isUserOverLimit(String user, QueueSchedulingInfo qsi) {
-      // what is our current capacity? It's capacity if we're running below capacity.
-      // If we're running over capacity, then its #running plus 1 (which is the
-      // extra slot we're getting). 
+    private boolean isUserOverLimit(JobInProgress j, QueueSchedulingInfo qsi) {
+      // what is our current capacity? It is equal to the queue-capacity if
+      // we're running below capacity. If we're running over capacity, then its
+      // #running plus slotPerTask of the job (which is the number of extra
+      // slots we're getting).
       int currentCapacity;
       TaskSchedulingInfo tsi = getTSI(qsi);
-      if (tsi.numRunningTasks < tsi.capacity) {
+      if (tsi.numSlotsOccupied < tsi.capacity) {
         currentCapacity = tsi.capacity;
       }
       else {
-        currentCapacity = tsi.numRunningTasks+1;
+        currentCapacity = tsi.numSlotsOccupied + getSlotsPerTask(j);
       }
       int limit = Math.max((int)(Math.ceil((double)currentCapacity/
           (double)qsi.numJobsByUser.size())), 
           (int)(Math.ceil((double)(qsi.ulMin*currentCapacity)/100.0)));
-      if (tsi.numRunningTasksByUser.get(user) >= limit) {
-        LOG.debug("User " + user + " is over limit, num running tasks = " + 
-            tsi.numRunningTasksByUser.get(user) + ", limit = " + limit);
+      String user = j.getProfile().getUser();
+      if (tsi.numSlotsOccupiedByUser.get(user) >= limit) {
+        LOG.debug("User " + user + " is over limit, num slots occupied = " + 
+            tsi.numSlotsOccupiedByUser.get(user) + ", limit = " + limit);
         return true;
       }
       else {
@@ -403,7 +435,7 @@
           continue;
         }
         // check if the job's user is over limit
-        if (isUserOverLimit(j.getProfile().getUser(), qsi)) {
+        if (isUserOverLimit(j, qsi)) {
           continue;
         } 
         //If this job meets memory requirements. Ask the JobInProgress for
@@ -489,8 +521,11 @@
     // The caller is responsible for ensuring that the QSI objects and the 
     // collections are up-to-date.
     private TaskLookupResult assignTasks(TaskTrackerStatus taskTracker) throws IOException {
+
+      printQSIs();
+
       for (QueueSchedulingInfo qsi : qsiForAssigningTasks) {
-        // we may have queues with gc=0. We shouldn't look at jobs from 
+        // we may have queues with capacity=0. We shouldn't look at jobs from 
         // these queues
         if (0 == getTSI(qsi).capacity) {
           continue;
@@ -516,20 +551,23 @@
       // nothing to give
       return TaskLookupResult.getNoTaskFoundResult();
     }
-    
+
     // for debugging.
     private void printQSIs() {
-      StringBuffer s = new StringBuffer();
-      for (QueueSchedulingInfo qsi: qsiForAssigningTasks) {
-        TaskSchedulingInfo tsi = getTSI(qsi);
-        Collection<JobInProgress> runJobs = 
-          scheduler.jobQueuesManager.getRunningJobQueue(qsi.queueName);
-        s.append(" Queue '" + qsi.queueName + "'(" + this.type + "): run=" + 
-            tsi.numRunningTasks + ", gc=" + tsi.capacity
-            + ", run jobs="+ runJobs.size() +
-            "*** ");
+      if (LOG.isDebugEnabled()) {
+        StringBuffer s = new StringBuffer();
+        for (QueueSchedulingInfo qsi : qsiForAssigningTasks) {
+          TaskSchedulingInfo tsi = getTSI(qsi);
+          Collection<JobInProgress> runJobs =
+              scheduler.jobQueuesManager.getRunningJobQueue(qsi.queueName);
+          s.append(String.format(" Queue '%s'(%s): runningTasks=%d, "
+              + "occupiedSlots=%d, capacity=%d, runJobs=%d", qsi.queueName,
+              this.type, Integer.valueOf(tsi.numRunningTasks), Integer
+                  .valueOf(tsi.numSlotsOccupied), Integer
+                  .valueOf(tsi.capacity), Integer.valueOf(runJobs.size())));
+        }
+        LOG.debug(s);
       }
-      LOG.debug(s);
     }
     
     /**
@@ -559,11 +597,14 @@
    * The scheduling algorithms for map tasks. 
    */
   private static class MapSchedulingMgr extends TaskSchedulingMgr {
-    MapSchedulingMgr(CapacityTaskScheduler dad) {
-      super(dad);
+
+    MapSchedulingMgr(CapacityTaskScheduler schedulr) {
+      super(schedulr);
       type = CapacityTaskScheduler.TYPE.MAP;
       queueComparator = mapComparator;
     }
+
+    @Override
     Task obtainNewTask(TaskTrackerStatus taskTracker, JobInProgress job) 
     throws IOException {
       ClusterStatus clusterStatus = 
@@ -572,16 +613,30 @@
       return job.obtainNewMapTask(taskTracker, numTaskTrackers, 
           scheduler.taskTrackerManager.getNumberOfUniqueHosts());
     }
+
+    @Override
     int getClusterCapacity() {
       return scheduler.taskTrackerManager.getClusterStatus().getMaxMapTasks();
     }
+
+    @Override
     int getRunningTasks(JobInProgress job) {
       return job.runningMaps();
     }
+
+    @Override
     int getPendingTasks(JobInProgress job) {
       return job.pendingMaps();
     }
 
+    @Override
+    int getSlotsPerTask(JobInProgress job) {
+      long myVmem = job.getJobConf().getMemoryForMapTask();
+      return (int) (Math.ceil((float) myVmem
+          / (float) scheduler.getMemSizeForMapSlot()));
+    }
+
+    @Override
     TaskSchedulingInfo getTSI(QueueSchedulingInfo qsi) {
       return qsi.mapTSI;
     }
@@ -601,11 +656,14 @@
    * The scheduling algorithms for reduce tasks. 
    */
   private static class ReduceSchedulingMgr extends TaskSchedulingMgr {
-    ReduceSchedulingMgr(CapacityTaskScheduler dad) {
-      super(dad);
+
+    ReduceSchedulingMgr(CapacityTaskScheduler schedulr) {
+      super(schedulr);
       type = CapacityTaskScheduler.TYPE.REDUCE;
       queueComparator = reduceComparator;
     }
+
+    @Override
     Task obtainNewTask(TaskTrackerStatus taskTracker, JobInProgress job) 
     throws IOException {
       ClusterStatus clusterStatus = 
@@ -614,16 +672,31 @@
       return job.obtainNewReduceTask(taskTracker, numTaskTrackers, 
           scheduler.taskTrackerManager.getNumberOfUniqueHosts());
     }
+
+    @Override
     int getClusterCapacity() {
-      return scheduler.taskTrackerManager.getClusterStatus().getMaxReduceTasks();
+      return scheduler.taskTrackerManager.getClusterStatus()
+          .getMaxReduceTasks();
     }
+
+    @Override
     int getRunningTasks(JobInProgress job) {
       return job.runningReduces();
     }
+
+    @Override
     int getPendingTasks(JobInProgress job) {
       return job.pendingReduces();
     }
 
+    @Override
+    int getSlotsPerTask(JobInProgress job) {
+      long myVmem = job.getJobConf().getMemoryForReduceTask();
+      return (int) (Math.ceil((float) myVmem
+          / (float) scheduler.getMemSizeForReduceSlot()));
+    }
+
+    @Override
     TaskSchedulingInfo getTSI(QueueSchedulingInfo qsi) {
       return qsi.reduceTSI;
     }
@@ -655,7 +728,10 @@
   protected CapacitySchedulerConf schedConf;
   /** whether scheduler has started or not */
   private boolean started = false;
-  
+
+  static String JOB_SCHEDULING_INFO_FORMAT_STRING =
+      "%s running map tasks using %d map slots,"
+          + " %s running reduce tasks using %d reduce slots.";
   /**
    * A clock class - can be mocked out for testing.
    */
@@ -664,6 +740,7 @@
       return System.currentTimeMillis();
     }
   }
+
   // can be replaced with a global type, if we have one
   protected static enum TYPE {
     MAP, REDUCE
@@ -709,12 +786,13 @@
         JobConf.normalizeMemoryConfigValue(conf.getLong(
             JobTracker.MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB_PROPERTY,
             JobConf.DISABLED_MEMORY_LIMIT));
-    LOG.info(new StringBuilder().append("Scheduler configured with ").append(
-        "(memSizeForMapSlotOnJT, memSizeForReduceSlotOnJT,").append(
-        " limitMaxMemForMapTasks, limitMaxMemForReduceTasks)").append(
-        memSizeForMapSlotOnJT).append(", ").append(memSizeForReduceSlotOnJT)
-        .append(", ").append(limitMaxMemForMapTasks).append(", ").append(
-            limitMaxMemForReduceTasks).append(")"));
+    LOG.info(String.format("Scheduler configured with "
+        + "(memSizeForMapSlotOnJT, memSizeForReduceSlotOnJT, "
+        + "limitMaxMemForMapTasks, limitMaxMemForReduceTasks)"
+        + " (%d,%d,%d,%d)", Long.valueOf(memSizeForMapSlotOnJT), Long
+        .valueOf(memSizeForReduceSlotOnJT), Long
+        .valueOf(limitMaxMemForMapTasks), Long
+        .valueOf(limitMaxMemForReduceTasks)));
   }
 
   long getMemSizeForMapSlot() {
@@ -733,6 +811,15 @@
     return limitMaxMemForReduceTasks;
   }
 
+  String[] getOrderedQueues(CapacityTaskScheduler.TYPE type) {
+    if (type.equals(CapacityTaskScheduler.TYPE.MAP)) {
+      return mapScheduler.getOrderedQueues();
+    } else if (type.equals(CapacityTaskScheduler.TYPE.REDUCE)) {
+      return reduceScheduler.getOrderedQueues();
+    }
+    return null;
+  }
+
   @Override
   public synchronized void start() throws IOException {
     if (started) return;
@@ -755,15 +842,15 @@
     Set<String> queuesWithoutConfiguredCapacity = new HashSet<String>();
     float totalCapacity = 0.0f;
     for (String queueName: queues) {
-      float gc = schedConf.getCapacity(queueName);
-      if(gc == -1.0) {
+      float capacity = schedConf.getCapacity(queueName);
+      if(capacity == -1.0) {
         queuesWithoutConfiguredCapacity.add(queueName);
       }else {
-        totalCapacity += gc;
+        totalCapacity += capacity;
       }
       int ulMin = schedConf.getMinimumUserLimitPercent(queueName); 
       // create our QSI and add to our hashmap
-      QueueSchedulingInfo qsi = new QueueSchedulingInfo(queueName, gc, 
+      QueueSchedulingInfo qsi = new QueueSchedulingInfo(queueName, capacity, 
                                                     ulMin, jobQueuesManager);
       queueInfoMap.put(queueName, qsi);
 
@@ -877,29 +964,49 @@
         if (j.getStatus().getRunState() != JobStatus.RUNNING) {
           continue;
         }
-        int runningMaps = j.runningMaps();
-        int runningReduces = j.runningReduces();
-        qsi.mapTSI.numRunningTasks += runningMaps;
-        qsi.reduceTSI.numRunningTasks += runningReduces;
+
+        int numMapsRunningForThisJob = mapScheduler.getRunningTasks(j);
+        int numReducesRunningForThisJob = reduceScheduler.getRunningTasks(j);
+        int numMapSlotsForThisJob = mapScheduler.getSlotsOccupied(j);
+        int numReduceSlotsForThisJob = reduceScheduler.getSlotsOccupied(j);
+        j.setSchedulingInfo(String.format(JOB_SCHEDULING_INFO_FORMAT_STRING,
+            Integer.valueOf(numMapsRunningForThisJob), Integer
+                .valueOf(numMapSlotsForThisJob), Integer
+                .valueOf(numReducesRunningForThisJob), Integer
+                .valueOf(numReduceSlotsForThisJob)));
+        qsi.mapTSI.numRunningTasks += numMapsRunningForThisJob;
+        qsi.reduceTSI.numRunningTasks += numReducesRunningForThisJob;
+        qsi.mapTSI.numSlotsOccupied += numMapSlotsForThisJob;
+        qsi.reduceTSI.numSlotsOccupied += numReduceSlotsForThisJob;
         Integer i =
-          qsi.mapTSI.numRunningTasksByUser.get(j.getProfile().getUser());
-        qsi.mapTSI.numRunningTasksByUser.put(j.getProfile().getUser(),
-            i+runningMaps);
-        i = qsi.reduceTSI.numRunningTasksByUser.get(j.getProfile().getUser());
-        qsi.reduceTSI.numRunningTasksByUser.put(j.getProfile().getUser(),
-            i+runningReduces);
-        LOG.debug("updateQSI: job " + j.getJobID().toString() + ": run(m) = " +
-            j.runningMaps() + ", run(r) = " + j.runningReduces() +
-            ", finished(m) = " + j.finishedMaps() + ", finished(r)= " +
-            j.finishedReduces() + ", failed(m) = " + j.failedMapTasks +
-            ", failed(r) = " + j.failedReduceTasks + ", spec(m) = " +
-            j.speculativeMapTasks + ", spec(r) = " + j.speculativeReduceTasks
-            + ", total(m) = " + j.numMapTasks + ", total(r) = " +
-            j.numReduceTasks);
+            qsi.mapTSI.numSlotsOccupiedByUser.get(j.getProfile().getUser());
+        qsi.mapTSI.numSlotsOccupiedByUser.put(j.getProfile().getUser(),
+            Integer.valueOf(i.intValue() + numMapSlotsForThisJob));
+        i = qsi.reduceTSI.numSlotsOccupiedByUser.get(j.getProfile().getUser());
+        qsi.reduceTSI.numSlotsOccupiedByUser.put(j.getProfile().getUser(),
+            Integer.valueOf(i.intValue() + numReduceSlotsForThisJob));
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(String.format("updateQSI: job %s: run(m)=%d, "
+              + "occupied(m)=%d, run(r)=%d, occupied(r)=%d, finished(m)=%d,"
+              + " finished(r)=%d, failed(m)=%d, failed(r)=%d, "
+              + "spec(m)=%d, spec(r)=%d, total(m)=%d, total(r)=%d", j
+              .getJobID().toString(), Integer
+              .valueOf(numMapsRunningForThisJob), Integer
+              .valueOf(numMapSlotsForThisJob), Integer
+              .valueOf(numReducesRunningForThisJob), Integer
+              .valueOf(numReduceSlotsForThisJob), Integer.valueOf(j
+              .finishedMaps()), Integer.valueOf(j.finishedReduces()), Integer
+              .valueOf(j.failedMapTasks),
+              Integer.valueOf(j.failedReduceTasks), Integer
+                  .valueOf(j.speculativeMapTasks), Integer
+                  .valueOf(j.speculativeReduceTasks), Integer
+                  .valueOf(j.numMapTasks), Integer.valueOf(j.numReduceTasks)));
+        }
+
         /*
          * it's fine walking down the entire list of running jobs - there
          * probably will not be many, plus, we may need to go through the
-         * list to compute numRunningTasksByUser. If this is expensive, we
+         * list to compute numSlotsOccupiedByUser. If this is expensive, we
          * can keep a list of running jobs per user. Then we only need to
          * consider the first few jobs per user.
          */
@@ -914,10 +1021,8 @@
    * The grand plan for assigning a task. 
    * First, decide whether a Map or Reduce task should be given to a TT 
    * (if the TT can accept either). 
-   * Next, pick a queue. We only look at queues that need a slot. Among
-   * these, we first look at queues whose ac is less than gc (queues that 
-   * gave up capacity in the past). Next, we look at any other queue that
-   * needs a slot. 
+   * Next, pick a queue. We only look at queues that need a slot. Among these,
+   * we first look at queues whose (# of running tasks)/capacity is the least.
    * Next, pick a job in a queue. we pick the job at the front of the queue
    * unless its user is over the user limit. 
    * Finally, given a job, pick a task from the job. 
@@ -1019,8 +1124,10 @@
     if (null == i) {
       i = 1;
       // set the count for running tasks to 0
-      qsi.mapTSI.numRunningTasksByUser.put(job.getProfile().getUser(), 0);
-      qsi.reduceTSI.numRunningTasksByUser.put(job.getProfile().getUser(), 0);
+      qsi.mapTSI.numSlotsOccupiedByUser.put(job.getProfile().getUser(),
+          Integer.valueOf(0));
+      qsi.reduceTSI.numSlotsOccupiedByUser.put(job.getProfile().getUser(),
+          Integer.valueOf(0));
     }
     else {
       i++;
@@ -1042,8 +1149,8 @@
     if (0 == i.intValue()) {
       qsi.numJobsByUser.remove(job.getProfile().getUser());
       // remove job footprint from our TSIs
-      qsi.mapTSI.numRunningTasksByUser.remove(job.getProfile().getUser());
-      qsi.reduceTSI.numRunningTasksByUser.remove(job.getProfile().getUser());
+      qsi.mapTSI.numSlotsOccupiedByUser.remove(job.getProfile().getUser());
+      qsi.reduceTSI.numSlotsOccupiedByUser.remove(job.getProfile().getUser());
       LOG.debug("No more jobs for user, number of users = " + qsi.numJobsByUser.size());
     }
     else {

Modified: hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/MemoryMatcher.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/MemoryMatcher.java?rev=783055&r1=783054&r2=783055&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/MemoryMatcher.java (original)
+++ hadoop/core/branches/HADOOP-3628-2/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/MemoryMatcher.java Tue Jun  9 16:11:19 2009
@@ -53,10 +53,9 @@
    * @return amount of memory that is used by the residing tasks,
    *          null if memory cannot be computed for some reason.
    */
-  private synchronized Long getMemReservedForTasks(
+  synchronized Long getMemReservedForTasks(
       TaskTrackerStatus taskTracker, CapacityTaskScheduler.TYPE taskType) {
     long vmem = 0;
-    long myVmem = 0;
 
     for (TaskStatus task : taskTracker.getTaskReports()) {
       // the following task states are one in which the slot is
@@ -90,6 +89,7 @@
         // tasks' memory limits to the nearest multiple of the slot-memory-size
         // set on JT. This essentially translates to tasks of a high memory job
         // using multiple slots.
+        long myVmem = 0;
         if (task.getIsMap() && taskType.equals(CapacityTaskScheduler.TYPE.MAP)) {
           myVmem = jConf.getMemoryForMapTask();
           myVmem =



Mime
View raw message