hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r785794 [1/3] - in /hadoop/core/branches/HADOOP-4687: core/ core/src/java/ core/src/java/org/apache/hadoop/fs/ core/src/java/org/apache/hadoop/util/ core/src/test/core/ core/src/test/core/org/apache/hadoop/util/ hdfs/ hdfs/src/java/ hdfs/sr...
Date Wed, 17 Jun 2009 20:55:54 GMT
Author: omalley
Date: Wed Jun 17 20:55:51 2009
New Revision: 785794

URL: http://svn.apache.org/viewvc?rev=785794&view=rev
Log:
HADOOP-4687. Merge -r 784663:785643 from trunk to branch.

Added:
    hadoop/core/branches/HADOOP-4687/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/
      - copied from r785643, hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/
    hadoop/core/branches/HADOOP-4687/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
      - copied unchanged from r785643, hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
    hadoop/core/branches/HADOOP-4687/mapred/conf/mapred-queues.xml.template
      - copied unchanged from r785643, hadoop/core/trunk/conf/mapred-queues.xml.template
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/Clock.java
      - copied unchanged from r785643, hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Clock.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/Queue.java
      - copied unchanged from r785643, hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Queue.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java
      - copied unchanged from r785643, hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java
      - copied unchanged from r785643, hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java
Removed:
    hadoop/core/branches/HADOOP-4687/mapred/conf/mapred-queue-acls.xml.template
Modified:
    hadoop/core/branches/HADOOP-4687/core/CHANGES.txt   (contents, props changed)
    hadoop/core/branches/HADOOP-4687/core/src/java/   (props changed)
    hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/fs/LocalDirAllocator.java
    hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/util/StringUtils.java
    hadoop/core/branches/HADOOP-4687/core/src/test/core/   (props changed)
    hadoop/core/branches/HADOOP-4687/core/src/test/core/org/apache/hadoop/util/TestStringUtils.java
    hadoop/core/branches/HADOOP-4687/hdfs/build.xml   (contents, props changed)
    hadoop/core/branches/HADOOP-4687/hdfs/src/java/   (props changed)
    hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
    hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
    hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java
    hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
    hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java
    hadoop/core/branches/HADOOP-4687/hdfs/src/test/hdfs/   (props changed)
    hadoop/core/branches/HADOOP-4687/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java
    hadoop/core/branches/HADOOP-4687/mapred/.gitignore   (contents, props changed)
    hadoop/core/branches/HADOOP-4687/mapred/conf/   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/conf/capacity-scheduler.xml.template   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/dynamic-scheduler/   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/DefaultTaskSelector.java
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/sqoop/   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/streaming/   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/src/contrib/vaidya/   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/src/java/   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/src/java/mapred-default.xml
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/AdminOperationsProtocol.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/ClusterStatus.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/IsolationRunner.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobHistory.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobQueueClient.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobQueueInfo.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/QueueManager.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/TaskInProgress.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/TaskRunner.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/TaskStatus.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
    hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueInformation.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/TestQueueAclsForCurrentUser.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/TestResourceEstimation.java
    hadoop/core/branches/HADOOP-4687/mapred/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
    hadoop/core/branches/HADOOP-4687/mapred/src/webapps/job/   (props changed)
    hadoop/core/branches/HADOOP-4687/mapred/src/webapps/job/jobqueue_details.jsp
    hadoop/core/branches/HADOOP-4687/mapred/src/webapps/job/jobtracker.jsp

Modified: hadoop/core/branches/HADOOP-4687/core/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/core/CHANGES.txt?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/core/CHANGES.txt (original)
+++ hadoop/core/branches/HADOOP-4687/core/CHANGES.txt Wed Jun 17 20:55:51 2009
@@ -68,6 +68,9 @@
     HADOOP-5698. Change org.apache.hadoop.examples.MultiFileWordCount to 
     use new mapreduce api. (Amareshwari Sriramadasu via sharad)
 
+    HADOOP-5913. Provide ability to an administrator to stop and start
+    job queues. (Rahul Kumar Singh and Hemanth Yamijala via yhemanth)
+
   NEW FEATURES
 
     HADOOP-4268. Change fsck to use ClientProtocol methods so that the
@@ -149,6 +152,9 @@
     HADOOP-5170. Allows jobs to set max maps/reduces per-node and per-cluster.
     (Matei Zaharia via ddas)
 
+    HADOOP-5897. Add name-node metrics to capture java heap usage.
+    (Suresh Srinivas via shv)
+
   IMPROVEMENTS
 
     HADOOP-4565. Added CombineFileInputFormat to use data locality information
@@ -444,6 +450,12 @@
     HADOOP-5938. Change org.apache.hadoop.mapred.jobcontrol to use new
     api. (Amareshwari Sriramadasu via sharad)
 
+    HADOOP-2141. Improves the speculative execution heuristic. The heuristic
+    is currently based on the progress-rates of tasks and the expected time
+    to complete. Also, statistics about trackers are collected, and speculative
+    tasks are not given to the ones deduced to be slow. 
+    (Andy Konwinski and ddas)
+
   OPTIMIZATIONS
 
     HADOOP-5595. NameNode does not need to run a replicator to choose a
@@ -820,6 +832,9 @@
     LD_LIBRARY_PATH and other environment variables.
     (Sreekanth Ramakrishnan via yhemanth)
 
+    HADOOP-4041. IsolationRunner does not work as documented.
+    (Philip Zeyliger via tomwhite)
+
 Release 0.20.1 - Unreleased
 
   INCOMPATIBLE CHANGES
@@ -948,6 +963,9 @@
     (usually HDFS) is started at nearly the same time as the JobTracker. 
     (Amar Kamat via ddas)
 
+    HADOOP-5920. Fixes a testcase failure for TestJobHistory. 
+    (Amar Kamat via ddas)
+
 Release 0.20.0 - 2009-04-15
 
   INCOMPATIBLE CHANGES
@@ -2982,6 +3000,10 @@
 
     HADOOP-5644. Namenode is stuck in safe mode. (suresh Srinivas via hairong)
 
+    HADOOP-6017. Lease Manager in NameNode does not handle certain characters
+    in filenames. This results in fatal errors in Secondary NameNode and while
+    restrating NameNode. (Tsz Wo (Nicholas), SZE via rangadi)
+
 Release 0.18.3 - 2009-01-27
 
   IMPROVEMENTS

Propchange: hadoop/core/branches/HADOOP-4687/core/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:776175-784663
+/hadoop/core/trunk/CHANGES.txt:776175-785643

Propchange: hadoop/core/branches/HADOOP-4687/core/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/core/src/java:713112
-/hadoop/core/trunk/src/core:776175-784663
+/hadoop/core/trunk/src/core:776175-785643

Modified: hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/fs/LocalDirAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/fs/LocalDirAllocator.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/fs/LocalDirAllocator.java (original)
+++ hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/fs/LocalDirAllocator.java Wed Jun 17 20:55:51 2009
@@ -33,7 +33,7 @@
  * files. The way it works is that it is kept track what disk was last
  * allocated for a file write. For the current request, the next disk from
  * the set of disks would be allocated if the free space on the disk is 
- * sufficient enough to accomodate the file that is being considered for
+ * sufficient enough to accommodate the file that is being considered for
  * creation. If the space requirements cannot be met, the next disk in order
  * would be tried and so on till a disk is found with sufficient capacity.
  * Once a disk with sufficient space is identified, a check is done to make
@@ -69,6 +69,9 @@
                  new TreeMap<String, AllocatorPerContext>();
   private String contextCfgItemName;
 
+  /** Used when size of file to be allocated is unknown. */
+  public static final int SIZE_UNKNOWN = -1;
+
   /**Create an allocator object
    * @param contextCfgItemName
    */
@@ -105,10 +108,11 @@
    */
   public Path getLocalPathForWrite(String pathStr, 
       Configuration conf) throws IOException {
-    return getLocalPathForWrite(pathStr, -1, conf);
+    return getLocalPathForWrite(pathStr, SIZE_UNKNOWN, conf);
   }
   
-  /** Get a path from the local FS. Pass size as -1 if not known apriori. We
+  /** Get a path from the local FS. Pass size as 
+   *  SIZE_UNKNOWN if not known apriori. We
    *  round-robin over the set of disks (via the configured dirs) and return
    *  the first complete path which has enough space 
    *  @param pathStr the requested path (this will be created on the first 
@@ -274,7 +278,7 @@
      */
     public synchronized Path getLocalPathForWrite(String path, 
         Configuration conf) throws IOException {
-      return getLocalPathForWrite(path, -1, conf);
+      return getLocalPathForWrite(path, SIZE_UNKNOWN, conf);
     }
 
     /** Get a path from the local FS. If size is known, we go
@@ -296,7 +300,7 @@
       }
       Path returnPath = null;
       
-      if(size == -1) {  //do roulette selection: pick dir with probability 
+      if(size == SIZE_UNKNOWN) {  //do roulette selection: pick dir with probability 
                     //proportional to available size
         long[] availableOnDisk = new long[dirDF.length];
         long totalAvailable = 0;
@@ -344,7 +348,8 @@
           "directory for " + pathStr);
     }
 
-    /** Creates a file on the local FS. Pass size as -1 if not known apriori. We
+    /** Creates a file on the local FS. Pass size as 
+     * {@link LocalDirAllocator.SIZE_UNKNOWN} if not known apriori. We
      *  round-robin over the set of disks (via the configured dirs) and return
      *  a file on the first path which has enough space. The file is guaranteed
      *  to go away when the JVM exits.

Modified: hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/util/StringUtils.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/util/StringUtils.java (original)
+++ hadoop/core/branches/HADOOP-4687/core/src/java/org/apache/hadoop/util/StringUtils.java Wed Jun 17 20:55:51 2009
@@ -677,4 +677,24 @@
   public static synchronized String limitDecimalTo2(double d) {
     return decimalFormat.format(d);
   }
+  
+  /**
+   * Concatenates strings, using a separator.
+   *
+   * @param separator Separator to join with.
+   * @param strings Strings to join.
+   */
+  public static String join(CharSequence separator, Iterable<String> strings) {
+    StringBuffer sb = new StringBuffer();
+    boolean first = true;
+    for (String s : strings) {
+      if (first) {
+        first = false;
+      } else {
+        sb.append(separator);
+      }
+      sb.append(s);
+    }
+    return sb.toString();
+  }
 }

Propchange: hadoop/core/branches/HADOOP-4687/core/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/core/src/test/core:713112
-/hadoop/core/trunk/src/test/core:776175-784663
+/hadoop/core/trunk/src/test/core:776175-785643

Modified: hadoop/core/branches/HADOOP-4687/core/src/test/core/org/apache/hadoop/util/TestStringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/core/src/test/core/org/apache/hadoop/util/TestStringUtils.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/core/src/test/core/org/apache/hadoop/util/TestStringUtils.java (original)
+++ hadoop/core/branches/HADOOP-4687/core/src/test/core/org/apache/hadoop/util/TestStringUtils.java Wed Jun 17 20:55:51 2009
@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.util;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import junit.framework.TestCase;
 
 public class TestStringUtils extends TestCase {
@@ -118,4 +121,15 @@
     assertEquals(-1259520L, StringUtils.TraditionalBinaryPrefix.string2long("-1230k"));
     assertEquals(956703965184L, StringUtils.TraditionalBinaryPrefix.string2long("891g"));
   }
+
+  public void testJoin() {
+    List<String> s = new ArrayList<String>();
+    s.add("a");
+    s.add("b");
+    s.add("c");
+    assertEquals("", StringUtils.join(":", s.subList(0, 0)));
+    assertEquals("a", StringUtils.join(":", s.subList(0, 1)));
+    assertEquals("a:b", StringUtils.join(":", s.subList(0, 2)));
+    assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3)));
+  }
 }

Modified: hadoop/core/branches/HADOOP-4687/hdfs/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/hdfs/build.xml?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/hdfs/build.xml (original)
+++ hadoop/core/branches/HADOOP-4687/hdfs/build.xml Wed Jun 17 20:55:51 2009
@@ -373,6 +373,7 @@
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data30bytes" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data60bytes" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data120bytes" todir="${test.cache.data}"/>
+    <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data1k" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19" todir="${test.cache.data}"/>
   </target>

Propchange: hadoop/core/branches/HADOOP-4687/hdfs/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -0,0 +1,2 @@
+/hadoop/core/branches/branch-0.19/hdfs/build.xml:713112
+/hadoop/core/trunk/build.xml:779102

Propchange: hadoop/core/branches/HADOOP-4687/hdfs/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/java:713112
-/hadoop/core/trunk/src/hdfs:776175-784663
+/hadoop/core/trunk/src/hdfs:776175-785643

Modified: hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Wed Jun 17 20:55:51 2009
@@ -48,16 +48,24 @@
  * methods to be called with lock held on {@link FSNamesystem}.
  */
 public class BlockManager {
+  // Default initial capacity and load factor of map
+  public static final int DEFAULT_INITIAL_MAP_CAPACITY = 16;
+  public static final float DEFAULT_MAP_LOAD_FACTOR = 0.75f;
+
   private final FSNamesystem namesystem;
 
-  long pendingReplicationBlocksCount = 0L, corruptReplicaBlocksCount,
-  underReplicatedBlocksCount = 0L, scheduledReplicationBlocksCount = 0L;
+  volatile long pendingReplicationBlocksCount = 0L;
+  volatile long corruptReplicaBlocksCount = 0L;
+  volatile long underReplicatedBlocksCount = 0L;
+  volatile long scheduledReplicationBlocksCount = 0L;
+  volatile long excessBlocksCount = 0L;
+  volatile long pendingDeletionBlocksCount = 0L;
 
   //
   // Mapping: Block -> { INode, datanodes, self ref }
   // Updated only in response to client-sent information.
   //
-  BlocksMap blocksMap = new BlocksMap();
+  final BlocksMap blocksMap;
 
   //
   // Store blocks-->datanodedescriptor(s) map of corrupt replicas
@@ -110,11 +118,17 @@
   ReplicationTargetChooser replicator;
 
   BlockManager(FSNamesystem fsn, Configuration conf) throws IOException {
+    this(fsn, conf, DEFAULT_INITIAL_MAP_CAPACITY);
+  }
+  
+  BlockManager(FSNamesystem fsn, Configuration conf, int capacity)
+      throws IOException {
     namesystem = fsn;
     pendingReplications = new PendingReplicationBlocks(
         conf.getInt("dfs.replication.pending.timeout.sec",
                     -1) * 1000L);
     setConfigurationParameters(conf);
+    blocksMap = new BlocksMap(capacity, DEFAULT_MAP_LOAD_FACTOR);
   }
 
   void setConfigurationParameters(Configuration conf) throws IOException {
@@ -324,8 +338,11 @@
 
   void removeFromInvalidates(String datanodeId, Block block) {
     Collection<Block> v = recentInvalidateSets.get(datanodeId);
-    if (v != null && v.remove(block) && v.isEmpty()) {
-      recentInvalidateSets.remove(datanodeId);
+    if (v != null && v.remove(block)) {
+      pendingDeletionBlocksCount--;
+      if (v.isEmpty()) {
+        recentInvalidateSets.remove(datanodeId);
+      }
     }
   }
 
@@ -344,6 +361,7 @@
       recentInvalidateSets.put(dn.getStorageID(), invalidateSet);
     }
     if (invalidateSet.add(b)) {
+      pendingDeletionBlocksCount++;
       NameNode.stateChangeLog.info("BLOCK* NameSystem.addToInvalidates: "
           + b.getBlockName() + " is added to invalidSet of " + dn.getName());
     }
@@ -366,7 +384,8 @@
    */
   private void dumpRecentInvalidateSets(PrintWriter out) {
     int size = recentInvalidateSets.values().size();
-    out.println("Metasave: Blocks waiting deletion from "+size+" datanodes.");
+    out.println("Metasave: Blocks " + pendingDeletionBlocksCount 
+        + " waiting deletion from " + size + " datanodes.");
     if (size == 0) {
       return;
     }
@@ -1101,10 +1120,12 @@
       excessBlocks = new TreeSet<Block>();
       excessReplicateMap.put(dn.getStorageID(), excessBlocks);
     }
-    excessBlocks.add(block);
-    NameNode.stateChangeLog.debug("BLOCK* NameSystem.chooseExcessReplicates: "
-        + "(" + dn.getName() + ", " + block
-        + ") is added to excessReplicateMap");
+    if (excessBlocks.add(block)) {
+      excessBlocksCount++;
+      NameNode.stateChangeLog.debug("BLOCK* NameSystem.chooseExcessReplicates:"
+          + " (" + dn.getName() + ", " + block
+          + ") is added to excessReplicateMap");
+    }
   }
 
   /**
@@ -1140,11 +1161,13 @@
       Collection<Block> excessBlocks = excessReplicateMap.get(node
           .getStorageID());
       if (excessBlocks != null) {
-        excessBlocks.remove(block);
-        NameNode.stateChangeLog.debug("BLOCK* NameSystem.removeStoredBlock: "
-            + block + " is removed from excessBlocks");
-        if (excessBlocks.size() == 0) {
-          excessReplicateMap.remove(node.getStorageID());
+        if (excessBlocks.remove(block)) {
+          excessBlocksCount--;
+          NameNode.stateChangeLog.debug("BLOCK* NameSystem.removeStoredBlock: "
+              + block + " is removed from excessBlocks");
+          if (excessBlocks.size() == 0) {
+            excessReplicateMap.remove(node.getStorageID());
+          }
         }
       }
 
@@ -1243,12 +1266,7 @@
   }
 
   int getActiveBlockCount() {
-    int activeBlocks = blocksMap.size();
-    for(Iterator<Collection<Block>> it =
-          recentInvalidateSets.values().iterator(); it.hasNext();) {
-      activeBlocks -= it.next().size();
-    }
-    return activeBlocks;
+    return blocksMap.size() - (int)pendingDeletionBlocksCount;
   }
 
   DatanodeDescriptor[] getNodes(Block block) {
@@ -1312,8 +1330,11 @@
    * Remove a datanode from the invalidatesSet
    * @param n datanode
    */
-  void removeFromInvalidates(DatanodeInfo n) {
-    recentInvalidateSets.remove(n.getStorageID());
+  void removeFromInvalidates(String storageID) {
+    Collection<Block> blocks = recentInvalidateSets.remove(storageID);
+    if (blocks != null) {
+      pendingDeletionBlocksCount -= blocks.size();
+    }
   }
 
   /**
@@ -1331,7 +1352,7 @@
       assert nodeId != null;
       DatanodeDescriptor dn = namesystem.getDatanode(nodeId);
       if (dn == null) {
-        recentInvalidateSets.remove(nodeId);
+        removeFromInvalidates(nodeId);
         return 0;
       }
 
@@ -1351,8 +1372,9 @@
       }
 
       // If we send everything in this message, remove this node entry
-      if (!it.hasNext())
-        recentInvalidateSets.remove(nodeId);
+      if (!it.hasNext()) {
+        removeFromInvalidates(nodeId);
+      }
 
       dn.addBlocksToBeInvalidated(blocksToInvalidate);
 
@@ -1397,4 +1419,14 @@
   void removeBlockFromMap(BlockInfo blockInfo) {
     blocksMap.removeBlock(blockInfo);
   }
+  
+  public int getCapacity() {
+    synchronized(namesystem) {
+      return blocksMap.getCapacity();
+    }
+  }
+  
+  public float getLoadFactor() {
+    return blocksMap.getLoadFactor();
+  }
 }

Modified: hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java (original)
+++ hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java Wed Jun 17 20:55:51 2009
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -306,7 +305,20 @@
     }
   }
 
-  private Map<Block, BlockInfo> map = new HashMap<Block, BlockInfo>();
+  // Used for tracking HashMap capacity growth
+  private int capacity;
+  private final float loadFactor;
+  
+  private Map<BlockInfo, BlockInfo> map;
+
+  BlocksMap(int initialCapacity, float loadFactor) {
+    this.capacity = 1;
+    // Capacity is initialized to the next multiple of 2 of initialCapacity
+    while (this.capacity < initialCapacity)
+      this.capacity <<= 1;
+    this.loadFactor = loadFactor;
+    this.map = new HashMap<BlockInfo, BlockInfo>(initialCapacity, loadFactor);
+  }
 
   /**
    * Add BlockInfo if mapping does not exist.
@@ -437,4 +449,18 @@
     
     return true;
   }
+  
+  /** Get the capacity of the HashMap that stores blocks */
+  public int getCapacity() {
+    // Capacity doubles every time the map size reaches the threshold
+    while (map.size() > (int)(capacity * loadFactor)) {
+      capacity <<= 1;
+    }
+    return capacity;
+  }
+  
+  /** Get the load factor of the map */
+  public float getLoadFactor() {
+    return loadFactor;
+  }
 }

Modified: hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java (original)
+++ hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java Wed Jun 17 20:55:51 2009
@@ -61,10 +61,6 @@
                                    "on " + dn.getName() +
                                    " by " + Server.getRemoteIp());
     }
-    if (NameNode.getNameNodeMetrics() != null) {
-      NameNode.getNameNodeMetrics().numBlocksCorrupted.set(
-        corruptReplicasMap.size());
-    }
   }
 
   /**
@@ -75,10 +71,6 @@
   void removeFromCorruptReplicasMap(Block blk) {
     if (corruptReplicasMap != null) {
       corruptReplicasMap.remove(blk);
-      if (NameNode.getNameNodeMetrics() != null) {
-        NameNode.getNameNodeMetrics().numBlocksCorrupted.set(
-          corruptReplicasMap.size());
-      }
     }
   }
 

Modified: hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Jun 17 20:55:51 2009
@@ -2146,7 +2146,7 @@
 
   void unprotectedRemoveDatanode(DatanodeDescriptor nodeDescr) {
     nodeDescr.resetBlocks();
-    blockManager.removeFromInvalidates(nodeDescr);
+    blockManager.removeFromInvalidates(nodeDescr.getStorageID());
     NameNode.stateChangeLog.debug(
                                   "BLOCK* NameSystem.unprotectedRemoveDatanode: "
                                   + nodeDescr.getName() + " is out of service now.");
@@ -2419,7 +2419,7 @@
       return new long[] {this.capacityTotal, this.capacityUsed, 
                          this.capacityRemaining,
                          getUnderReplicatedBlocks(),
-                         getCorruptReplicaBlocksCount(),
+                         getCorruptReplicaBlocks(),
                          getMissingBlocksCount()};
     }
   }
@@ -3469,7 +3469,7 @@
   }
 
   /** Returns number of blocks with corrupt replicas */
-  public long getCorruptReplicaBlocksCount() {
+  public long getCorruptReplicaBlocks() {
     return blockManager.corruptReplicaBlocksCount;
   }
 
@@ -3477,6 +3477,18 @@
     return blockManager.scheduledReplicationBlocksCount;
   }
 
+  public long getPendingDeletionBlocks() {
+    return blockManager.pendingDeletionBlocksCount;
+  }
+
+  public long getExcessBlocks() {
+    return blockManager.excessBlocksCount;
+  }
+  
+  public int getBlockCapacity() {
+    return blockManager.getCapacity();
+  }
+
   public String getFSState() {
     return isInSafeMode() ? "safeMode" : "Operational";
   }

Modified: hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original)
+++ hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Wed Jun 17 20:55:51 2009
@@ -281,11 +281,12 @@
                ", replaceBy=" + replaceBy);
     }
 
+    final int len = overwrite.length();
     for(Map.Entry<String, Lease> entry : findLeaseWithPrefixPath(src, sortedLeasesByPath)) {
       final String oldpath = entry.getKey();
       final Lease lease = entry.getValue();
-      final String newpath = oldpath.replaceFirst(
-          java.util.regex.Pattern.quote(overwrite), replaceBy);
+      //overwrite must be a prefix of oldpath
+      final String newpath = replaceBy + oldpath.substring(len);
       if (LOG.isDebugEnabled()) {
         LOG.debug("changeLease: replacing " + oldpath + " with " + newpath);
       }

Modified: hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java (original)
+++ hadoop/core/branches/HADOOP-4687/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMetrics.java Wed Jun 17 20:55:51 2009
@@ -43,19 +43,22 @@
 public class FSNamesystemMetrics implements Updater {
   private static Log log = LogFactory.getLog(FSNamesystemMetrics.class);
   private final MetricsRecord metricsRecord;
-  private final MetricsRegistry registry = new MetricsRegistry();
-
+  final MetricsRegistry registry = new MetricsRegistry();
    
-  private final MetricsIntValue filesTotal = new MetricsIntValue("FilesTotal", registry);
-  private final MetricsLongValue blocksTotal = new MetricsLongValue("BlocksTotal", registry);
-  private final MetricsIntValue capacityTotalGB = new MetricsIntValue("CapacityTotalGB", registry);
-  private final MetricsIntValue capacityUsedGB = new MetricsIntValue("CapacityUsedGB", registry);
-  private final MetricsIntValue capacityRemainingGB = new MetricsIntValue("CapacityRemainingGB", registry);
-  private final MetricsIntValue totalLoad = new MetricsIntValue("TotalLoad", registry);
-  private final MetricsIntValue pendingReplicationBlocks = new MetricsIntValue("PendingReplicationBlocks", registry);
-  private final MetricsIntValue underReplicatedBlocks = new MetricsIntValue("UnderReplicatedBlocks", registry);
-  private final MetricsIntValue scheduledReplicationBlocks = new MetricsIntValue("ScheduledReplicationBlocks", registry);
-  private final MetricsIntValue missingBlocks = new MetricsIntValue("MissingBlocks", registry);    
+  final MetricsIntValue filesTotal = new MetricsIntValue("FilesTotal", registry);
+  final MetricsLongValue blocksTotal = new MetricsLongValue("BlocksTotal", registry);
+  final MetricsIntValue capacityTotalGB = new MetricsIntValue("CapacityTotalGB", registry);
+  final MetricsIntValue capacityUsedGB = new MetricsIntValue("CapacityUsedGB", registry);
+  final MetricsIntValue capacityRemainingGB = new MetricsIntValue("CapacityRemainingGB", registry);
+  final MetricsIntValue totalLoad = new MetricsIntValue("TotalLoad", registry);
+  final MetricsIntValue pendingDeletionBlocks = new MetricsIntValue("PendingDeletionBlocks", registry);
+  final MetricsIntValue corruptBlocks = new MetricsIntValue("CorruptBlocks", registry);
+  final MetricsIntValue excessBlocks = new MetricsIntValue("ExcessBlocks", registry);
+  final MetricsIntValue pendingReplicationBlocks = new MetricsIntValue("PendingReplicationBlocks", registry);
+  final MetricsIntValue underReplicatedBlocks = new MetricsIntValue("UnderReplicatedBlocks", registry);
+  final MetricsIntValue scheduledReplicationBlocks = new MetricsIntValue("ScheduledReplicationBlocks", registry);
+  final MetricsIntValue missingBlocks = new MetricsIntValue("MissingBlocks", registry);    
+  final MetricsIntValue blockCapacity = new MetricsIntValue("BlockCapacity", registry);
 
   private final FSNamesystem fsNameSystem;
 
@@ -103,12 +106,16 @@
       capacityRemainingGB.set(roundBytesToGBytes(fsNameSystem.
                                                getCapacityRemaining()));
       totalLoad.set(fsNameSystem.getTotalLoad());
+      corruptBlocks.set((int)fsNameSystem.getCorruptReplicaBlocks());
+      excessBlocks.set((int)fsNameSystem.getExcessBlocks());
+      pendingDeletionBlocks.set((int)fsNameSystem.getPendingDeletionBlocks());
       pendingReplicationBlocks.set((int)fsNameSystem.
                                    getPendingReplicationBlocks());
       underReplicatedBlocks.set((int)fsNameSystem.getUnderReplicatedBlocks());
       scheduledReplicationBlocks.set((int)fsNameSystem.
                                       getScheduledReplicationBlocks());
       missingBlocks.set((int)fsNameSystem.getMissingBlocksCount());
+      blockCapacity.set(fsNameSystem.getBlockCapacity());
 
       for (MetricsBase m : registry.getMetricsList()) {
         m.pushMetric(metricsRecord);

Propchange: hadoop/core/branches/HADOOP-4687/hdfs/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
-/hadoop/core/trunk/src/test/hdfs:776175-784663
+/hadoop/core/trunk/src/test/hdfs:776175-785643

Modified: hadoop/core/branches/HADOOP-4687/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java (original)
+++ hadoop/core/branches/HADOOP-4687/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java Wed Jun 17 20:55:51 2009
@@ -82,6 +82,17 @@
       fs.mkdirs(dir3);
       fs.rename(dir1, dir3);
 
+      // create file3
+      Path file3 = new Path(dir3, "file3");
+      FSDataOutputStream stm3 = TestFileCreation.createFile(fs, file3, 1);
+      TestFileCreation.writeFile(stm3);
+      // rename file3 to some bad name
+      try {
+        fs.rename(file3, new Path(dir3, "$ "));
+      } catch(Exception e) {
+        e.printStackTrace();
+      }
+      
       // restart cluster with the same namenode port as before.
       // This ensures that leases are persisted in fsimage.
       cluster.shutdown();

Modified: hadoop/core/branches/HADOOP-4687/mapred/.gitignore
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/.gitignore?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/.gitignore (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/.gitignore Wed Jun 17 20:55:51 2009
@@ -27,6 +27,8 @@
 conf/hadoop-policy.xml
 conf/capacity-scheduler.xml
 conf/mapred-queue-acls.xml
+conf/mapred-queue-acls.xml.template
+conf/mapred-queues.xml
 docs/api/
 logs/
 src/contrib/index/conf/index-config.xml

Propchange: hadoop/core/branches/HADOOP-4687/mapred/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -0,0 +1,2 @@
+/hadoop/core/branches/branch-0.19/mapred/.gitignore:713112
+/hadoop/core/trunk/.gitignore:784664-785643

Propchange: hadoop/core/branches/HADOOP-4687/mapred/conf/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Wed Jun 17 20:55:51 2009
@@ -8,3 +8,5 @@
 hadoop-policy.xml
 capacity-scheduler.xml
 mapred-queue-acls.xml
+mapred-queues.xml
+

Propchange: hadoop/core/branches/HADOOP-4687/mapred/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -0,0 +1,2 @@
+/hadoop/core/branches/branch-0.19/mapred/conf:713112
+/hadoop/core/trunk/conf:784664-785643

Propchange: hadoop/core/branches/HADOOP-4687/mapred/conf/capacity-scheduler.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/mapred/conf/capacity-scheduler.xml.template:713112
-/hadoop/core/trunk/conf/capacity-scheduler.xml.template:776175-784663
+/hadoop/core/trunk/conf/capacity-scheduler.xml.template:776175-785643

Propchange: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -0,0 +1,2 @@
+/hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
+/hadoop/core/trunk/src/contrib:784664-785643

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Wed Jun 17 20:55:51 2009
@@ -516,6 +516,27 @@
       LOG.debug(s);
     }
     
+    /**
+     * Check if one of the tasks have a speculative task to execute on the 
+     * particular task tracker.
+     * 
+     * @param tips tasks of a job
+     * @param progress percentage progress of the job
+     * @param tts task tracker status for which we are asking speculative tip
+     * @return true if job has a speculative task to run on particular TT.
+     */
+    boolean hasSpeculativeTask(TaskInProgress[] tips, float progress, 
+        TaskTrackerStatus tts) {
+      long currentTime = System.currentTimeMillis();
+      for(TaskInProgress tip : tips)  {
+        if(tip.isRunning() 
+            && !(tip.hasRunOnMachine(tts.getHost(), tts.getTrackerName())) 
+            && tip.canBeSpeculated(currentTime)) {
+          return true;
+        }
+      }
+      return false;
+    }
   }
 
   /**

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java Wed Jun 17 20:55:51 2009
@@ -38,6 +38,7 @@
 import org.apache.hadoop.mapred.JobStatusChangeEvent.EventType;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecurityUtil.AccessControlList;
 
 
 public class TestCapacityScheduler extends TestCase {
@@ -158,7 +159,7 @@
     
     public FakeJobInProgress(JobID jId, JobConf jobConf,
         FakeTaskTrackerManager taskTrackerManager, String user) {
-      super(jId, jobConf);
+      super(jId, jobConf, null);
       this.taskTrackerManager = taskTrackerManager;
       this.startTime = System.currentTimeMillis();
       this.status = new JobStatus(jId, 0f, 0f, JobStatus.PREP);
@@ -307,18 +308,58 @@
       }
       return true;
     }
+    
+    @Override
+    /*
+     *hasSpeculativeMap and hasSpeculativeReduce is reset by FakeJobInProgress
+     *after the speculative tip has been scheduled.
+     */
+    boolean canBeSpeculated(long currentTime) {
+      if(isMap && hasSpeculativeMap) {
+        return fakeJob.getJobConf().getMapSpeculativeExecution();
+      } 
+      if (!isMap && hasSpeculativeReduce) {
+        return fakeJob.getJobConf().getReduceSpeculativeExecution();
+      }
+      return false;
+    }
+    
+    @Override
+    public boolean isRunning() {
+      return !activeTasks.isEmpty();
+    }
+    
   }
   
   static class FakeQueueManager extends QueueManager {
-    private Set<String> queues = null;
+    private Set<String> queueNames = null;
+    private static final AccessControlList allEnabledAcl = new AccessControlList("*");
+    
     FakeQueueManager() {
       super(new Configuration());
     }
-    void setQueues(Set<String> queues) {
-      this.queues = queues;
+    
+    void setQueues(Set<String> queueNames) {
+      this.queueNames = queueNames;
+      
+      // sync up queues with the parent class.
+      Queue[] queues = new Queue[queueNames.size()];
+      int i = 0;
+      for (String queueName : queueNames) {
+        HashMap<String, AccessControlList> aclsMap 
+          = new HashMap<String, AccessControlList>();
+        for (Queue.QueueOperation oper : Queue.QueueOperation.values()) {
+          String key = QueueManager.toFullPropertyName(queueName,
+                                                        oper.getAclName());
+          aclsMap.put(key, allEnabledAcl);
+        }
+        queues[i++] = new Queue(queueName, aclsMap, Queue.QueueState.RUNNING);
+      }
+      super.setQueues(queues);
     }
+    
     public synchronized Set<String> getQueues() {
-      return queues;
+      return queueNames;
     }
   }
   

Propchange: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/dynamic-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1 +1,2 @@
 /hadoop/core/branches/branch-0.19/src/contrib/dynamic-scheduler:713112
+/hadoop/core/trunk/src/contrib/dynamic-scheduler:784975-785643

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/DefaultTaskSelector.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/DefaultTaskSelector.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/DefaultTaskSelector.java (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/DefaultTaskSelector.java Wed Jun 17 20:55:51 2009
@@ -33,9 +33,8 @@
   public int neededSpeculativeMaps(JobInProgress job) {
     int count = 0;
     long time = System.currentTimeMillis();
-    double avgProgress = job.getStatus().mapProgress();
     for (TaskInProgress tip: job.maps) {
-      if (tip.isRunning() && tip.hasSpeculativeTask(time, avgProgress)) {
+      if (tip.isRunning() && tip.canBeSpeculated(time)) {
         count++;
       }
     }
@@ -48,7 +47,7 @@
     long time = System.currentTimeMillis();
     double avgProgress = job.getStatus().reduceProgress();
     for (TaskInProgress tip: job.reduces) {
-      if (tip.isRunning() && tip.hasSpeculativeTask(time, avgProgress)) {
+      if (tip.isRunning() && tip.canBeSpeculated(time)) {
         count++;
       }
     }

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java Wed Jun 17 20:55:51 2009
@@ -87,15 +87,6 @@
     double reduceFairShare = 0; // Fair share of reduce slots at last update
   }
   
-  /**
-   * A clock class - can be mocked out for testing.
-   */
-  static class Clock {
-    long getTime() {
-      return System.currentTimeMillis();
-    }
-  }
-  
   public FairScheduler() {
     this(new Clock(), true);
   }

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java Wed Jun 17 20:55:51 2009
@@ -34,6 +34,7 @@
 import org.apache.hadoop.mapred.JobStatus;
 import org.apache.hadoop.mapred.FairScheduler.JobInfo;
 import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapred.UtilsForTests.FakeClock;
 
 public class TestFairScheduler extends TestCase {
   final static String TEST_DIR = new File(System.getProperty("test.build.data",
@@ -52,7 +53,7 @@
     
     public FakeJobInProgress(JobConf jobConf,
         FakeTaskTrackerManager taskTrackerManager) throws IOException {
-      super(new JobID("test", ++jobCounter), jobConf);
+      super(new JobID("test", ++jobCounter), jobConf, null);
       this.taskTrackerManager = taskTrackerManager;
       this.startTime = System.currentTimeMillis();
       this.status = new JobStatus();
@@ -220,19 +221,6 @@
     }
   }
   
-  protected class FakeClock extends FairScheduler.Clock {
-    private long time = 0;
-    
-    public void advance(long millis) {
-      time += millis;
-    }
-
-    @Override
-    long getTime() {
-      return time;
-    }
-  }
-  
   protected JobConf conf;
   protected FairScheduler scheduler;
   private FakeTaskTrackerManager taskTrackerManager;

Propchange: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/sqoop/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1 +1,2 @@
 /hadoop/core/branches/branch-0.19/src/contrib/sqoop:713112
+/hadoop/core/trunk/src/contrib/sqoop:784975-785643

Propchange: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/streaming/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/streaming:713112
-/hadoop/core/trunk/src/contrib/streaming:776175-784663
+/hadoop/core/trunk/src/contrib/streaming:776175-785643

Propchange: hadoop/core/branches/HADOOP-4687/mapred/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
-/hadoop/core/trunk/src/contrib/vaidya:776175-784663
+/hadoop/core/trunk/src/contrib/vaidya:776175-785643

Propchange: hadoop/core/branches/HADOOP-4687/mapred/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun 17 20:55:51 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/mapred/src/java:713112
-/hadoop/core/trunk/src/mapred:776175-784663
+/hadoop/core/trunk/src/mapred:776175-785643

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/java/mapred-default.xml?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/java/mapred-default.xml (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/java/mapred-default.xml Wed Jun 17 20:55:51 2009
@@ -501,6 +501,31 @@
   <description>If true, then multiple instances of some reduce tasks 
                may be executed in parallel.</description>
 </property>
+<property>
+  <name>mapred.speculative.execution.speculativeCap</name>
+  <value>0.1</value>
+  <description>The max percent (0-1) of running tasks that
+  can be speculatively re-executed at any time.</description>
+</property>
+ 
+<property>
+  <name>mapred.speculative.execution.slowTaskThreshold</name>
+  <value>1.0</value>The number of standard deviations by which a task's 
+  ave progress-rates must be lower than the average of all running tasks'
+  for the task to be considered too slow.
+  <description>
+  </description>
+</property>
+
+<property>
+  <name>mapred.speculative.execution.slowNodeThreshold</name>
+  <value>1.0</value>
+  <description>The number of standard deviations by which a Task 
+  Tracker's ave map and reduce progress-rates (finishTime-dispatchTime)
+  must be lower than the average of all successful map/reduce task's for
+  the TT to be considered too slow to give a speculative task to.
+  </description>
+</property>
 
 <property>
   <name>mapred.job.reuse.jvm.num.tasks</name>

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/AdminOperationsProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/AdminOperationsProtocol.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/AdminOperationsProtocol.java (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/AdminOperationsProtocol.java Wed Jun 17 20:55:51 2009
@@ -31,13 +31,16 @@
   /**
    * Version 1: Initial version. Added refreshQueueAcls.
    * Version 2: Added node refresh facility
+   * Version 3: Changed refreshQueueAcls to refreshQueues
    */
-  public static final long versionID = 2L;
+  public static final long versionID = 3L;
 
   /**
-   * Refresh the queue acls in use currently.
+   * Refresh the queues used by the jobtracker and scheduler.
+   * 
+   * Access control lists and queue states are refreshed.
    */
-  void refreshQueueAcls() throws IOException;
+  void refreshQueues() throws IOException;
   
   /**
    * Refresh the node list at the {@link JobTracker} 

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/ClusterStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/ClusterStatus.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/ClusterStatus.java (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/ClusterStatus.java Wed Jun 17 20:55:51 2009
@@ -93,6 +93,16 @@
   }
 
   /**
+   * Construct a new cluster status.
+   * 
+   * @param trackers no. of tasktrackers in the cluster
+   * @param blacklists no of blacklisted task trackers in the cluster
+   * @param ttExpiryInterval the tasktracker expiry interval
+   * @param maps no. of currently running map-tasks in the cluster
+   * @param reduces no. of currently running reduce-tasks in the cluster
+   * @param maxMaps the maximum no. of map tasks in the cluster
+   * @param maxReduces the maximum no. of reduce tasks in the cluster
+   * @param state the {@link JobTracker.State} of the <code>JobTracker</code>
    * @param numDecommissionedNodes number of decommission trackers
    */
   ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 
@@ -133,6 +143,16 @@
   }
 
   /**
+   * Construct a new cluster status.
+   * 
+   * @param activeTrackers active tasktrackers in the cluster
+   * @param blacklistedTrackers blacklisted tasktrackers in the cluster
+   * @param ttExpiryInterval the tasktracker expiry interval
+   * @param maps no. of currently running map-tasks in the cluster
+   * @param reduces no. of currently running reduce-tasks in the cluster
+   * @param maxMaps the maximum no. of map tasks in the cluster
+   * @param maxReduces the maximum no. of reduce tasks in the cluster
+   * @param state the {@link JobTracker.State} of the <code>JobTracker</code>
    * @param numDecommissionNodes number of decommission trackers
    */
   ClusterStatus(Collection<String> activeTrackers, 

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/IsolationRunner.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/IsolationRunner.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/IsolationRunner.java (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/IsolationRunner.java Wed Jun 17 20:55:51 2009
@@ -36,7 +36,17 @@
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.JvmTask;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 
+/**
+ * IsolationRunner is intended to facilitate debugging by re-running a specific
+ * task, given left-over task files for a (typically failed) past job.  
+ * Currently, it is limited to re-running map tasks.
+ *
+ * Users may coerce MapReduce to keep task files around by setting 
+ * keep.failed.task.files.  See mapred_tutorial.xml for more documentation.
+ */
 public class IsolationRunner {
   private static final Log LOG = 
     LogFactory.getLog(IsolationRunner.class.getName());
@@ -109,82 +119,57 @@
     }
   }
   
-  private static ClassLoader makeClassLoader(JobConf conf, 
+  private ClassLoader makeClassLoader(JobConf conf, 
                                              File workDir) throws IOException {
-    List<URL> cp = new ArrayList<URL>();
-
+    List<String> classPaths = new ArrayList();
+    // Add jar clas files (includes lib/* and classes/*)
     String jar = conf.getJar();
-    if (jar != null) {                      // if jar exists, it into workDir
-      File[] libs = new File(workDir, "lib").listFiles();
-      if (libs != null) {
-        for (int i = 0; i < libs.length; i++) {
-          cp.add(new URL("file:" + libs[i].toString()));
-        }
-      }
-      cp.add(new URL("file:" + new File(workDir, "classes/").toString()));
-      cp.add(new URL("file:" + workDir.toString() + "/"));
+    if (jar != null) {
+      TaskRunner.appendJobJarClasspaths(conf.getJar(), classPaths);
     }
-    return new URLClassLoader(cp.toArray(new URL[cp.size()]));
-  }
-  
-  /**
-   * Create empty sequence files for any of the map outputs that we don't have.
-   * @param fs the filesystem to create the files in
-   * @param dir the directory name to create the files in
-   * @param conf the jobconf
-   * @throws IOException if something goes wrong writing
-   */
-  private static void fillInMissingMapOutputs(FileSystem fs, 
-                                              TaskAttemptID taskId,
-                                              int numMaps,
-                                              JobConf conf) throws IOException {
-    Class<? extends WritableComparable> keyClass
-        = conf.getMapOutputKeyClass().asSubclass(WritableComparable.class);
-    Class<? extends Writable> valueClass
-        = conf.getMapOutputValueClass().asSubclass(Writable.class);
-    MapOutputFile namer = new MapOutputFile(taskId.getJobID());
-    namer.setConf(conf);
-    for(int i=0; i<numMaps; i++) {
-      Path f = namer.getInputFile(i, taskId);
-      if (!fs.exists(f)) {
-        LOG.info("Create missing input: " + f);
-        SequenceFile.Writer out =
-          SequenceFile.createWriter(fs, conf, f, keyClass, valueClass);
-        out.close();
-      }
-    }    
+    // Add the workdir, too.
+    classPaths.add(workDir.toString());
+    // Note: TaskRunner.run() does more, including DistributedCache files.
+    
+    // Convert to URLs
+    URL[] urls = new URL[classPaths.size()];
+    for (int i = 0; i < classPaths.size(); ++i) {
+      urls[i] = new File(classPaths.get(i)).toURL();
+    }
+    return new URLClassLoader(urls);
   }
   
   /**
-   * Run a single task
-   * @param args the first argument is the task directory
+   * Main method.
    */
-  public static void main(String[] args
-                          ) throws ClassNotFoundException, IOException, 
-                                   InterruptedException {
+  boolean run(String[] args) 
+      throws ClassNotFoundException, IOException, InterruptedException {
     if (args.length != 1) {
       System.out.println("Usage: IsolationRunner <path>/job.xml");
-      System.exit(1);
+      return false;
     }
     File jobFilename = new File(args[0]);
     if (!jobFilename.exists() || !jobFilename.isFile()) {
       System.out.println(jobFilename + " is not a valid job file.");
-      System.exit(1);
+      return false;
     }
     JobConf conf = new JobConf(new Path(jobFilename.toString()));
     TaskAttemptID taskId = TaskAttemptID.forName(conf.get("mapred.task.id"));
+    if (taskId == null) {
+      System.out.println("mapred.task.id not found in configuration;" + 
+          " job.xml is not a task config");
+    }
     boolean isMap = conf.getBoolean("mapred.task.is.map", true);
+    if (!isMap) {
+      System.out.println("Only map tasks are supported.");
+      return false;
+    }
     int partition = conf.getInt("mapred.task.partition", 0);
     
     // setup the local and user working directories
     FileSystem local = FileSystem.getLocal(conf);
     LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");
-    File workDirName = new File(lDirAlloc.getLocalPathToRead(
-                                  TaskTracker.getLocalTaskDir(
-                                    taskId.getJobID().toString(), 
-                                    taskId.toString())
-                                  + Path.SEPARATOR + "work",
-                                  conf). toString());
+    File workDirName = TaskRunner.formWorkDir(lDirAlloc, taskId, false, conf);
     local.setWorkingDirectory(new Path(workDirName.toString()));
     FileSystem.get(conf).setWorkingDirectory(conf.getWorkingDirectory());
     
@@ -193,23 +178,29 @@
     Thread.currentThread().setContextClassLoader(classLoader);
     conf.setClassLoader(classLoader);
     
-    Task task;
-    if (isMap) {
-      Path localSplit = new Path(new Path(jobFilename.toString()).getParent(), 
-                                 "split.dta");
-      DataInputStream splitFile = FileSystem.getLocal(conf).open(localSplit);
-      String splitClass = Text.readString(splitFile);
-      BytesWritable split = new BytesWritable();
-      split.readFields(splitFile);
-      splitFile.close();
-      task = new MapTask(jobFilename.toString(), taskId, partition, splitClass, split);
-    } else {
-      int numMaps = conf.getNumMapTasks();
-      fillInMissingMapOutputs(local, taskId, numMaps, conf);
-      task = new ReduceTask(jobFilename.toString(), taskId, partition, numMaps);
-    }
+    Path localSplit = new Path(new Path(jobFilename.toString()).getParent(), 
+                               "split.dta");
+    DataInputStream splitFile = FileSystem.getLocal(conf).open(localSplit);
+    String splitClass = Text.readString(splitFile);
+    BytesWritable split = new BytesWritable();
+    split.readFields(splitFile);
+    splitFile.close();
+    Task task = new MapTask(jobFilename.toString(), taskId, partition, splitClass, split);
     task.setConf(conf);
     task.run(conf, new FakeUmbilical());
+    return true;
   }
 
+
+  /**
+   * Run a single task.
+   *
+   * @param args the first argument is the task directory
+   */
+  public static void main(String[] args) 
+      throws ClassNotFoundException, IOException, InterruptedException {
+    if (!new IsolationRunner().run(args)) {
+      System.exit(1);
+    }
+  }
 }

Modified: hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobHistory.java?rev=785794&r1=785793&r2=785794&view=diff
==============================================================================
--- hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/core/branches/HADOOP-4687/mapred/src/java/org/apache/hadoop/mapred/JobHistory.java Wed Jun 17 20:55:51 2009
@@ -398,6 +398,13 @@
   }
   
   /**
+   * Get the history location
+   */
+  static Path getJobHistoryLocation() {
+    return new Path(LOG_DIR);
+  } 
+  
+  /**
    * Base class contais utility stuff to manage types key value pairs with enums. 
    */
   static class KeyValuePair{



Mime
View raw message