hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1137300 - in /hadoop/common/branches/branch-0.20-security-204: ./ ivy/ src/contrib/ src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ src/contrib/data_join/ src/contrib/fairscheduler/ src/contrib/hdfsproxy/ src/contrib/hdfs...
Date Sun, 19 Jun 2011 06:30:56 GMT
Author: omalley
Date: Sun Jun 19 06:30:55 2011
New Revision: 1137300

URL: http://svn.apache.org/viewvc?rev=1137300&view=rev
Log:
Fix up the merges from the 205 branch.

Added:
    hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueMetrics.java
      - copied unchanged from r1131280, hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/QueueMetrics.java
    hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/mapred/TestQueueMetrics.java
      - copied unchanged from r1131280, hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/mapred/TestQueueMetrics.java
Modified:
    hadoop/common/branches/branch-0.20-security-204/CHANGES.txt   (props changed)
    hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties
    hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml
    hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
    hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml
    hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml
    hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml
    hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
    hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml
    hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties
    hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java
    hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
    hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
    hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
    hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java
    hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java

Propchange: hadoop/common/branches/branch-0.20-security-204/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sun Jun 19 06:30:55 2011
@@ -1,5 +1,5 @@
 /hadoop/common/branches/branch-0.20/CHANGES.txt:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,1044225
-/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837,1100336,1101315,1101629,1101729,1102378,1102869,1103940,1125139,1125170,1125587,1125589,1127362,1130409,1131277,1131286,1131290,1131299,1131461,1131737,1134140
+/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837,1100336,1101315,1101629,1101729,1102378,1102869,1103940,1125139,1125170,1125587,1125589,1127362,1130409,1131277,1131280,1131286,1131290,1131299,1131461,1131737,1134140
 /hadoop/common/branches/branch-0.20-security-203/CHANGES.txt:1096071,1097012-1099333,1102071,1128115
 /hadoop/common/branches/branch-0.20-security-205/CHANGES.txt:1132788,1133133,1133274,1133282,1133475
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226

Modified: hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties (original)
+++ hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties Sun Jun 19 06:30:55
2011
@@ -47,6 +47,7 @@ hsqldb.version=1.8.0.10
 ivy.version=2.1.0
 
 jasper.version=5.5.12
+jackson.version=1.0.1
 #not able to figureout the version of jsp & jsp-api version to get it resolved throught
ivy
 # but still declared here as we are going to have a local copy from the lib folder
 jsp.version=2.1

Modified: hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml Sun Jun
19 06:30:55 2011
@@ -283,7 +283,7 @@
     <mkdir dir="${hadoop.log.dir}"/>
     <junit
       printsummary="yes" showoutput="${test.output}" 
-      haltonfailure="no" fork="yes" maxmemory="256m"
+      haltonfailure="no" fork="yes" maxmemory="512m"
       errorProperty="tests.failed" failureProperty="tests.failed"
       timeout="${test.timeout}">
       

Modified: hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
Sun Jun 19 06:30:55 2011
@@ -432,7 +432,8 @@ public class TestCapacityScheduler exten
       queues.clear();
       for (String qName : newQueues) {
         try {
-          queues.put(qName, new Queue(qName, acls, Queue.QueueState.RUNNING));
+          queues.put(qName, new Queue(qName, acls, Queue.QueueState.RUNNING,
+                     QueueMetrics.create(qName, new Configuration())));
         } catch (Throwable t) {
           throw new RuntimeException("Unable to initialize queue " + qName, t);
         }

Modified: hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml Sun Jun
19 06:30:55 2011
@@ -40,5 +40,29 @@
       name="commons-math"
       rev="${commons-math.version}"
       conf="common->default"/>
+    <dependency org="junit"
+      name="junit"
+      rev="${junit.version}"
+      conf="common->default"/>
+    <dependency org="org.mortbay.jetty"
+      name="jetty-util"
+      rev="${jetty-util.version}"
+      conf="common->master"/>
+    <dependency org="org.mortbay.jetty"
+      name="jetty"
+      rev="${jetty.version}"
+      conf="common->default"/>
+    <dependency org="org.codehaus.jackson"
+      name="jackson-core-asl"
+      rev="${jackson.version}"
+      conf="common->default"/>
+    <dependency org="org.codehaus.jackson"
+      name="jackson-mapper-asl"
+      rev="${jackson.version}"
+      conf="common->default"/>
+    <dependency org="commons-httpclient"
+      name="commons-httpclient"
+      rev="${commons-httpclient.version}"
+      conf="common->master"/> 
     </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml Sun
Jun 19 06:30:55 2011
@@ -47,8 +47,32 @@
       rev="${jsp-api-2.1.version}"
       conf="common->master"/>
     <dependency org="org.mortbay.jetty"
-      name="jsp-2.1"
-      rev="${jsp-2.1.version}"
+      name="jetty"
+      rev="${jetty.version}"
+      conf="common->default"/>
+    <dependency org="org.codehaus.jackson"
+      name="jackson-core-asl"
+      rev="${jackson.version}"
+      conf="common->default"/>
+    <dependency org="org.codehaus.jackson"
+      name="jackson-mapper-asl"
+      rev="${jackson.version}"
+      conf="common->default"/>
+    <dependency org="commons-httpclient"
+      name="commons-httpclient"
+      rev="${commons-httpclient.version}"
+      conf="common->master"/> 
+    <dependency org="commons-configuration"
+      name="commons-configuration"
+      rev="${commons-configuration.version}"
+      conf="common->master"/>
+    <dependency org="org.apache.commons"
+      name="commons-math"
+      rev="${commons-math.version}"
+      conf="common->master"/>
+    <dependency org="commons-lang"
+      name="commons-lang"
+      rev="${commons-lang.version}"
       conf="common->master"/>
   </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml Sun Jun
19 06:30:55 2011
@@ -80,7 +80,8 @@
     </and>
   </condition>
 
-  <property name="ivy.settings.file" location="${hadoop.root}/ivy/ivysettings.xml"/>
+  <property name="ivy.settings.file" 
+            location="${hadoop.root}/ivy/ivysettings.xml"/>
 
   <target name="ivy-init" depends="ivy-init-antlib">
     <ivy:settings id="${ant.project.name}.ivy.settings"/>
@@ -239,7 +240,8 @@
     <copy file="${proxy.conf.dir}/tomcat-web.xml" todir="${src.test.resources}"/>
     <copy file="${proxy.conf.dir}/tomcat-forward-web.xml" todir="${src.test.resources}"/>
 
-    <junit fork="yes" printsummary="yes" errorProperty="tests.failed" failureProperty="tests.failed">
+    <junit fork="yes" printsummary="yes" errorProperty="tests.failed" 
+           failureProperty="tests.failed" showoutput="${test.output}">
       <classpath refid="test.classpath"/>
       <sysproperty key="test.build.data" value="${build.test}/data"/>
       <sysproperty key="build.test" value="${build.test}"/>

Modified: hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
Sun Jun 19 06:30:55 2011
@@ -26,6 +26,7 @@ import java.util.Random;
 
 import junit.framework.TestCase;
 
+import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.log4j.Level;
@@ -46,13 +47,8 @@ import org.apache.hadoop.security.author
  * A JUnit test for HdfsProxy
  */
 public class TestHdfsProxy extends TestCase {
-  {
-    ((Log4JLogger) LogFactory.getLog("org.apache.hadoop.hdfs.StateChange"))
-        .getLogger().setLevel(Level.OFF);
-    ((Log4JLogger) DataNode.LOG).getLogger().setLevel(Level.OFF);
-    ((Log4JLogger) FSNamesystem.LOG).getLogger().setLevel(Level.OFF);
-  }
 
+  static final Log LOG = LogFactory.getLog(TestHdfsProxy.class);
   static final URI LOCAL_FS = URI.create("file:///");
 
   private static final int NFILES = 10;
@@ -221,12 +217,13 @@ public class TestHdfsProxy extends TestC
       final FileSystem localfs = FileSystem.get(LOCAL_FS, dfsConf);
       final FileSystem hdfs = cluster.getFileSystem();
       final Configuration proxyConf = new Configuration(false);
-      proxyConf.set("hdfsproxy.dfs.namenode.address", hdfs.getUri().getHost() + ":"
-          + hdfs.getUri().getPort());
+      proxyConf.set("hdfsproxy.dfs.namenode.address", hdfs.getUri().getHost() +
+          ":" + hdfs.getUri().getPort());
       proxyConf.set("hdfsproxy.https.address", "localhost:0");
       final String namenode = hdfs.getUri().toString();
       if (namenode.startsWith("hdfs://")) {
         MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR + "/srcdat");
+
         hdfs.copyFromLocalFile
 	    (new Path("file:///" + TEST_ROOT_DIR + "/srcdat"),
              new Path(namenode + "/destdat" ));
@@ -240,7 +237,8 @@ public class TestHdfsProxy extends TestC
         final String realProxyAddr = proxyAddr.getHostName() + ":"
             + proxy.getPort();
         final Path proxyUrl = new Path("hftp://" + realProxyAddr);
-	final FileSystem hftp = proxyUrl.getFileSystem(dfsConf);
+        final FileSystem hftp = proxyUrl.getFileSystem(dfsConf);
+
         FileUtil.copy(hftp, new Path(proxyUrl, "/destdat"),
                       hdfs, new Path(namenode + "/copied1"),
                       false, true, proxyConf);
@@ -260,13 +258,21 @@ public class TestHdfsProxy extends TestC
         deldir(localfs, TEST_ROOT_DIR + "/srcdat");
         deldir(localfs, TEST_ROOT_DIR + "/copied2");
       }
-    } finally {
       if (cluster != null) {
         cluster.shutdown();
       }
       if (proxy != null) {
         proxy.stop();
       }
+    } catch (Exception t) {
+      LOG.fatal("caught exception in test", t);
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+      if (proxy != null) {
+        proxy.stop();
+      }
+      throw t;
     }
   }
 }

Modified: hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml Sun Jun 19 06:30:55
2011
@@ -32,6 +32,18 @@
       name="log4j"
       rev="${log4j.version}"
       conf="common->master"/>
+    <dependency org="junit"
+      name="junit"
+      rev="${junit.version}"
+      conf="common->default"/>
+    <dependency org="org.mortbay.jetty"
+      name="jetty-util"
+      rev="${jetty-util.version}"
+      conf="common->master"/>
+    <dependency org="org.mortbay.jetty"
+      name="jetty"
+      rev="${jetty.version}"
+      conf="common->default"/>
     <dependency org="org.apache.lucene"
       name="lucene-core"
       rev="${lucene-core.version}"
@@ -44,5 +56,9 @@
       name="commons-math"
       rev="${commons-math.version}"
       conf="common->default"/>
+    <dependency org="org.codehaus.jackson"
+      name="jackson-core-asl"
+      rev="${jackson.version}"
+      conf="common->default"/>
     </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties
Sun Jun 19 06:30:55 2011
@@ -4,6 +4,5 @@
 #Please list the dependencies name with version if they are different from the ones 
 #listed in the global libraries.properties file (in alphabetical order)
 
-jackson.version=1.0.1
 commons-configuration.version=1.6
 commons-math.version=2.1

Modified: hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java
Sun Jun 19 06:30:55 2011
@@ -22,17 +22,20 @@ import java.io.*;
 import java.util.Enumeration;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.mortbay.log.Log;
 
 /**
  * A collection of file-processing util methods
  */
 public class FileUtil {
+  private static final Log LOG = LogFactory.getLog(FileUtil.class);
+
   /**
    * convert an array of FileStatus to an array of Path
    * 
@@ -598,9 +601,9 @@ public class FileUtil {
     try {
       shExec.execute();
     }catch(IOException e) {
-      if(Log.isDebugEnabled()) {
-        Log.debug("Error while changing permission : " + filename 
-            +" Exception: " + StringUtils.stringifyException(e));
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Error while changing permission : " + filename 
+                  +" Exception: " + StringUtils.stringifyException(e));
       }
     }
     return shExec.getExitCode();

Modified: hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
Sun Jun 19 06:30:55 2011
@@ -93,8 +93,8 @@ public class TrackerDistributedCacheMana
   
   private static final Random random = new Random();
   
-  BaseDirManager baseDirManager = new BaseDirManager();
-  CleanupThread cleanupThread;
+  protected BaseDirManager baseDirManager = new BaseDirManager();
+  protected CleanupThread cleanupThread;
 
   public TrackerDistributedCacheManager(Configuration conf,
                                         TaskController controller
@@ -874,7 +874,7 @@ public class TrackerDistributedCacheMana
   /**
    * A thread to check and cleanup the unused files periodically
    */
-  private class CleanupThread extends Thread {
+  protected class CleanupThread extends Thread {
     // How often do we check if we need to clean up cache files?
     private long cleanUpCheckPeriod = 60000L; // 1 minute
     public CleanupThread(Configuration conf) {
@@ -882,6 +882,7 @@ public class TrackerDistributedCacheMana
         conf.getLong("mapreduce.tasktracker.distributedcache.checkperiod",
             cleanUpCheckPeriod);
     }
+
     private volatile boolean running = true;
     
     public void stopRunning() {
@@ -894,19 +895,33 @@ public class TrackerDistributedCacheMana
         try {
           Thread.sleep(cleanUpCheckPeriod);
           baseDirManager.checkAndCleanup();
-        } catch (Exception e) {
+        } catch (IOException e) {
           LOG.error("Exception in DistributedCache CleanupThread.", e);
-          // This thread should keep running and never crash.
+        } catch(InterruptedException e) {
+          LOG.info("Cleanup...",e); 
+          //To force us to exit cleanly
+          running = false;
+        } catch (Throwable t) {
+          exitTaskTracker(t);
         }
       }
     }
+    
+    /**
+     * Exit the task tracker because of a fatal error.
+     */
+    protected void exitTaskTracker(Throwable t) {
+      LOG.fatal("Distributed Cache cleanup thread received runtime exception." +
+      		" Exiting the TaskTracker", t);
+      Runtime.getRuntime().exit(-1);
+    }
   }
 
   /**
    * This class holds properties of each base directories and is responsible
    * for clean up unused cache files in base directories.
    */
-  private class BaseDirManager {
+  protected class BaseDirManager {
 
     // For holding the properties of each cache directory
     private class CacheDir {

Modified: hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Sun Jun 19 06:30:55 2011
@@ -91,6 +91,7 @@ public class JobInProgress {
   JobStatus status;
   String jobFile = null;
   Path localJobFile = null;
+  final QueueMetrics queueMetrics;
 
   TaskInProgress maps[] = new TaskInProgress[0];
   TaskInProgress reduces[] = new TaskInProgress[0];
@@ -339,8 +340,9 @@ public class JobInProgress {
     this.resourceEstimator = new ResourceEstimator(this);
     this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.PREP);
     this.status.setUsername(conf.getUser());
+    String queueName = conf.getQueueName();
     this.profile = new JobProfile(conf.getUser(), jobid, "", "",
-                                  conf.getJobName(), conf.getQueueName());
+                                  conf.getJobName(), queueName);
     this.memoryPerMap = conf.getMemoryForMapTask();
     this.memoryPerReduce = conf.getMemoryForReduceTask();
     this.maxTaskFailuresPerTracker = conf.getMaxTaskFailuresPerTracker();
@@ -377,6 +379,7 @@ public class JobInProgress {
       this.status = new JobStatus(jobId, 0.0f, 0.0f, JobStatus.PREP);
       this.status.setUsername(jobInfo.getUser().toString());
       this.jobtracker.getInstrumentation().addPrepJob(conf, jobId);
+      // Add the queue-level metric below (after the profile has been initialized)
       this.startTime = jobtracker.getClock().getTime();
       status.setStartTime(startTime);
       this.localFs = jobtracker.getLocalFileSystem();
@@ -424,9 +427,9 @@ public class JobInProgress {
       
       this.priority = conf.getJobPriority();
       this.status.setJobPriority(this.priority);
+      String queueName = conf.getQueueName();
       this.profile = new JobProfile(user, jobId, 
-          jobFile, url, conf.getJobName(),
-          conf.getQueueName());
+          jobFile, url, conf.getJobName(), queueName);
 
       Queue queue = this.jobtracker.getQueueManager().getQueue(queueName);
       if (queue == null) {
@@ -487,7 +490,15 @@ public class JobInProgress {
       FileSystem.closeAllForUGI(UserGroupInformation.getCurrentUser());
     }
   }
-    
+
+  /**
+   * Get the QueueMetrics object associated with this job
+   * @return QueueMetrics
+   */
+  public QueueMetrics getQueueMetrics() {
+    return this.queueMetrics;
+  }
+
   private void checkTaskLimits() throws IOException {
     // if the number of tasks is larger than a configured value
     // then fail the job.
@@ -695,6 +706,8 @@ public class JobInProgress {
 
     jobtracker.getInstrumentation().addWaitingMaps(getJobID(), numMapTasks);
     jobtracker.getInstrumentation().addWaitingReduces(getJobID(), numReduceTasks);
+    this.queueMetrics.addWaitingMaps(getJobID(), numMapTasks);
+    this.queueMetrics.addWaitingReduces(getJobID(), numReduceTasks);
 
     maps = new TaskInProgress[numMapTasks];
     for(int i=0; i < numMapTasks; ++i) {
@@ -1695,6 +1708,7 @@ public class JobInProgress {
       if (tip.getActiveTasks().size() > 1)
         speculativeMapTasks++;
       metrics.launchMap(id);
+      this.queueMetrics.launchMap(id);
     } else {
       ++runningReduceTasks;
       name = Values.REDUCE.name();
@@ -1702,6 +1716,7 @@ public class JobInProgress {
       if (tip.getActiveTasks().size() > 1)
         speculativeReduceTasks++;
       metrics.launchReduce(id);
+      this.queueMetrics.launchReduce(id);
     }
     // Note that the logs are for the scheduled tasks only. Tasks that join on 
     // restart has already their logs in place.
@@ -1852,9 +1867,11 @@ public class JobInProgress {
     map.put(taskTracker, info);
     if (type == TaskType.MAP) {
       jobtracker.getInstrumentation().addReservedMapSlots(reservedSlots);
+      this.queueMetrics.addReservedMapSlots(reservedSlots);
     }
     else {
       jobtracker.getInstrumentation().addReservedReduceSlots(reservedSlots);
+      this.queueMetrics.addReservedReduceSlots(reservedSlots);
     }
     jobtracker.incrementReservations(type, reservedSlots);
   }
@@ -1884,10 +1901,12 @@ public class JobInProgress {
     map.remove(taskTracker);
     if (type == TaskType.MAP) {
       jobtracker.getInstrumentation().decReservedMapSlots(info.getNumSlots());
+      this.queueMetrics.decReservedMapSlots(info.getNumSlots());
     }
     else {
       jobtracker.getInstrumentation().decReservedReduceSlots(
         info.getNumSlots());
+      this.queueMetrics.decReservedReduceSlots(info.getNumSlots());
     }
     jobtracker.decrementReservations(type, info.getNumSlots());
   }
@@ -2596,6 +2615,7 @@ public class JobInProgress {
       }
       finishedMapTasks += 1;
       metrics.completeMap(taskid);
+      this.queueMetrics.completeMap(taskid);
       // remove the completed map from the resp running caches
       retireMap(tip);
       if ((finishedMapTasks + failedMapTIPs) == (numMapTasks)) {
@@ -2611,6 +2631,7 @@ public class JobInProgress {
       }
       finishedReduceTasks += 1;
       metrics.completeReduce(taskid);
+      this.queueMetrics.completeReduce(taskid);
       // remove the completed reduces from the running reducers set
       retireReduce(tip);
       if ((finishedReduceTasks + failedReduceTIPs) == (numReduceTasks)) {
@@ -2655,14 +2676,18 @@ public class JobInProgress {
     //update the metrics
     if (oldState == JobStatus.PREP) {
       this.jobtracker.getInstrumentation().decPrepJob(conf, jobId);
+      this.queueMetrics.decPrepJob(conf, jobId);
     } else if (oldState == JobStatus.RUNNING) {
       this.jobtracker.getInstrumentation().decRunningJob(conf, jobId);
+      this.queueMetrics.decRunningJob(conf, jobId);
     }
     
     if (newState == JobStatus.PREP) {
       this.jobtracker.getInstrumentation().addPrepJob(conf, jobId);
+      this.queueMetrics.addPrepJob(conf, jobId);
     } else if (newState == JobStatus.RUNNING) {
       this.jobtracker.getInstrumentation().addRunningJob(conf, jobId);
+      this.queueMetrics.addRunningJob(conf, jobId);
     }
     
   }
@@ -2717,6 +2742,7 @@ public class JobInProgress {
       garbageCollect();
       
       metrics.completeJob(this.conf, this.status.getJobID());
+      this.queueMetrics.completeJob(this.conf, this.status.getJobID());
     }
   }
   
@@ -2757,9 +2783,11 @@ public class JobInProgress {
       if (jobTerminationState == JobStatus.FAILED) {
         jobtracker.getInstrumentation().failedJob(
             this.conf, this.status.getJobID());
+        this.queueMetrics.failedJob(this.conf, this.status.getJobID());
       } else {
         jobtracker.getInstrumentation().killedJob(
             this.conf, this.status.getJobID());
+        this.queueMetrics.killedJob(this.conf, this.status.getJobID());
       }
     }
   }
@@ -2910,9 +2938,11 @@ public class JobInProgress {
         if (tip.isMapTask() && !metricsDone) {
           runningMapTasks -= 1;
           metrics.failedMap(taskid);
+          this.queueMetrics.failedMap(taskid);
         } else if (!metricsDone) {
           runningReduceTasks -= 1;
           metrics.failedReduce(taskid);
+          this.queueMetrics.failedReduce(taskid);
         }
       }
       
@@ -3155,6 +3185,8 @@ public class JobInProgress {
       // Let the JobTracker know that a job is complete
       jobtracker.getInstrumentation().decWaitingMaps(getJobID(), pendingMaps());
       jobtracker.getInstrumentation().decWaitingReduces(getJobID(), pendingReduces());
+      this.queueMetrics.decWaitingMaps(getJobID(), pendingMaps());
+      this.queueMetrics.decWaitingReduces(getJobID(), pendingReduces());
       jobtracker.storeCompletedJob(this);
       jobtracker.finalizeJob(this);
 

Modified: hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Sun Jun 19 06:30:55 2011
@@ -4028,6 +4028,8 @@ public class JobTracker implements MRCon
       }
     }
     myInstrumentation.submitJob(job.getJobConf(), jobId);
+    job.getQueueMetrics().submitJob(job.getJobConf(), jobId);
+
     LOG.info("Job " + jobId + " added successfully for user '" 
              + job.getJobConf().getUser() + "' to queue '" 
              + job.getJobConf().getQueueName() + "'");

Modified: hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
Sun Jun 19 06:30:55 2011
@@ -61,6 +61,7 @@ class LocalJobRunner implements JobSubmi
   private final TaskController taskController = new DefaultTaskController();
 
   private JobTrackerInstrumentation myMetrics = null;
+  private QueueMetrics queueMetrics = null;
 
   private static final String jobDir =  "localRunner/";
   
@@ -207,8 +208,10 @@ class LocalJobRunner implements JobSubmi
             map.setConf(localConf);
             map_tasks += 1;
             myMetrics.launchMap(mapId);
+            queueMetrics.launchMap(mapId);
             map.run(localConf, this);
             myMetrics.completeMap(mapId);
+            queueMetrics.completeMap(mapId);
             map_tasks -= 1;
             updateCounters(map);
           } else {
@@ -253,8 +256,10 @@ class LocalJobRunner implements JobSubmi
               reduce.setConf(localConf);
               reduce_tasks += 1;
               myMetrics.launchReduce(reduce.getTaskID());
+              queueMetrics.launchReduce(reduce.getTaskID());
               reduce.run(localConf, this);
               myMetrics.completeReduce(reduce.getTaskID());
+              queueMetrics.completeReduce(reduce.getTaskID());
               reduce_tasks -= 1;
               updateCounters(reduce);
             } else {
@@ -413,6 +418,7 @@ class LocalJobRunner implements JobSubmi
     this.fs = FileSystem.getLocal(conf);
     this.conf = conf;
     myMetrics = JobTrackerInstrumentation.create(null, new JobConf(conf));
+    queueMetrics = QueueMetrics.create(conf.getQueueName(), new JobConf(conf));
     taskController.setConf(conf);
   }
 

Modified: hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java
Sun Jun 19 06:30:55 2011
@@ -37,6 +37,7 @@ class Queue {
   private String name;
   private Map<String,AccessControlList> acls;
   private QueueState state = QueueState.RUNNING;
+  private QueueMetrics queueMetrics;
 
   /**
    * An Object that can be used by schedulers to fill in
@@ -69,10 +70,12 @@ class Queue {
    * @param acls ACLs for the queue
    * @param state state of the queue
    */
-  Queue(String name, Map<String, AccessControlList> acls, QueueState state) {
+  Queue(String name, Map<String, AccessControlList> acls, QueueState state,
+        QueueMetrics metrics) {
 	  this.name = name;
 	  this.acls = acls;
 	  this.state = state;
+	  this.queueMetrics = metrics;
   }
 
   /**
@@ -149,4 +152,12 @@ class Queue {
   void setSchedulingInfo(Object schedulingInfo) {
     this.schedulingInfo = schedulingInfo;
   }
+
+  /**
+   * Return the QueueMetrics object for this queue
+   * @return QueueMetrics
+   */
+  public QueueMetrics getMetrics() {
+    return this.queueMetrics;
+  }
 }

Modified: hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java
Sun Jun 19 06:30:55 2011
@@ -118,7 +118,7 @@ class QueueManager {
         LOG.error("The queue, " + name + " does not have a configured ACL list");
       }
       queues.put(name, new Queue(name, getQueueAcls(name, conf),
-          getQueueState(name, conf)));
+          getQueueState(name, conf), QueueMetrics.create(name, conf)));
     }
     
     return queues;
@@ -136,7 +136,17 @@ class QueueManager {
   public synchronized Set<String> getQueues() {
     return queues.keySet();
   }
-  
+
+  /**
+   * Return a specific queue configured in the system.
+   * 
+   * @param queueName Name of the queue requested
+   * @return Queue object corresponding to queueName
+   */
+  public synchronized Queue getQueue(String queueName) {
+    return queues.get(queueName);
+  }
+
   /**
    * Return true if the given user is part of the ACL for the given
    * {@link QueueACL} name for the given queue.



Mime
View raw message