hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1076952 - in /hadoop/common/branches/branch-0.20-security-patches/src/contrib: ./ capacity-scheduler/src/test/org/apache/hadoop/mapred/
Date Fri, 04 Mar 2011 03:25:27 GMT
Author: omalley
Date: Fri Mar  4 03:25:26 2011
New Revision: 1076952

URL: http://svn.apache.org/viewvc?rev=1076952&view=rev
Log:
commit ca5451bbfca5210a230c504cf5646e66fda7a202
Author: Lee Tucker <ltucker@yahoo-inc.com>
Date:   Thu Jul 30 17:40:40 2009 -0700

    Applying patch 2855416.mr522.patch

Added:
    hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java
Removed:
    hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestJobInitialization.java
    hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestQueueCapacities.java
Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml
    hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml?rev=1076952&r1=1076951&r2=1076952&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/contrib/build-contrib.xml Fri
Mar  4 03:25:26 2011
@@ -96,6 +96,7 @@
     <pathelement location="${conf.dir}"/>
     <pathelement location="${hadoop.root}/build"/>
     <pathelement location="${build.examples}"/>
+    <pathelement location="${hadoop.root}/build/examples"/>
     <path refid="contrib-classpath"/>
   </path>
 

Modified: hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java?rev=1076952&r1=1076951&r2=1076952&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java
Fri Mar  4 03:25:26 2011
@@ -146,7 +146,7 @@ public class ClusterWithCapacitySchedule
   }
 
   protected JobConf getJobConf() {
-    return this.jobConf;
+    return new JobConf(this.jobConf);
   }
 
   protected JobTracker getJobTracker() {

Added: hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java?rev=1076952&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java
(added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java
Fri Mar  4 03:25:26 2011
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.hadoop.examples.SleepJob;
+
+
+public class TestCapacitySchedulerWithJobTracker extends
+    ClusterWithCapacityScheduler {
+
+  /**
+   * Test case which checks if the jobs which fail initialization are removed
+   * from the {@link CapacityTaskScheduler} waiting queue.
+   * 
+   * @throws Exception
+   */
+  public void testFailingJobInitalization() throws Exception {
+    Properties schedulerProps = new Properties();
+    schedulerProps.put("mapred.capacity-scheduler.queue.default.capacity",
+        "100");
+    Properties clusterProps = new Properties();
+    clusterProps.put("mapred.tasktracker.map.tasks.maximum", String.valueOf(1));
+    clusterProps.put("mapred.tasktracker.reduce.tasks.maximum", String
+        .valueOf(1));
+    clusterProps.put("mapred.jobtracker.maxtasks.per.job", String.valueOf(1));
+    // cluster capacity 1 maps, 1 reduces
+    startCluster(1, clusterProps, schedulerProps);
+    JobConf conf = getJobConf();
+    conf.setSpeculativeExecution(false);
+    conf.set("mapred.committer.job.setup.cleanup.needed", "false");
+    conf.setNumTasksToExecutePerJvm(-1);
+    SleepJob sleepJob = new SleepJob();
+    sleepJob.setConf(conf);
+    JobConf job = sleepJob.setupJobConf(3, 3, 1, 1, 1, 1);
+    RunningJob rjob;
+    try {
+      rjob = runJob(job, false);
+      fail("The job should have thrown Exception");
+    } catch (Exception e) {
+      CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker()
+          .getTaskScheduler();
+      JobQueuesManager mgr = scheduler.jobQueuesManager;
+      assertEquals("Failed job present in Waiting queue", 0, mgr
+          .getWaitingJobCount("default"));
+    }
+  }
+
+  /**
+   * Test case which checks {@link JobTracker} and {@link CapacityTaskScheduler}
+   * 
+   * Test case submits 2 jobs in two different capacity scheduler queues. And
+   * checks if the jobs successfully complete.
+   * 
+   * @throws Exception
+   */
+  public void testJobTrackerIntegration() throws Exception {
+
+    Properties schedulerProps = new Properties();
+    String[] queues = new String[] { "Q1", "Q2" };
+    RunningJob jobs[] = new RunningJob[2];
+    for (String q : queues) {
+      schedulerProps.put(CapacitySchedulerConf
+          .toFullPropertyName(q, "capacity"), "50");
+      schedulerProps.put(CapacitySchedulerConf.toFullPropertyName(q,
+          "minimum-user-limit-percent"), "100");
+    }
+
+    Properties clusterProps = new Properties();
+    clusterProps.put("mapred.tasktracker.map.tasks.maximum", String.valueOf(2));
+    clusterProps.put("mapred.tasktracker.reduce.tasks.maximum", String
+        .valueOf(2));
+    clusterProps.put("mapred.queue.names", queues[0] + "," + queues[1]);
+    startCluster(2, clusterProps, schedulerProps);
+
+    JobConf conf = getJobConf();
+    conf.setSpeculativeExecution(false);
+    conf.set("mapred.committer.job.setup.cleanup.needed", "false");
+    conf.setNumTasksToExecutePerJvm(-1);
+    conf.setQueueName(queues[0]);
+    SleepJob sleepJob1 = new SleepJob();
+    sleepJob1.setConf(conf);
+    JobConf sleepJobConf = sleepJob1.setupJobConf(1, 1, 1, 1, 1, 1);
+    jobs[0] = runJob(sleepJobConf, true);
+
+    JobConf conf2 = getJobConf();
+    conf2.setSpeculativeExecution(false);
+    conf2.set("mapred.committer.job.setup.cleanup.needed", "false");
+    conf2.setNumTasksToExecutePerJvm(-1);
+    conf2.setQueueName(queues[1]);
+    SleepJob sleepJob2 = new SleepJob();
+    sleepJob2.setConf(conf2);
+    JobConf sleep2 = sleepJob2.setupJobConf(3, 3, 5, 3, 5, 3);
+    jobs[1] = runJob(sleep2, false);
+    assertTrue("Sleep job submitted to queue 1 is not successful", jobs[0]
+        .isSuccessful());
+    assertTrue("Sleep job submitted to queue 2 is not successful", jobs[1]
+        .isSuccessful());
+  }
+
+  private RunningJob runJob(JobConf conf, boolean inBackGround)
+      throws IOException {
+    if (!inBackGround) {
+      RunningJob rjob = JobClient.runJob(conf);
+      return rjob;
+    } else {
+      RunningJob rJob = new JobClient(conf).submitJob(conf);
+      return rJob;
+    }
+  }
+}



Mime
View raw message