Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 12069 invoked from network); 19 Oct 2009 11:05:20 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 19 Oct 2009 11:05:20 -0000 Received: (qmail 11279 invoked by uid 500); 19 Oct 2009 11:05:19 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 11227 invoked by uid 500); 19 Oct 2009 11:05:19 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 11217 invoked by uid 99); 19 Oct 2009 11:05:19 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 19 Oct 2009 11:05:19 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 19 Oct 2009 11:05:16 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 231B323888D7; Mon, 19 Oct 2009 11:04:54 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r826635 - in /hadoop/mapreduce/trunk: ./ src/examples/org/apache/hadoop/examples/pi/ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapreduce/ src/java/org/apache/hadoop/mapreduce/protocol/ src/java/org/apache/hadoop/mapreduc... Date: Mon, 19 Oct 2009 11:04:53 -0000 To: mapreduce-commits@hadoop.apache.org From: sharad@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20091019110454.231B323888D7@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: sharad Date: Mon Oct 19 11:04:53 2009 New Revision: 826635 URL: http://svn.apache.org/viewvc?rev=826635&view=rev Log: MAPREDUCE-1048. (Revert) Add occupied/reserved slot usage summary on jobtracker UI. Removed: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java Modified: hadoop/mapreduce/trunk/CHANGES.txt hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp Modified: hadoop/mapreduce/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/CHANGES.txt (original) +++ hadoop/mapreduce/trunk/CHANGES.txt Mon Oct 19 11:04:53 2009 @@ -10,9 +10,6 @@ MAPREDUCE-999. Improve Sqoop test speed and refactor tests. (Aaron Kimball via tomwhite) - - MAPREDUCE-1048. Add occupied/reserved slot usage summary on jobtracker UI. - (Amareshwari Sriramadasu via sharad) MAPREDUCE-906. Update Sqoop documentation. (Aaron Kimball via cdouglas) Modified: hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java (original) +++ hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java Mon Oct 19 11:04:53 2009 @@ -38,9 +38,9 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.ClusterStatus; +import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobTracker; -import org.apache.hadoop.mapreduce.Cluster; -import org.apache.hadoop.mapreduce.ClusterMetrics; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; @@ -379,14 +379,14 @@ public static class MixMachine extends Machine { private static final MixMachine INSTANCE = new MixMachine(); - private Cluster cluster; + private JobClient jobclient; /** {@inheritDoc} */ @Override public synchronized void init(Job job) throws IOException { final Configuration conf = job.getConfiguration(); - if (cluster == null) - cluster = new Cluster(JobTracker.getAddress(conf), conf); + if (jobclient == null) + jobclient = new JobClient(JobTracker.getAddress(conf), conf); chooseMachine(conf).init(job); } @@ -398,11 +398,9 @@ try { for(;; Thread.sleep(2000)) { //get cluster status - final ClusterMetrics status = cluster.getClusterStatus(); - final int m = - status.getMapSlotCapacity() - status.getOccupiedMapSlots(); - final int r = - status.getReduceSlotCapacity() - status.getOccupiedReduceSlots(); + final ClusterStatus status = jobclient.getClusterStatus(); + final int m = status.getMaxMapTasks() - status.getMapTasks(); + final int r = status.getMaxReduceTasks() - status.getReduceTasks(); if (m >= parts || r >= parts) { //favor ReduceSide machine final Machine value = r >= parts? Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Mon Oct 19 11:04:53 2009 @@ -1703,14 +1703,6 @@ return trackersReservedForReduces.size(); } - public int getReservedMapSlots(TaskTracker taskTracker) { - return trackersReservedForMaps.get(taskTracker).getNumSlots(); - } - - public int getReservedReduceSlots(TaskTracker taskTracker) { - return trackersReservedForReduces.get(taskTracker).getNumSlots(); - } - private int getTrackerTaskFailures(String trackerName) { String trackerHostName = convertTrackerNameToHostName(trackerName); Integer failedTasks = trackerToFailuresMap.get(trackerHostName); Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Mon Oct 19 11:04:53 2009 @@ -20,6 +20,7 @@ import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.io.UnsupportedEncodingException; @@ -1214,10 +1215,6 @@ // int totalMaps = 0; int totalReduces = 0; - private int occupiedMapSlots = 0; - private int occupiedReduceSlots = 0; - private int reservedMapSlots = 0; - private int reservedReduceSlots = 0; private HashMap taskTrackers = new HashMap(); MapuniqueHostsMap = new ConcurrentHashMap(); @@ -2317,7 +2314,6 @@ HeartbeatResponse response = new HeartbeatResponse(newResponseId, null); List actions = new ArrayList(); isBlacklisted = faultyTrackers.isBlacklisted(status.getHost()); - removeTrackerReservations(getTaskTracker(trackerName)); // Check for new tasks to be executed on the tasktracker if (acceptNewTasks && !isBlacklisted) { TaskTrackerStatus taskTrackerStatus = getTaskTrackerStatus(trackerName) ; @@ -2337,7 +2333,6 @@ } } } - addTrackerReservations(getTaskTracker(trackerName)); // Check for tasks to be killed List killTasksList = getTasksToKill(trackerName); @@ -2428,8 +2423,6 @@ if (oldStatus != null) { totalMaps -= oldStatus.countMapTasks(); totalReduces -= oldStatus.countReduceTasks(); - occupiedMapSlots -= oldStatus.countOccupiedMapSlots(); - occupiedReduceSlots -= oldStatus.countOccupiedReduceSlots(); if (!faultyTrackers.isBlacklisted(oldStatus.getHost())) { int mapSlots = oldStatus.getMaxMapSlots(); totalMapTaskCapacity -= mapSlots; @@ -2452,8 +2445,6 @@ if (status != null) { totalMaps += status.countMapTasks(); totalReduces += status.countReduceTasks(); - occupiedMapSlots += status.countOccupiedMapSlots(); - occupiedReduceSlots += status.countOccupiedReduceSlots(); if (!faultyTrackers.isBlacklisted(status.getHost())) { int mapSlots = status.getMaxMapSlots(); totalMapTaskCapacity += mapSlots; @@ -2522,18 +2513,6 @@ } - // remove the tracker reservations from statistics - private void removeTrackerReservations(TaskTracker tt) { - reservedMapSlots -= tt.getReservedMapSlots(); - reservedReduceSlots -= tt.getReservedReduceSlots(); - } - - // add the tracker reservations to statistics - private void addTrackerReservations(TaskTracker tt) { - reservedMapSlots += tt.getReservedMapSlots(); - reservedReduceSlots += tt.getReservedReduceSlots(); - } - private void updateNodeHealthStatus(TaskTrackerStatus trackerStatus) { TaskTrackerHealthStatus status = trackerStatus.getHealthStatus(); synchronized (faultyTrackers) { @@ -2580,10 +2559,9 @@ } } } - removeTrackerReservations(getTaskTracker(trackerName)); + updateTaskStatuses(trackerStatus); updateNodeHealthStatus(trackerStatus); - addTrackerReservations(getTaskTracker(trackerName)); return true; } @@ -2972,9 +2950,8 @@ } public synchronized ClusterMetrics getClusterMetrics() { - return new ClusterMetrics(occupiedMapSlots, occupiedReduceSlots, - reservedMapSlots, reservedReduceSlots, - totalMapTaskCapacity, totalReduceTaskCapacity, taskTrackers.size() - + return new ClusterMetrics(totalMaps, totalReduces, totalMapTaskCapacity, + totalReduceTaskCapacity, taskTrackers.size() - getBlacklistedTrackerCount(), getBlacklistedTrackerCount(), getExcludedNodes().size()) ; } @@ -3747,7 +3724,6 @@ // Cleanup taskTracker.cancelAllReservations(); - removeTrackerReservations(taskTracker); // Purge 'marked' tasks, needs to be done // here to prevent hanging references! Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Mon Oct 19 11:04:53 2009 @@ -506,7 +506,7 @@ } public ClusterMetrics getClusterMetrics() { - return new ClusterMetrics(map_tasks, reduce_tasks, 0, 0, 1, 1, 1, 0, 0); + return new ClusterMetrics(map_tasks, reduce_tasks, 1, 1, 1, 0, 0); } public State getJobTrackerState() throws IOException, InterruptedException { Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java Mon Oct 19 11:04:53 2009 @@ -35,10 +35,10 @@ * Number of blacklisted and decommissioned trackers. * *
  • - * Slot capacity of the cluster. + * Task capacity of the cluster. *
    • *
  • - * The number of currently occupied/reserved map & reduce slots. + * The number of currently running map & reduce tasks. *
    • *

    * @@ -48,30 +48,24 @@ * @see Cluster */ public class ClusterMetrics implements Writable { - private int occupiedMapSlots; - private int occupiedReduceSlots; - private int reservedMapSlots; - private int reservedReduceSlots; - private int totalMapSlots; - private int totalReduceSlots; - private int numTrackers; - private int numBlacklistedTrackers; - private int numDecommissionedTrackers; + int runningMaps; + int runningReduces; + int mapSlots; + int reduceSlots; + int numTrackers; + int numBlacklistedTrackers; + int numDecommissionedTrackers; public ClusterMetrics() { } - public ClusterMetrics(int occupiedMapSlots, int occupiedReduceSlots, - int reservedMapSlots, int reservedReduceSlots, - int mapSlots, int reduceSlots, - int numTrackers, int numBlacklistedTrackers, - int numDecommisionedNodes) { - this.occupiedMapSlots = occupiedMapSlots; - this.occupiedReduceSlots = occupiedReduceSlots; - this.reservedMapSlots = reservedMapSlots; - this.reservedReduceSlots = reservedReduceSlots; - this.totalMapSlots = mapSlots; - this.totalReduceSlots = reduceSlots; + public ClusterMetrics(int runningMaps, int runningReduces, int mapSlots, + int reduceSlots, int numTrackers, int numBlacklistedTrackers, + int numDecommisionedNodes) { + this.runningMaps = runningMaps; + this.runningReduces = runningReduces; + this.mapSlots = mapSlots; + this.reduceSlots = reduceSlots; this.numTrackers = numTrackers; this.numBlacklistedTrackers = numBlacklistedTrackers; this.numDecommissionedTrackers = numDecommisionedNodes; @@ -83,7 +77,7 @@ * @return occupied map slot count */ public int getOccupiedMapSlots() { - return occupiedMapSlots; + return runningMaps; } /** @@ -92,34 +86,16 @@ * @return occupied reduce slot count */ public int getOccupiedReduceSlots() { - return occupiedReduceSlots; - } - - /** - * Get number of reserved map slots in the cluster. - * - * @return reserved map slot count - */ - public int getReservedMapSlots() { - return reservedMapSlots; + return runningReduces; } /** - * Get the number of reserved reduce slots in the cluster. - * - * @return reserved reduce slot count - */ - public int getReservedReduceSlots() { - return reservedReduceSlots; - } - - /** * Get the total number of map slots in the cluster. * * @return map slot capacity */ public int getMapSlotCapacity() { - return totalMapSlots; + return mapSlots; } /** @@ -128,7 +104,7 @@ * @return reduce slot capacity */ public int getReduceSlotCapacity() { - return totalReduceSlots; + return reduceSlots; } /** @@ -160,12 +136,10 @@ @Override public void readFields(DataInput in) throws IOException { - occupiedMapSlots = in.readInt(); - occupiedReduceSlots = in.readInt(); - reservedMapSlots = in.readInt(); - reservedReduceSlots = in.readInt(); - totalMapSlots = in.readInt(); - totalReduceSlots = in.readInt(); + runningMaps = in.readInt(); + runningReduces = in.readInt(); + mapSlots = in.readInt(); + reduceSlots = in.readInt(); numTrackers = in.readInt(); numBlacklistedTrackers = in.readInt(); numDecommissionedTrackers = in.readInt(); @@ -173,12 +147,10 @@ @Override public void write(DataOutput out) throws IOException { - out.writeInt(occupiedMapSlots); - out.writeInt(occupiedReduceSlots); - out.writeInt(reservedMapSlots); - out.writeInt(reservedReduceSlots); - out.writeInt(totalMapSlots); - out.writeInt(totalReduceSlots); + out.writeInt(runningMaps); + out.writeInt(runningReduces); + out.writeInt(mapSlots); + out.writeInt(reduceSlots); out.writeInt(numTrackers); out.writeInt(numBlacklistedTrackers); out.writeInt(numDecommissionedTrackers); Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java Mon Oct 19 11:04:53 2009 @@ -85,9 +85,8 @@ * Version 27: Changed protocol to use new api objects. And the protocol is * renamed from JobSubmissionProtocol to ClientProtocol. * Version 28: Added getJobHistoryDir() as part of MAPREDUCE-975. - * Version 29: Added reservedSlots to ClusterMetrics. */ - public static final long versionID = 29L; + public static final long versionID = 28L; /** * Allocate a name for the job. Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java Mon Oct 19 11:04:53 2009 @@ -100,30 +100,6 @@ } /** - * Get the reserved map slots for the tracker. - * - * @return reserved map slots - */ - public int getReservedMapSlots() { - if (jobForFallowMapSlot != null) { - return jobForFallowMapSlot.getReservedMapSlots(this); - } - return 0; - } - - /** - * Get the reserved reduce slots for the tracker. - * - * @return reserved reduce slots - */ - public int getReservedReduceSlots() { - if (jobForFallowReduceSlot != null) { - return jobForFallowReduceSlot.getReservedReduceSlots(this); - } - return 0; - } - - /** * Get the {@link JobInProgress} for which the fallow slot(s) are held. * @param taskType {@link TaskType} of the task * @return the task for which the fallow slot(s) are held, Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java Mon Oct 19 11:04:53 2009 @@ -223,7 +223,7 @@ } static short sendHeartBeat(JobTracker jt, TaskTrackerStatus status, - boolean initialContact, boolean acceptNewTasks, + boolean initialContact, String tracker, short responseId) throws IOException { if (status == null) { @@ -231,13 +231,13 @@ JobInProgress.convertTrackerNameToHostName(tracker)); } - jt.heartbeat(status, false, initialContact, acceptNewTasks, responseId); + jt.heartbeat(status, false, initialContact, false, responseId); return ++responseId ; } static void establishFirstContact(JobTracker jt, String tracker) throws IOException { - sendHeartBeat(jt, null, true, false, tracker, (short) 0); + sendHeartBeat(jt, null, true, tracker, (short) 0); } static class FakeTaskInProgress extends TaskInProgress { Modified: hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp?rev=826635&r1=826634&r2=826635&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp (original) +++ hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp Mon Oct 19 11:04:53 2009 @@ -25,7 +25,6 @@ import="java.util.*" import="java.text.DecimalFormat" import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapreduce.*" import="org.apache.hadoop.util.*" %> <%! private static final long serialVersionUID = 1L; @@ -45,29 +44,22 @@ public void generateSummaryTable(JspWriter out, ClusterStatus status, JobTracker tracker) throws IOException { - ClusterMetrics metrics = tracker.getClusterMetrics(); String tasksPerNode = status.getTaskTrackers() > 0 ? percentFormat.format(((double)(status.getMaxMapTasks() + status.getMaxReduceTasks())) / status.getTaskTrackers()): "-"; out.print("\n"+ "" + - "" + - "" + - "" + + "" + "" + - "" + - "" + + "" + + "" + "" + "\n"); out.print("
    QueuesRunning Map TasksRunning Reduce TasksOccupied Map SlotsOccupied Reduce SlotsReserved Map SlotsReserved Reduce SlotsMapsReducesTotal SubmissionsNodesMap Slot CapacityReduce Slot CapacityAvg. Slots/NodeNodesMap Task CapacityReduce Task CapacityAvg. Tasks/NodeBlacklisted NodesExcluded Nodes
    " + tracker.getRootQueues().length + "" + status.getMapTasks() + "" + status.getReduceTasks() + "" + - metrics.getOccupiedMapSlots() + "" + - metrics.getOccupiedReduceSlots() + "" + - metrics.getReservedMapSlots() + "" + - metrics.getReservedReduceSlots() + "" + tracker.getTotalSubmissions() + "" + status.getTaskTrackers() +