hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tjungb...@apache.org
Subject svn commit: r1372505 - in /hama/trunk: ./ core/src/main/java/org/apache/hama/bsp/ core/src/main/java/org/apache/hama/bsp/sync/ core/src/main/java/org/apache/hama/bsp/taskallocation/
Date Mon, 13 Aug 2012 16:47:59 GMT
Author: tjungblut
Date: Mon Aug 13 16:47:59 2012
New Revision: 1372505

URL: http://svn.apache.org/viewvc?rev=1372505&view=rev
Log:
[HAMA-621]: Input Splits are not initialized for a job

Modified:
    hama/trunk/CHANGES.txt
    hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java
    hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java
    hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java
    hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java
    hama/trunk/pom.xml

Modified: hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hama/trunk/CHANGES.txt?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/CHANGES.txt (original)
+++ hama/trunk/CHANGES.txt Mon Aug 13 16:47:59 2012
@@ -30,11 +30,12 @@ Release 0.5 - April 10, 2012 
 
   BUG FIXES
 
+   HAMA-621: Input Splits are not initialized for a job (surajmenon via tjungblut)
    HAMA-595: Fix NullPointerException in Task Scheduler (surajmenon)
 
   IMPROVEMENTS
 
-    HAMA-584: Change Pagerank IO format to human-readable text for easy debug (tjungblut
vai edwardyoon)
+    HAMA-584: Change Pagerank IO format to human-readable text for easy debug (tjungblut
via edwardyoon)
     HAMA-590: Fix TestSubmitGraphJob tests (tjungblut)
     HAMA-582: Task's error logs should be displayed on client-end when job is failed (edwardyoon)
     HAMA-580: Improve input of graph module (tjungblut)

Modified: hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java
URL: http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java (original)
+++ hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java Mon Aug 13 16:47:59
2012
@@ -348,8 +348,12 @@ public class JobInProgress {
         GroomServerStatus groomStatus = taskAllocationStrategy
             .getGroomToAllocate(groomStatuses, selectedGrooms,
                 taskCountInGroomMap, resources, tasks[i]);
-        if (groomStatus != null)
+        if (groomStatus != null){
           result = tasks[i].constructTask(groomStatus);
+        }
+        else if (LOG.isDebugEnabled()){
+        	LOG.debug("Could not find a groom to schedule task");
+        }
         if (result != null) {
           updateGroomTaskDetails(tasks[i].getGroomServerStatus(), result);
         }

Modified: hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java
URL: http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java (original)
+++ hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java Mon Aug 13 16:47:59
2012
@@ -199,6 +199,10 @@ public class TaskInProgress {
     } else {
       String splitClass = null;
       BytesWritable split = null;
+      if (rawSplit != null) {
+    	  splitClass = rawSplit.getClassName();
+    	  split = rawSplit.getBytes();
+      }
       currentTaskId = taskId;
       String groomName = groomStatus.getGroomHostName();
       Task t = new BSPTask(jobId, jobFile, taskId, partition, splitClass, split);
@@ -209,44 +213,16 @@ public class TaskInProgress {
 
   }
 
-  // /* Remove */
-  // private Task getGroomForTask(TaskAttemptID taskid,
-  // Map<String, GroomServerStatus> grooms,
-  // Map<GroomServerStatus, Integer> tasksInGroomMap) {
-  // String splitClass = null;
-  // BytesWritable split = null;
-  // Task t = null;
-  // if (rawSplit != null) {
-  // splitClass = rawSplit.getClassName();
-  // split = rawSplit.getBytes();
-  // String[] possibleLocations = rawSplit.getLocations();
-  // String groomName = getGroomToSchedule(taskid, grooms, tasksInGroomMap,
-  // possibleLocations);
-  // if (groomName != null) {
-  // t = new BSPTask(jobId, jobFile, taskid, partition, splitClass, split);
-  // activeTasks.put(taskid, groomName);
-  // myGroomStatus = grooms.get(groomName);
-  // }
-  // }
-  //
-  // if (t == null) {
-  // String groomName = getAnyGroomToSchedule(grooms, tasksInGroomMap);
-  // if (groomName != null) {
-  // t = new BSPTask(jobId, jobFile, taskid, partition, splitClass, split);
-  // activeTasks.put(taskid, groomName);
-  // myGroomStatus = grooms.get(groomName);
-  // }
-  // }
-  //
-  // return t;
-  // }
-
   private Task getGroomForRecoverTaskInHosts(TaskAttemptID taskid,
       Map<String, GroomServerStatus> grooms,
       Map<GroomServerStatus, Integer> tasksInGroomMap,
       String[] possibleLocations) {
     String splitClass = null;
     BytesWritable split = null;
+    if (rawSplit != null) {
+  	  splitClass = rawSplit.getClassName();
+  	  split = rawSplit.getBytes();
+    }
     Task t = null;
     String groomName = getGroomToSchedule(taskid, grooms, tasksInGroomMap,
         possibleLocations);

Modified: hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java
URL: http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java (original)
+++ hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java Mon Aug 13 16:47:59
2012
@@ -464,7 +464,7 @@ public abstract class ZKSyncClient imple
     } else {
       for (String node : list) {
         clearZKNodes(path + "/" + node);
-        LOG.info("Deleting " + path + "/" + node);
+        LOG.debug("Deleting " + path + "/" + node);
         zk.delete(path + "/" + node, -1); // delete any version of this
         // node.
       }

Modified: hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java
URL: http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java
(original)
+++ hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java
Mon Aug 13 16:47:59 2012
@@ -83,15 +83,26 @@ public class BestEffortDataLocalTaskAllo
     for (int i = 0; i < possibleLocations.length; ++i) {
       String location = possibleLocations[i];
       GroomServerStatus groom = grooms.get(location);
-      if (groom == null)
+      if (groom == null){
+        if(LOG.isDebugEnabled()){
+          LOG.debug("Could not find groom for location " + location);
+        }
         continue;
+      }
       Integer taskInGroom = tasksInGroomMap.get(groom);
       taskInGroom = (taskInGroom == null) ? 0 : taskInGroom;
+      if(LOG.isDebugEnabled()){
+        LOG.debug("taskInGroom = " + taskInGroom + " max tasks = " + groom.getMaxTasks()
+            + " location = " + location + " groomhostname = " + groom.getGroomHostName());
+      }
       if (taskInGroom < groom.getMaxTasks()
           && location.equals(groom.getGroomHostName())) {
         return groom.getGroomHostName();
       }
     }
+    if(LOG.isDebugEnabled()){
+      LOG.debug("Returning null");
+    }
     return null;
   }
 
@@ -100,8 +111,12 @@ public class BestEffortDataLocalTaskAllo
       Map<String, GroomServerStatus> groomStatuses, String[] selectedGrooms,
       Map<GroomServerStatus, Integer> taskCountInGroomMap,
       BSPResource[] resources, TaskInProgress taskInProgress) {
-    if (!taskInProgress.canStartTask())
+    if (!taskInProgress.canStartTask()) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Exceeded allowed attempts.");
+      }
       return null;
+    }
 
     String groomName = null;
     if (selectedGrooms != null) {

Modified: hama/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/pom.xml?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/pom.xml (original)
+++ hama/trunk/pom.xml Mon Aug 13 16:47:59 2012
@@ -269,7 +269,7 @@
         <version>2.6</version>
         <configuration>
           <parallel>methods</parallel>
-          <threadCount>10</threadCount>
+          <threadCount>1</threadCount>
           <forkMode>pertest</forkMode>
         </configuration>
       </plugin>



Mime
View raw message