myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smare...@apache.org
Subject incubator-myriad git commit: MYRIAD-133 Multiple flexed up NMs try to run on same node, altogether
Date Sat, 17 Oct 2015 20:28:05 GMT
Repository: incubator-myriad
Updated Branches:
  refs/heads/master 645d17d8a -> 421edd230


MYRIAD-133 Multiple flexed up NMs try to run on same node, altogether

* We now check both tasks in active and staging states before launching a new NM

* Unit tested by launching multiple NMs
* This fix has been tested internally by MapR QA

This closes: #9
Review: https://github.com/apache/incubator-myriad/pull/9


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/421edd23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/421edd23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/421edd23

Branch: refs/heads/master
Commit: 421edd230a46938331d94ffa60de1c0ec4e573ec
Parents: 645d17d
Author: Swapnil Daingade <sdaingade@maprtech.com>
Authored: Sat Oct 17 13:27:12 2015 -0700
Committer: Santosh Marella <marella@gmail.com>
Committed: Sat Oct 17 13:27:12 2015 -0700

----------------------------------------------------------------------
 .../ebay/myriad/scheduler/SchedulerUtils.java   | 19 +++----
 .../handlers/ResourceOffersEventHandler.java    |  8 ++-
 .../com/ebay/myriad/state/SchedulerState.java   | 60 +++++++++++---------
 .../recovery/MyriadFileSystemRMStateStore.java  |  2 +-
 4 files changed, 47 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/421edd23/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java
b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java
index 392b69b..3c868e6 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java
@@ -41,19 +41,16 @@ public class SchedulerUtils {
         Preconditions.checkArgument(offer != null);
         String offerHostname = offer.getHostname();
 
-        if (CollectionUtils.isEmpty(tasks)) {
-            return true;
-        }
-        boolean uniqueHostname = true;
-        for (NodeTask task : tasks) {
-            if (offerHostname.equalsIgnoreCase(task.getHostname()) &&
-                task.getTaskPrefix().equalsIgnoreCase(taskToLaunch.getTaskPrefix())) {
-                uniqueHostname = false;
-                break;
+        if (!CollectionUtils.isEmpty(tasks)) {
+            for (NodeTask task : tasks) {
+                if (offerHostname.equalsIgnoreCase(task.getHostname())) {
+                    LOGGER.debug("Offer's hostname {} is not unique", offerHostname);
+                    return false;
+                }
             }
         }
-        LOGGER.debug("Offer's hostname {} is unique: {}", offerHostname, uniqueHostname);
-        return uniqueHostname;
+        LOGGER.debug("Offer's hostname {} is unique", offerHostname);
+        return true;        
     }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/421edd23/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
index 4b0262a..de5211d 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
@@ -49,6 +49,7 @@ import javax.inject.Inject;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -113,9 +114,12 @@ public class ResourceOffersEventHandler implements EventHandler<ResourceOffersEv
             ServiceResourceProfile profile = taskToLaunch.getProfile();
             Constraint constraint = taskToLaunch.getConstraint();
 
+            Set<NodeTask> launchedTasks = new HashSet<>();
+            launchedTasks.addAll(schedulerState.getActiveTasksByType(taskPrefix));
+            launchedTasks.addAll(schedulerState.getStagingTasksByType(taskPrefix));
+
             if (matches(offer, taskToLaunch, constraint)
-                && SchedulerUtils.isUniqueHostname(offer, taskToLaunch,
-                schedulerState.getActiveTasks())) {
+                && SchedulerUtils.isUniqueHostname(offer, taskToLaunch, launchedTasks))
{
               try {
                 final TaskInfo task = 
                       taskFactoryMap.get(taskPrefix).createTask(offer, schedulerState.getFrameworkID(),
pendingTaskId,

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/421edd23/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java b/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java
index c3a3b7d..f0ea778 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java
@@ -209,7 +209,7 @@ public class SchedulerState {
       SchedulerStateForType stateTask = statesForTaskType.get(taskPrefix);
       return (stateTask == null ? new HashSet<Protos.TaskID>() : stateTask.getPendingTaskIds());
 
     }
-    
+
     public synchronized Set<Protos.TaskID> getActiveTaskIds() {
       Set<Protos.TaskID> returnSet = new HashSet<>();
       for (Map.Entry<String, SchedulerStateForType> entry : statesForTaskType.entrySet())
{
@@ -217,24 +217,43 @@ public class SchedulerState {
       }
       return returnSet;
     }
-    
+
     public synchronized Set<Protos.TaskID> getActiveTaskIds(String taskPrefix) {
       SchedulerStateForType stateTask = statesForTaskType.get(taskPrefix);
       return (stateTask == null ? new HashSet<Protos.TaskID>() : stateTask.getActiveTaskIds());
     }
 
-    public synchronized Collection<NodeTask> getActiveTasks() {
-      List<NodeTask> activeNodeTasks = new ArrayList<>();
-      Set<Protos.TaskID> activeTaskIds = getActiveTaskIds();
-      if (activeTaskIds.isEmpty()) {
-        return activeNodeTasks;
-      }
-      for (Map.Entry<Protos.TaskID, NodeTask> entry : tasks.entrySet()) {
-        if (activeTaskIds.contains(entry.getKey())) {
-          activeNodeTasks.add(entry.getValue());
+    public synchronized Set<NodeTask> getActiveTasks() {
+        return getTasks(getActiveTaskIds());
+    }
+
+    public Set<NodeTask> getActiveTasksByType(String taskPrefix) {
+        return getTasks(getActiveTaskIds(taskPrefix));
+    }
+
+    public Set<NodeTask> getStagingTasks() {
+        return getTasks(getStagingTaskIds());
+    }
+
+    public Set<NodeTask> getStagingTasksByType(String taskPrefix) {
+        return getTasks(getStagingTaskIds(taskPrefix));
+    }
+
+    public Set<NodeTask> getPendingTasksByType(String taskPrefix) {
+        return getTasks(getPendingTaskIds(taskPrefix));
+    }
+
+    public synchronized Set<NodeTask> getTasks(Set<Protos.TaskID> taskIds) {
+        Set<NodeTask> nodeTasks = new HashSet<>();
+        if (CollectionUtils.isNotEmpty(taskIds)
+            && CollectionUtils.isNotEmpty(tasks.values())) {
+            for (Map.Entry<Protos.TaskID, NodeTask> entry : tasks.entrySet()) {
+                if (taskIds.contains(entry.getKey())) {
+                    nodeTasks.add(entry.getValue());
+                }
+            }
         }
-      }
-      return activeNodeTasks;
+            return Collections.unmodifiableSet(nodeTasks);
     }
 
     public synchronized Collection<Protos.TaskID> getActiveTaskIDsForProfile(ServiceResourceProfile
serviceProfile) {
@@ -252,21 +271,6 @@ public class SchedulerState {
       return Collections.unmodifiableCollection(activeTaskIDs);
     }
 
-    public Collection<NodeTask> getActiveTasksByType(String taskPrefix) {
-      List<NodeTask> activeNodeTasks = new ArrayList<>();
-      Set<Protos.TaskID> activeTaskIds = getActiveTaskIds(taskPrefix);
-      if (activeTaskIds.isEmpty()) {
-        return activeNodeTasks;
-      }
-
-      for (Map.Entry<Protos.TaskID, NodeTask> entry : tasks.entrySet()) {
-        if (activeTaskIds.contains(entry.getKey())) {
-          activeNodeTasks.add(entry.getValue());
-        }
-      }
-      return activeNodeTasks;
-   }
-
    // TODO (sdaingade) Clone NodeTask
     public synchronized NodeTask getNodeTask(SlaveID slaveId, String taskPrefix) {
       if (taskPrefix == null) {

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/421edd23/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
index cf9aaad..7020a67 100644
--- a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
+++ b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
@@ -76,7 +76,7 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore
       in.close();
     } catch (IOException e) {
       LOGGER.error("State information for Myriad could not be loaded from: "
-        + myriadStatePath, e);
+        + myriadStatePath);
     }
     return rmState;
   }


Mime
View raw message