Return-Path: X-Original-To: apmail-myriad-commits-archive@minotaur.apache.org Delivered-To: apmail-myriad-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B1FC41848D for ; Sat, 17 Oct 2015 20:28:16 +0000 (UTC) Received: (qmail 42663 invoked by uid 500); 17 Oct 2015 20:28:16 -0000 Delivered-To: apmail-myriad-commits-archive@myriad.apache.org Received: (qmail 42634 invoked by uid 500); 17 Oct 2015 20:28:16 -0000 Mailing-List: contact commits-help@myriad.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@myriad.incubator.apache.org Delivered-To: mailing list commits@myriad.incubator.apache.org Received: (qmail 42625 invoked by uid 99); 17 Oct 2015 20:28:16 -0000 Received: from Unknown (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 17 Oct 2015 20:28:16 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id E78EF1A2735 for ; Sat, 17 Oct 2015 20:28:15 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.791 X-Spam-Level: * X-Spam-Status: No, score=1.791 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, T_RP_MATCHES_RCVD=-0.01, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-us-east.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id GFxO5_ojxaUB for ; Sat, 17 Oct 2015 20:28:06 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-east.apache.org (ASF Mail Server at mx1-us-east.apache.org) with SMTP id 19BAF439E9 for ; Sat, 17 Oct 2015 20:28:06 +0000 (UTC) Received: (qmail 42585 invoked by uid 99); 17 Oct 2015 20:28:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 17 Oct 2015 20:28:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 66E3ADFBD5; Sat, 17 Oct 2015 20:28:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: smarella@apache.org To: commits@myriad.incubator.apache.org Message-Id: <0dd5ebd32c71472382c48c2e14564a2a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: incubator-myriad git commit: MYRIAD-133 Multiple flexed up NMs try to run on same node, altogether Date: Sat, 17 Oct 2015 20:28:05 +0000 (UTC) Repository: incubator-myriad Updated Branches: refs/heads/master 645d17d8a -> 421edd230 MYRIAD-133 Multiple flexed up NMs try to run on same node, altogether * We now check both tasks in active and staging states before launching a new NM * Unit tested by launching multiple NMs * This fix has been tested internally by MapR QA This closes: #9 Review: https://github.com/apache/incubator-myriad/pull/9 Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/421edd23 Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/421edd23 Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/421edd23 Branch: refs/heads/master Commit: 421edd230a46938331d94ffa60de1c0ec4e573ec Parents: 645d17d Author: Swapnil Daingade Authored: Sat Oct 17 13:27:12 2015 -0700 Committer: Santosh Marella Committed: Sat Oct 17 13:27:12 2015 -0700 ---------------------------------------------------------------------- .../ebay/myriad/scheduler/SchedulerUtils.java | 19 +++---- .../handlers/ResourceOffersEventHandler.java | 8 ++- .../com/ebay/myriad/state/SchedulerState.java | 60 +++++++++++--------- .../recovery/MyriadFileSystemRMStateStore.java | 2 +- 4 files changed, 47 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/421edd23/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java index 392b69b..3c868e6 100644 --- a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java +++ b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/SchedulerUtils.java @@ -41,19 +41,16 @@ public class SchedulerUtils { Preconditions.checkArgument(offer != null); String offerHostname = offer.getHostname(); - if (CollectionUtils.isEmpty(tasks)) { - return true; - } - boolean uniqueHostname = true; - for (NodeTask task : tasks) { - if (offerHostname.equalsIgnoreCase(task.getHostname()) && - task.getTaskPrefix().equalsIgnoreCase(taskToLaunch.getTaskPrefix())) { - uniqueHostname = false; - break; + if (!CollectionUtils.isEmpty(tasks)) { + for (NodeTask task : tasks) { + if (offerHostname.equalsIgnoreCase(task.getHostname())) { + LOGGER.debug("Offer's hostname {} is not unique", offerHostname); + return false; + } } } - LOGGER.debug("Offer's hostname {} is unique: {}", offerHostname, uniqueHostname); - return uniqueHostname; + LOGGER.debug("Offer's hostname {} is unique", offerHostname); + return true; } /** http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/421edd23/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java index 4b0262a..de5211d 100644 --- a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java +++ b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java @@ -49,6 +49,7 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -113,9 +114,12 @@ public class ResourceOffersEventHandler implements EventHandler launchedTasks = new HashSet<>(); + launchedTasks.addAll(schedulerState.getActiveTasksByType(taskPrefix)); + launchedTasks.addAll(schedulerState.getStagingTasksByType(taskPrefix)); + if (matches(offer, taskToLaunch, constraint) - && SchedulerUtils.isUniqueHostname(offer, taskToLaunch, - schedulerState.getActiveTasks())) { + && SchedulerUtils.isUniqueHostname(offer, taskToLaunch, launchedTasks)) { try { final TaskInfo task = taskFactoryMap.get(taskPrefix).createTask(offer, schedulerState.getFrameworkID(), pendingTaskId, http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/421edd23/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java b/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java index c3a3b7d..f0ea778 100644 --- a/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java +++ b/myriad-scheduler/src/main/java/com/ebay/myriad/state/SchedulerState.java @@ -209,7 +209,7 @@ public class SchedulerState { SchedulerStateForType stateTask = statesForTaskType.get(taskPrefix); return (stateTask == null ? new HashSet() : stateTask.getPendingTaskIds()); } - + public synchronized Set getActiveTaskIds() { Set returnSet = new HashSet<>(); for (Map.Entry entry : statesForTaskType.entrySet()) { @@ -217,24 +217,43 @@ public class SchedulerState { } return returnSet; } - + public synchronized Set getActiveTaskIds(String taskPrefix) { SchedulerStateForType stateTask = statesForTaskType.get(taskPrefix); return (stateTask == null ? new HashSet() : stateTask.getActiveTaskIds()); } - public synchronized Collection getActiveTasks() { - List activeNodeTasks = new ArrayList<>(); - Set activeTaskIds = getActiveTaskIds(); - if (activeTaskIds.isEmpty()) { - return activeNodeTasks; - } - for (Map.Entry entry : tasks.entrySet()) { - if (activeTaskIds.contains(entry.getKey())) { - activeNodeTasks.add(entry.getValue()); + public synchronized Set getActiveTasks() { + return getTasks(getActiveTaskIds()); + } + + public Set getActiveTasksByType(String taskPrefix) { + return getTasks(getActiveTaskIds(taskPrefix)); + } + + public Set getStagingTasks() { + return getTasks(getStagingTaskIds()); + } + + public Set getStagingTasksByType(String taskPrefix) { + return getTasks(getStagingTaskIds(taskPrefix)); + } + + public Set getPendingTasksByType(String taskPrefix) { + return getTasks(getPendingTaskIds(taskPrefix)); + } + + public synchronized Set getTasks(Set taskIds) { + Set nodeTasks = new HashSet<>(); + if (CollectionUtils.isNotEmpty(taskIds) + && CollectionUtils.isNotEmpty(tasks.values())) { + for (Map.Entry entry : tasks.entrySet()) { + if (taskIds.contains(entry.getKey())) { + nodeTasks.add(entry.getValue()); + } + } } - } - return activeNodeTasks; + return Collections.unmodifiableSet(nodeTasks); } public synchronized Collection getActiveTaskIDsForProfile(ServiceResourceProfile serviceProfile) { @@ -252,21 +271,6 @@ public class SchedulerState { return Collections.unmodifiableCollection(activeTaskIDs); } - public Collection getActiveTasksByType(String taskPrefix) { - List activeNodeTasks = new ArrayList<>(); - Set activeTaskIds = getActiveTaskIds(taskPrefix); - if (activeTaskIds.isEmpty()) { - return activeNodeTasks; - } - - for (Map.Entry entry : tasks.entrySet()) { - if (activeTaskIds.contains(entry.getKey())) { - activeNodeTasks.add(entry.getValue()); - } - } - return activeNodeTasks; - } - // TODO (sdaingade) Clone NodeTask public synchronized NodeTask getNodeTask(SlaveID slaveId, String taskPrefix) { if (taskPrefix == null) { http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/421edd23/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java index cf9aaad..7020a67 100644 --- a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java +++ b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java @@ -76,7 +76,7 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore in.close(); } catch (IOException e) { LOGGER.error("State information for Myriad could not be loaded from: " - + myriadStatePath, e); + + myriadStatePath); } return rmState; }