Return-Path: X-Original-To: apmail-aurora-commits-archive@minotaur.apache.org Delivered-To: apmail-aurora-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 14BE817470 for ; Sat, 2 May 2015 17:26:34 +0000 (UTC) Received: (qmail 79807 invoked by uid 500); 2 May 2015 17:26:34 -0000 Delivered-To: apmail-aurora-commits-archive@aurora.apache.org Received: (qmail 79771 invoked by uid 500); 2 May 2015 17:26:33 -0000 Mailing-List: contact commits-help@aurora.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@aurora.apache.org Delivered-To: mailing list commits@aurora.apache.org Received: (qmail 79762 invoked by uid 99); 2 May 2015 17:26:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 02 May 2015 17:26:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CB288E056A; Sat, 2 May 2015 17:26:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wfarner@apache.org To: commits@aurora.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: aurora git commit: Enable GC executor to gc STARTING tasks which don't exist on the host Date: Sat, 2 May 2015 17:26:33 +0000 (UTC) Repository: aurora Updated Branches: refs/heads/master 5fc7baf5a -> 35c511ee6 Enable GC executor to gc STARTING tasks which don't exist on the host Reviewed at https://reviews.apache.org/r/33739/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/35c511ee Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/35c511ee Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/35c511ee Branch: refs/heads/master Commit: 35c511ee69899557c309338e266b94bcd9b4e303 Parents: 5fc7baf Author: Zeke Harris Authored: Sat May 2 10:26:24 2015 -0700 Committer: Bill Farner Committed: Sat May 2 10:26:24 2015 -0700 ---------------------------------------------------------------------- .../apache/aurora/executor/gc_executor.py | 29 ++++++++++---------- .../apache/aurora/executor/test_gc_executor.py | 10 +++---- 2 files changed, 19 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/35c511ee/src/main/python/apache/aurora/executor/gc_executor.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/gc_executor.py b/src/main/python/apache/aurora/executor/gc_executor.py index a7776b5..d4392fa 100644 --- a/src/main/python/apache/aurora/executor/gc_executor.py +++ b/src/main/python/apache/aurora/executor/gc_executor.py @@ -229,15 +229,15 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable): Local vs Scheduler => Action =================================== ACTIVE ACTIVE => no-op - ACTIVE STARTING => no-op + ACTIVE ASSIGNED => no-op ACTIVE TERMINAL => maybe kill task* ACTIVE !EXISTS => maybe kill task* TERMINAL ACTIVE => send actual status** - TERMINAL STARTING => send actual status** + TERMINAL ASSIGNED => send actual status** TERMINAL TERMINAL => no-op TERMINAL !EXISTS => gc locally !EXISTS ACTIVE => send LOST** - !EXISTS STARTING => no-op + !EXISTS ASSIGNED => no-op !EXISTS TERMINAL => gc remotely * - Only kill if this does not appear to be a race condition. @@ -254,21 +254,20 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable): updates - dictionary of updates sent to the scheduler (task_id: ScheduleStatus) """ def partition(rt): - active, starting, finished = set(), set(), set() + active, assigned, finished = set(), set(), set() for task, schedule_status in rt.items(): if schedule_status in TERMINAL_STATES: finished.add(task) - elif (schedule_status == ScheduleStatus.STARTING or - schedule_status == ScheduleStatus.ASSIGNED): - starting.add(task) + elif schedule_status == ScheduleStatus.ASSIGNED: + assigned.add(task) else: active.add(task) - return active, starting, finished + return active, assigned, finished local_active, local_finished = self.partition_tasks() - sched_active, sched_starting, sched_finished = partition(retained_tasks) + sched_active, sched_assigned, sched_finished = partition(retained_tasks) local_tasks = local_active | local_finished - sched_tasks = sched_active | sched_starting | sched_finished + sched_tasks = sched_active | sched_assigned | sched_finished self.log('Told to retain the following task ids:') for task_id, schedule_status in retained_tasks.items(): @@ -287,7 +286,7 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable): updates = {} for task in local_active: - if task.task_id not in (sched_active | sched_starting): + if task.task_id not in (sched_active | sched_assigned): self.log('Inspecting task %s for termination.' % task.task_id) if not self.maybe_terminate_unknown_task(task): local_gc.update(self.should_gc_task(task)) @@ -296,8 +295,8 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable): if task.task_id not in sched_tasks: self.log('Queueing task %s for local deletion.' % task.task_id) local_gc.add(task) - if task.task_id in (sched_active | sched_starting): - self.log('Task %s finished but scheduler thinks active/starting.' % task.task_id) + if task.task_id in (sched_active | sched_assigned): + self.log('Task %s finished but scheduler thinks active/assigned.' % task.task_id) states = self.get_states(task) if states: _, last_state = states[-1] @@ -324,9 +323,9 @@ class ThermosGCExecutor(ExecutorBase, ExceptionalThread, Observable): self.send_update( driver, task_id, mesos_pb2.TASK_LOST, 'GC executor found no trace of task.') - for task_id in sched_starting: + for task_id in sched_assigned: if task_id not in local_task_ids: - self.log('Know nothing about task %s, but scheduler says STARTING - passing' % task_id) + self.log('Know nothing about task %s, but scheduler says ASSIGNED - passing' % task_id) return local_gc, remote_gc_ids, updates http://git-wip-us.apache.org/repos/asf/aurora/blob/35c511ee/src/test/python/apache/aurora/executor/test_gc_executor.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/executor/test_gc_executor.py b/src/test/python/apache/aurora/executor/test_gc_executor.py index 856483d..17d3590 100644 --- a/src/test/python/apache/aurora/executor/test_gc_executor.py +++ b/src/test/python/apache/aurora/executor/test_gc_executor.py @@ -66,7 +66,7 @@ FINISHED_TASKS = { # TODO(wickman) These should be constant sets in the Thermos thrift THERMOS_LIVES = (TaskState.ACTIVE, TaskState.CLEANING, TaskState.FINALIZING) THERMOS_TERMINALS = (TaskState.SUCCESS, TaskState.FAILED, TaskState.KILLED, TaskState.LOST) -STARTING_STATES = (ScheduleStatus.STARTING, ScheduleStatus.ASSIGNED) +ASSIGNED_STATES = (ScheduleStatus.ASSIGNED, ) TASK_ID = 'gc_executor_task_id' EVENT_WAIT_TIMEOUT_SECS = 10 @@ -232,15 +232,15 @@ def test_state_reconciliation_no_ops(): assert tgc.len_results == (0, 0, 0, 0) assert llen(lgc, rgc, updates) == (0, 0, 0) - # active vs. starting - for st0, st1 in product(THERMOS_LIVES, STARTING_STATES): + # active vs. assigned + for st0, st1 in product(THERMOS_LIVES, ASSIGNED_STATES): tgc, driver = make_pair({make_task('foo'): st0}, {}) lgc, rgc, updates = tgc.reconcile_states(driver, {'foo': st1}) assert tgc.len_results == (0, 0, 0, 0) assert llen(lgc, rgc, updates) == (0, 0, 0) - # nexist vs. starting - for st1 in STARTING_STATES: + # nexist vs. assigned + for st1 in ASSIGNED_STATES: tgc, driver = make_pair({}, {}) lgc, rgc, updates = tgc.reconcile_states(driver, {'foo': st1}) assert tgc.len_results == (0, 0, 0, 0)