airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From saguz...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-1601] Add configurable task cleanup time
Date Wed, 13 Sep 2017 21:22:52 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master a8d486de3 -> 17189ddcc


[AIRFLOW-1601] Add configurable task cleanup time

When task processes are SIGTERMed, they have by
default 5 seconds to
cleanup before a SIGKILL arrives. This allows this
value to be
configurable.

Closes #2601 from saguziel/aguziel-configure-task-
killer


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/17189ddc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/17189ddc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/17189ddc

Branch: refs/heads/master
Commit: 17189ddcc8f7b8efaca6384ea854f5573e11ea4e
Parents: a8d486d
Author: Alex Guziel <alex.guziel@airbnb.com>
Authored: Wed Sep 13 14:22:48 2017 -0700
Committer: Alex Guziel <alex.guziel@airbnb.com>
Committed: Wed Sep 13 14:22:48 2017 -0700

----------------------------------------------------------------------
 airflow/config_templates/default_airflow.cfg | 4 ++++
 airflow/config_templates/default_test.cfg    | 1 +
 airflow/utils/helpers.py                     | 3 ++-
 3 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/17189ddc/airflow/config_templates/default_airflow.cfg
----------------------------------------------------------------------
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index 948c72c..94efe60 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -133,6 +133,10 @@ task_log_reader = file.task
 # RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False).
 enable_xcom_pickling = True
 
+# When a task is killed forcefully, this is the amount of time in seconds that
+# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED
+killed_task_cleanup_time = 60
+
 [cli]
 # In what way should the cli access the API. The LocalClient will use the
 # database directly, while the json_client will use the api running on the

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/17189ddc/airflow/config_templates/default_test.cfg
----------------------------------------------------------------------
diff --git a/airflow/config_templates/default_test.cfg b/airflow/config_templates/default_test.cfg
index 2a090d4..b065313 100644
--- a/airflow/config_templates/default_test.cfg
+++ b/airflow/config_templates/default_test.cfg
@@ -38,6 +38,7 @@ dags_are_paused_at_creation = False
 fernet_key = {FERNET_KEY}
 non_pooled_task_slot_count = 128
 enable_xcom_pickling = False
+killed_task_cleanup_time = 5
 
 [cli]
 api_client = airflow.api.client.local_client

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/17189ddc/airflow/utils/helpers.py
----------------------------------------------------------------------
diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py
index 9a94125..4b8944e 100644
--- a/airflow/utils/helpers.py
+++ b/airflow/utils/helpers.py
@@ -31,11 +31,12 @@ import subprocess
 import sys
 import warnings
 
+from airflow import configuration
 from airflow.exceptions import AirflowException
 
 # When killing processes, time to wait after issuing a SIGTERM before issuing a
 # SIGKILL.
-DEFAULT_TIME_TO_WAIT_AFTER_SIGTERM = 5
+DEFAULT_TIME_TO_WAIT_AFTER_SIGTERM = configuration.getint('core', 'KILLED_TASK_CLEANUP_TIME')
 
 
 def validate_key(k, max_length=250):


Mime
View raw message