Mailing-List: contact commits-help@airflow.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@airflow.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: criccomini@apache.org
To: commits@airflow.incubator.apache.org
Date: Tue, 09 May 2017 17:35:46 -0000
Message-Id: <0f68da24ee174c35aab1572fdd2c046d@git.apache.org>
In-Reply-To: <ca18733594b14fc9abeeac1d97da99a2@git.apache.org>
References: <ca18733594b14fc9abeeac1d97da99a2@git.apache.org>
Subject: [17/36] incubator-airflow git commit: [AIRFLOW-1035] Use binary
 exponential backoff
archived-at: Tue, 09 May 2017 17:35:41 -0000

[AIRFLOW-1035] Use binary exponential backoff

Closes #2196 from IvanVergiliev/exponential-
backoff

(cherry picked from commit 4ec932b551774bb394c5770c4d2660f565a4c592)
Signed-off-by: Bolke de Bruin <bolke@xs4all.nl>


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/4199cd3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/4199cd3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/4199cd3d

Branch: refs/heads/v1-8-stable
Commit: 4199cd3d23d35183253c5d078e0f9937e87df232
Parents: ceb2ac3
Author: Ivan Vergiliev <ivan.vergiliev@gmail.com>
Authored: Fri Apr 7 19:35:03 2017 +0200
Committer: Bolke de Bruin <bolke@xs4all.nl>
Committed: Fri Apr 7 19:35:23 2017 +0200

----------------------------------------------------------------------
 airflow/models.py | 10 +++++++++-
 tests/models.py   | 17 ++++++++++-------
 2 files changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/4199cd3d/airflow/models.py
----------------------------------------------------------------------
diff --git a/airflow/models.py b/airflow/models.py
index 47413e0..5db0287 100755
--- a/airflow/models.py
+++ b/airflow/models.py
@@ -1161,7 +1161,15 @@ class TaskInstance(Base):
         """
         delay = self.task.retry_delay
         if self.task.retry_exponential_backoff:
-            delay_backoff_in_seconds = delay.total_seconds() ** self.try_number
+            # timedelta has a maximum representable value. The exponentiation
+            # here means this value can be exceeded after a certain number
+            # of tries (around 50 if the initial delay is 1s, even fewer if
+            # the delay is larger). Cap the value here before creating a
+            # timedelta object so the operation doesn't fail.
+            delay_backoff_in_seconds = min(
+                delay.total_seconds() * (2 ** (self.try_number - 1)),
+                timedelta.max.total_seconds() - 1
+            )
             delay = timedelta(seconds=delay_backoff_in_seconds)
             if self.task.max_retry_delay:
                 delay = min(self.task.max_retry_delay, delay)

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/4199cd3d/tests/models.py
----------------------------------------------------------------------
diff --git a/tests/models.py b/tests/models.py
index 9478088..8223276 100644
--- a/tests/models.py
+++ b/tests/models.py
@@ -757,9 +757,8 @@ class TaskInstanceTest(unittest.TestCase):
         self.assertEqual(ti.try_number, 4)
 
     def test_next_retry_datetime(self):
-        delay = datetime.timedelta(seconds=3)
-        delay_squared = datetime.timedelta(seconds=9)
-        max_delay = datetime.timedelta(seconds=10)
+        delay = datetime.timedelta(seconds=30)
+        max_delay = datetime.timedelta(minutes=60)
 
         dag = models.DAG(dag_id='fail_dag')
         task = BashOperator(
@@ -778,13 +777,17 @@ class TaskInstanceTest(unittest.TestCase):
 
         ti.try_number = 1
         dt = ti.next_retry_datetime()
-        self.assertEqual(dt, ti.end_date+delay)
+        self.assertEqual(dt, ti.end_date + delay)
 
-        ti.try_number = 2
+        ti.try_number = 6
         dt = ti.next_retry_datetime()
-        self.assertEqual(dt, ti.end_date+delay_squared)
+        self.assertEqual(dt, ti.end_date + (2 ** 5) * delay)
 
-        ti.try_number = 3
+        ti.try_number = 8
+        dt = ti.next_retry_datetime()
+        self.assertEqual(dt, ti.end_date+max_delay)
+
+        ti.try_number = 50
         dt = ti.next_retry_datetime()
         self.assertEqual(dt, ti.end_date+max_delay)