airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jlo...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-842] do not query the DB with an empty IN clause
Date Mon, 13 Feb 2017 19:32:33 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master 2ce7556a5 -> 485280a9f


[AIRFLOW-842] do not query the DB with an empty IN clause

This is done to silence warnings coming from
sqlachemy, e.g.:

sqlalchemy/sql/default_comparator.py:161:
SAWarning: The IN-predicate on
"dag_run.dag_id" was invoked with an empty
sequence. This results in a
contradiction, which nonetheless can be expensive
to evaluate. Consider
alternative strategies for improved performance.

Closes #2072 from imbaczek/bug842


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/485280a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/485280a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/485280a9

Branch: refs/heads/master
Commit: 485280a9f180bd6f85ec031bd7fa100a2ba89a7b
Parents: 2ce7556
Author: Marek Baczynski <marek.baczynski@intel.com>
Authored: Mon Feb 13 14:32:13 2017 -0500
Committer: Jeremiah Lowin <jlowin@apache.org>
Committed: Mon Feb 13 14:32:13 2017 -0500

----------------------------------------------------------------------
 airflow/models.py |  8 ++++++++
 tests/core.py     | 16 +++++++++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/485280a9/airflow/models.py
----------------------------------------------------------------------
diff --git a/airflow/models.py b/airflow/models.py
index 6cf7ad9..503f146 100755
--- a/airflow/models.py
+++ b/airflow/models.py
@@ -3734,6 +3734,10 @@ class DagStat(Base):
         :param full_query: whether to check dag_runs for new drs not in dag_stats
         :type full_query: bool
         """
+        # avoid querying with an empty IN clause
+        if not dag_ids:
+            return
+
         dag_ids = set(dag_ids)
 
         qry = (
@@ -3745,6 +3749,10 @@ class DagStat(Base):
         qry.delete(synchronize_session='fetch')
         session.commit()
 
+        # avoid querying with an empty IN clause
+        if not dirty_ids:
+            return
+
         qry = (
             session.query(DagRun.dag_id, DagRun.state, func.count('*'))
             .filter(DagRun.dag_id.in_(dirty_ids))

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/485280a9/tests/core.py
----------------------------------------------------------------------
diff --git a/tests/core.py b/tests/core.py
index fba05f7..fcf69ad 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -978,12 +978,18 @@ class CoreTest(unittest.TestCase):
         session.query(models.DagStat).delete()
         session.commit()
 
+        with warnings.catch_warnings(record=True) as caught_warnings:
+            models.DagStat.clean_dirty([], session=session)
+        self.assertEqual([], caught_warnings)
+
         run1 = self.dag_bash.create_dagrun(
             run_id="run1",
             execution_date=DEFAULT_DATE,
             state=State.RUNNING)
 
-        models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+        with warnings.catch_warnings(record=True) as caught_warnings:
+            models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+        self.assertEqual([], caught_warnings)
 
         qry = session.query(models.DagStat).all()
 
@@ -998,7 +1004,9 @@ class CoreTest(unittest.TestCase):
             execution_date=DEFAULT_DATE+timedelta(days=1),
             state=State.RUNNING)
 
-        models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+        with warnings.catch_warnings(record=True) as caught_warnings:
+            models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+        self.assertEqual([], caught_warnings)
 
         qry = session.query(models.DagStat).all()
 
@@ -1011,7 +1019,9 @@ class CoreTest(unittest.TestCase):
         session.query(models.DagRun).first().state = State.SUCCESS
         session.commit()
 
-        models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+        with warnings.catch_warnings(record=True) as caught_warnings:
+            models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+        self.assertEqual([], caught_warnings)
 
         qry = session.query(models.DagStat).filter(models.DagStat.state == State.SUCCESS).all()
         self.assertEqual(1, len(qry))


Mime
View raw message