airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fo...@apache.org
Subject [2/6] incubator-airflow git commit: [AIRFLOW-2203] Cache static rules (trigger/weight)
Date Wed, 14 Mar 2018 08:15:41 GMT
[AIRFLOW-2203] Cache static rules (trigger/weight)

No need to recalculate them everytime just to see if they are valid


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/16ab314c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/16ab314c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/16ab314c

Branch: refs/heads/master
Commit: 16ab314c21d131136fb00adcc395d71a6f8b31b2
Parents: 781c5bf
Author: wongwill86 <wongwill86@gmail.com>
Authored: Mon Mar 12 17:32:24 2018 -0400
Committer: Fokko Driesprong <fokkodriesprong@godatadriven.com>
Committed: Wed Mar 14 09:11:38 2018 +0100

----------------------------------------------------------------------
 airflow/utils/trigger_rule.py    |  9 +++++++--
 airflow/utils/weight_rule.py     |  9 +++++++--
 tests/utils/test_trigger_rule.py | 28 ++++++++++++++++++++++++++++
 tests/utils/test_weight_rule.py  | 25 +++++++++++++++++++++++++
 4 files changed, 67 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/16ab314c/airflow/utils/trigger_rule.py
----------------------------------------------------------------------
diff --git a/airflow/utils/trigger_rule.py b/airflow/utils/trigger_rule.py
index f830909..5daaeac 100644
--- a/airflow/utils/trigger_rule.py
+++ b/airflow/utils/trigger_rule.py
@@ -25,12 +25,17 @@ class TriggerRule(object):
     ONE_FAILED = 'one_failed'
     DUMMY = 'dummy'
 
+    _ALL_TRIGGER_RULES = {}
     @classmethod
     def is_valid(cls, trigger_rule):
         return trigger_rule in cls.all_triggers()
 
     @classmethod
     def all_triggers(cls):
-        return [getattr(cls, attr)
+        if not cls._ALL_TRIGGER_RULES:
+            cls._ALL_TRIGGER_RULES = {
+                getattr(cls, attr)
                 for attr in dir(cls)
-                if not attr.startswith("__") and not callable(getattr(cls, attr))]
+                if not attr.startswith("_") and not callable(getattr(cls, attr))
+            }
+        return cls._ALL_TRIGGER_RULES

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/16ab314c/airflow/utils/weight_rule.py
----------------------------------------------------------------------
diff --git a/airflow/utils/weight_rule.py b/airflow/utils/weight_rule.py
index fde0d90..51f45a6 100644
--- a/airflow/utils/weight_rule.py
+++ b/airflow/utils/weight_rule.py
@@ -22,12 +22,17 @@ class WeightRule(object):
     UPSTREAM = 'upstream'
     ABSOLUTE = 'absolute'
 
+    _ALL_WEIGHT_RULES = {}
     @classmethod
     def is_valid(cls, weight_rule):
         return weight_rule in cls.all_weight_rules()
 
     @classmethod
     def all_weight_rules(cls):
-        return [getattr(cls, attr)
+        if not cls._ALL_WEIGHT_RULES:
+            cls._ALL_WEIGHT_RULES = {
+                getattr(cls, attr)
                 for attr in dir(cls)
-                if not attr.startswith("__") and not callable(getattr(cls, attr))]
+                if not attr.startswith("_") and not callable(getattr(cls, attr))
+            }
+        return cls._ALL_WEIGHT_RULES

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/16ab314c/tests/utils/test_trigger_rule.py
----------------------------------------------------------------------
diff --git a/tests/utils/test_trigger_rule.py b/tests/utils/test_trigger_rule.py
new file mode 100644
index 0000000..f89a56e
--- /dev/null
+++ b/tests/utils/test_trigger_rule.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from airflow.utils.trigger_rule import TriggerRule
+
+
+class TestTriggerRule(unittest.TestCase):
+
+    def test_valid_trigger_rules(self):
+        self.assertTrue(TriggerRule.is_valid(TriggerRule.ALL_SUCCESS))
+        self.assertTrue(TriggerRule.is_valid(TriggerRule.ALL_FAILED))
+        self.assertTrue(TriggerRule.is_valid(TriggerRule.ALL_DONE))
+        self.assertTrue(TriggerRule.is_valid(TriggerRule.ONE_SUCCESS))
+        self.assertTrue(TriggerRule.is_valid(TriggerRule.ONE_FAILED))
+        self.assertTrue(TriggerRule.is_valid(TriggerRule.DUMMY))
+        self.assertEqual(len(TriggerRule.all_triggers()), 6)

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/16ab314c/tests/utils/test_weight_rule.py
----------------------------------------------------------------------
diff --git a/tests/utils/test_weight_rule.py b/tests/utils/test_weight_rule.py
new file mode 100644
index 0000000..8ca618d
--- /dev/null
+++ b/tests/utils/test_weight_rule.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from airflow.utils.weight_rule import WeightRule
+
+
+class TestWeightRule(unittest.TestCase):
+
+    def test_valid_weight_rules(self):
+        self.assertTrue(WeightRule.is_valid(WeightRule.DOWNSTREAM))
+        self.assertTrue(WeightRule.is_valid(WeightRule.UPSTREAM))
+        self.assertTrue(WeightRule.is_valid(WeightRule.ABSOLUTE))
+        self.assertEqual(len(WeightRule.all_weight_rules()), 3)


Mime
View raw message