airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-1145] Fix closest_date_partition function with before set to True If we're looking for the closest date before, we should take the latest date in the list of date before.
Date Sat, 13 May 2017 19:10:51 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/v1-8-test 0db3249df -> 9e6dfcbdb


[AIRFLOW-1145] Fix closest_date_partition function with before set to True
If we're looking for the closest date before, we should take the latest date in the list of
date before.

Closes #2257 from julien-gm/fix_closest-date-
partition

(cherry picked from commit 0da540bf7840ae3cac866e352fba2b8b5cd9a625)
Signed-off-by: Bolke de Bruin <bolke@xs4all.nl>


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/9e6dfcbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/9e6dfcbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/9e6dfcbd

Branch: refs/heads/v1-8-test
Commit: 9e6dfcbdb4a4bbd63a8683778e3b3dafe149150c
Parents: 0db3249
Author: julien-gm <julien-gm@users.noreply.github.com>
Authored: Sat May 13 21:10:13 2017 +0200
Committer: Bolke de Bruin <bolke@xs4all.nl>
Committed: Sat May 13 21:10:46 2017 +0200

----------------------------------------------------------------------
 airflow/macros/hive.py    |  4 ++--
 tests/macros/__init__.py  | 14 ++++++++++++++
 tests/macros/test_hive.py | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 57 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/9e6dfcbd/airflow/macros/hive.py
----------------------------------------------------------------------
diff --git a/airflow/macros/hive.py b/airflow/macros/hive.py
index 13b8234..c68c293 100644
--- a/airflow/macros/hive.py
+++ b/airflow/macros/hive.py
@@ -61,8 +61,8 @@ def _closest_date(target_dt, date_list, before_target=None):
     :returns: The closest date
     :rtype: datetime.date or None
     '''
-    fb = lambda d: d - target_dt if d >= target_dt else datetime.timedelta.max
-    fa = lambda d: d - target_dt if d <= target_dt else datetime.timedelta.min
+    fb = lambda d: target_dt - d if d <= target_dt else datetime.timedelta.max
+    fa = lambda d: d - target_dt if d >= target_dt else datetime.timedelta.max
     fnone = lambda d: target_dt - d if d < target_dt else d - target_dt
     if before_target is None:
         return min(date_list, key=fnone).date()

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/9e6dfcbd/tests/macros/__init__.py
----------------------------------------------------------------------
diff --git a/tests/macros/__init__.py b/tests/macros/__init__.py
new file mode 100644
index 0000000..759b563
--- /dev/null
+++ b/tests/macros/__init__.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/9e6dfcbd/tests/macros/test_hive.py
----------------------------------------------------------------------
diff --git a/tests/macros/test_hive.py b/tests/macros/test_hive.py
new file mode 100644
index 0000000..17ee68f
--- /dev/null
+++ b/tests/macros/test_hive.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime, timedelta
+import unittest
+
+from airflow.macros import hive
+
+
+class Hive(unittest.TestCase):
+    def test_closest_ds_partition(self):
+        d1 = datetime.strptime('2017-04-24', '%Y-%m-%d')
+        d2 = datetime.strptime('2017-04-25', '%Y-%m-%d')
+        d3 = datetime.strptime('2017-04-26', '%Y-%m-%d')
+        d4 = datetime.strptime('2017-04-28', '%Y-%m-%d')
+        d5 = datetime.strptime('2017-04-29', '%Y-%m-%d')
+        target_dt = datetime.strptime('2017-04-27', '%Y-%m-%d')
+        date_list = [d1, d2, d3, d4, d5]
+
+        self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list, True)))
+        self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, date_list, False)))
+
+        # when before is not set, the closest date should be returned
+        self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, [d1, d2, d3, d5],
None)))
+        self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, [d1, d2, d4, d5])))
+        self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list)))
+
+
+if __name__ == '__main__':
+    unittest.main()


Mime
View raw message