Mailing-List: contact commits-help@airflow.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@airflow.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: bolke@apache.org
To: commits@airflow.incubator.apache.org
Message-Id: <7e5643bb16464c19b419d1237ae1c310@git.apache.org>
Subject: incubator-airflow git commit: [AIRFLOW-1145] Fix
 closest_date_partition function with before set to True If we're looking for
 the closest date before,
 we should take the latest date in the list of date before.
Date: Sat, 13 May 2017 19:10:51 +0000 (UTC)
archived-at: Sat, 13 May 2017 19:10:57 -0000

Repository: incubator-airflow
Updated Branches:
  refs/heads/v1-8-test 0db3249df -> 9e6dfcbdb


[AIRFLOW-1145] Fix closest_date_partition function with before set to True
If we're looking for the closest date before, we should take the latest date in the list of date before.

Closes #2257 from julien-gm/fix_closest-date-
partition

(cherry picked from commit 0da540bf7840ae3cac866e352fba2b8b5cd9a625)
Signed-off-by: Bolke de Bruin <bolke@xs4all.nl>


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/9e6dfcbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/9e6dfcbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/9e6dfcbd

Branch: refs/heads/v1-8-test
Commit: 9e6dfcbdb4a4bbd63a8683778e3b3dafe149150c
Parents: 0db3249
Author: julien-gm <julien-gm@users.noreply.github.com>
Authored: Sat May 13 21:10:13 2017 +0200
Committer: Bolke de Bruin <bolke@xs4all.nl>
Committed: Sat May 13 21:10:46 2017 +0200

----------------------------------------------------------------------
 airflow/macros/hive.py    |  4 ++--
 tests/macros/__init__.py  | 14 ++++++++++++++
 tests/macros/test_hive.py | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 57 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/9e6dfcbd/airflow/macros/hive.py
----------------------------------------------------------------------
diff --git a/airflow/macros/hive.py b/airflow/macros/hive.py
index 13b8234..c68c293 100644
--- a/airflow/macros/hive.py
+++ b/airflow/macros/hive.py
@@ -61,8 +61,8 @@ def _closest_date(target_dt, date_list, before_target=None):
     :returns: The closest date
     :rtype: datetime.date or None
     '''
-    fb = lambda d: d - target_dt if d >= target_dt else datetime.timedelta.max
-    fa = lambda d: d - target_dt if d <= target_dt else datetime.timedelta.min
+    fb = lambda d: target_dt - d if d <= target_dt else datetime.timedelta.max
+    fa = lambda d: d - target_dt if d >= target_dt else datetime.timedelta.max
     fnone = lambda d: target_dt - d if d < target_dt else d - target_dt
     if before_target is None:
         return min(date_list, key=fnone).date()

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/9e6dfcbd/tests/macros/__init__.py
----------------------------------------------------------------------
diff --git a/tests/macros/__init__.py b/tests/macros/__init__.py
new file mode 100644
index 0000000..759b563
--- /dev/null
+++ b/tests/macros/__init__.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/9e6dfcbd/tests/macros/test_hive.py
----------------------------------------------------------------------
diff --git a/tests/macros/test_hive.py b/tests/macros/test_hive.py
new file mode 100644
index 0000000..17ee68f
--- /dev/null
+++ b/tests/macros/test_hive.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime, timedelta
+import unittest
+
+from airflow.macros import hive
+
+
+class Hive(unittest.TestCase):
+    def test_closest_ds_partition(self):
+        d1 = datetime.strptime('2017-04-24', '%Y-%m-%d')
+        d2 = datetime.strptime('2017-04-25', '%Y-%m-%d')
+        d3 = datetime.strptime('2017-04-26', '%Y-%m-%d')
+        d4 = datetime.strptime('2017-04-28', '%Y-%m-%d')
+        d5 = datetime.strptime('2017-04-29', '%Y-%m-%d')
+        target_dt = datetime.strptime('2017-04-27', '%Y-%m-%d')
+        date_list = [d1, d2, d3, d4, d5]
+
+        self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list, True)))
+        self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, date_list, False)))
+
+        # when before is not set, the closest date should be returned
+        self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, [d1, d2, d3, d5], None)))
+        self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, [d1, d2, d4, d5])))
+        self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list)))
+
+
+if __name__ == '__main__':
+    unittest.main()