airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject [1/9] incubator-airflow git commit: [AIRFLOW-737] Fix HDFS Sensor directory.
Date Mon, 16 Jan 2017 16:48:28 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/v1-8-test 5782ad4b6 -> df9464bba


[AIRFLOW-737] Fix HDFS Sensor directory.

Due to a bad ordering in the fake snakebite
client, one test was wrongly True.

Closes #1989 from
vfoucault/fixbug/hdfssensor_folder


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/8fd65f8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/8fd65f8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/8fd65f8a

Branch: refs/heads/v1-8-test
Commit: 8fd65f8ac4a305da01324c5120fbfab8b51f5d46
Parents: 4a5e17b
Author: Vianney Foucault <vianney.foucault@gmail.com>
Authored: Fri Jan 13 21:15:14 2017 +0100
Committer: Bolke de Bruin <bolke@xs4all.nl>
Committed: Fri Jan 13 21:15:14 2017 +0100

----------------------------------------------------------------------
 airflow/contrib/sensors/hdfs_sensors.py | 1 +
 tests/core.py                           | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8fd65f8a/airflow/contrib/sensors/hdfs_sensors.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/sensors/hdfs_sensors.py b/airflow/contrib/sensors/hdfs_sensors.py
index a12e295..4e9bb9b 100644
--- a/airflow/contrib/sensors/hdfs_sensors.py
+++ b/airflow/contrib/sensors/hdfs_sensors.py
@@ -62,6 +62,7 @@ class HdfsSensorFolder(HdfsSensor):
             return len(result) == 1 and result[0]['path'] == self.filepath
         else:
             logging.info('Poking for filepath {self.filepath} to a non empty directory'.format(**locals()))
+            result.pop(0)
             return bool(result) and result[0]['file_type'] == 'f'
 
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8fd65f8a/tests/core.py
----------------------------------------------------------------------
diff --git a/tests/core.py b/tests/core.py
index 6fb735f..0f7e41d 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -1840,10 +1840,6 @@ class FakeSnakeBiteClient(object):
                 {'group': u'supergroup', 'permission': 493, 'file_type': 'd', 'access_time':
0, 'block_replication': 0,
                  'modification_time': 1481132141540, 'length': 0, 'blocksize': 0, 'owner':
u'hdfs',
                  'path': '/datadirectory/empty_directory'}]
-        elif path[0] == '/datadirectory/not_empty_directory':
-            return [{'group': u'supergroup', 'permission': 420, 'file_type': 'f', 'access_time':
1481122343796,
-                     'block_replication': 3, 'modification_time': 1481122343862, 'length':
0, 'blocksize': 134217728,
-                     'owner': u'hdfs', 'path': '/datadirectory/not_empty_directory/test_file'}]
         elif path[0] == '/datadirectory/not_empty_directory' and include_toplevel:
             return [
                 {'group': u'supergroup', 'permission': 493, 'file_type': 'd', 'access_time':
0, 'block_replication': 0,
@@ -1852,6 +1848,10 @@ class FakeSnakeBiteClient(object):
                 {'group': u'supergroup', 'permission': 420, 'file_type': 'f', 'access_time':
1481122343796,
                  'block_replication': 3, 'modification_time': 1481122343862, 'length': 0,
'blocksize': 134217728,
                  'owner': u'hdfs', 'path': '/datadirectory/not_empty_directory/test_file'}]
+        elif path[0] == '/datadirectory/not_empty_directory':
+            return [{'group': u'supergroup', 'permission': 420, 'file_type': 'f', 'access_time':
1481122343796,
+                     'block_replication': 3, 'modification_time': 1481122343862, 'length':
0, 'blocksize': 134217728,
+                     'owner': u'hdfs', 'path': '/datadirectory/not_empty_directory/test_file'}]
         elif path[0] == '/datadirectory/not_existing_file_or_directory':
             raise FakeSnakeBiteClientException
         elif path[0] == '/datadirectory/regex_dir':


Mime
View raw message