airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fo...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-2328] Fix empty GCS blob in S3ToGoogleCloudStorageOperator
Date Sat, 21 Apr 2018 06:37:38 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master 8e83e2b3e -> b5f758bb6


[AIRFLOW-2328] Fix empty GCS blob in S3ToGoogleCloudStorageOperator

Closes #3231 from wileeam/fix-empty-blob-in-s3-to-
gcs-operator


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/b5f758bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/b5f758bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/b5f758bb

Branch: refs/heads/master
Commit: b5f758bb618af2878cf90e5418d3abc8c234b45f
Parents: 8e83e2b
Author: Guillermo Rodríguez Cano <guillermo.rodriguezcano@bonnierbroadcasting.com>
Authored: Sat Apr 21 08:36:38 2018 +0200
Committer: Fokko Driesprong <fokkodriesprong@godatadriven.com>
Committed: Sat Apr 21 08:36:38 2018 +0200

----------------------------------------------------------------------
 airflow/contrib/operators/s3_to_gcs_operator.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/b5f758bb/airflow/contrib/operators/s3_to_gcs_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/s3_to_gcs_operator.py b/airflow/contrib/operators/s3_to_gcs_operator.py
index d48e468..d105596 100644
--- a/airflow/contrib/operators/s3_to_gcs_operator.py
+++ b/airflow/contrib/operators/s3_to_gcs_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -102,11 +102,12 @@ class S3ToGoogleCloudStorageOperator(S3ListOperator):
         self.replace = replace
 
         if dest_gcs and not self._gcs_object_is_directory(self.dest_gcs):
-            self.log.info('Destination Google Cloud Storage path is not a '
-                          'valid "directory", define one and end the path '
-                          'with a slash: "/".')
+            self.log.info(
+                'Destination Google Cloud Storage path is not a valid '
+                '"directory", define a path that ends with a slash "/" or '
+                'leave it empty for the root of the bucket.')
             raise AirflowException('The destination Google Cloud Storage path '
-                                   'must end with a slash "/".')
+                                   'must end with a slash "/" or be empty.')
 
     def execute(self, context):
         # use the super method to list all the files in an S3 bucket/key
@@ -188,4 +189,4 @@ class S3ToGoogleCloudStorageOperator(S3ListOperator):
     def _gcs_object_is_directory(self, object):
         bucket, blob = _parse_gcs_url(object)
 
-        return blob.endswith('/')
+        return len(blob) == 0 or blob.endswith('/')


Mime
View raw message