airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fo...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-2226] Rename google_cloud_storage_default to google_cloud_default
Date Mon, 19 Mar 2018 21:02:18 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master e83b01551 -> bb287ecf5


[AIRFLOW-2226] Rename google_cloud_storage_default to google_cloud_default

The Google cloud operators uses both
google_cloud_storage_default and
google_cloud_default as a default conn_id. This is
confusing and the
google_cloud_storage_default conn_id isnt
initialized by default in db.py
Therefore we rename the
google_cloud_storage_default to
google_cloud_default for simplicity and
convenience

Closes #3141 from Fokko/airflow-2226


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/bb287ecf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/bb287ecf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/bb287ecf

Branch: refs/heads/master
Commit: bb287ecf5bd1777124801c14750c6b685cc27c76
Parents: e83b015
Author: Fokko Driesprong <fokkodriesprong@godatadriven.com>
Authored: Mon Mar 19 22:02:10 2018 +0100
Committer: Fokko Driesprong <fokkodriesprong@godatadriven.com>
Committed: Mon Mar 19 22:02:10 2018 +0100

----------------------------------------------------------------------
 UPDATING.md                                           | 4 ++++
 airflow/contrib/hooks/gcs_hook.py                     | 2 +-
 airflow/contrib/operators/bigquery_operator.py        | 4 ++--
 airflow/contrib/operators/file_to_gcs.py              | 4 ++--
 airflow/contrib/operators/gcs_copy_operator.py        | 2 +-
 airflow/contrib/operators/gcs_download_operator.py    | 2 +-
 airflow/contrib/operators/gcs_list_operator.py        | 2 +-
 airflow/contrib/operators/gcs_operator.py             | 2 +-
 airflow/contrib/operators/gcs_to_bq.py                | 2 +-
 airflow/contrib/operators/gcs_to_gcs.py               | 2 +-
 airflow/contrib/operators/mysql_to_gcs.py             | 2 +-
 airflow/contrib/operators/postgres_to_gcs_operator.py | 2 +-
 airflow/contrib/sensors/gcs_sensor.py                 | 6 +++---
 13 files changed, 20 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/UPDATING.md
----------------------------------------------------------------------
diff --git a/UPDATING.md b/UPDATING.md
index e03243f..21581bf 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -24,6 +24,10 @@ Dataflow job labeling is now supported in Dataflow{Java,Python}Operator
with a d
 "airflow-version" label, please upgrade your google-cloud-dataflow or apache-beam version
 to 2.2.0 or greater.
 
+### Google cloud connection string
+
+With Airflow 1.9 or lower there where two connection strings for the Google Cloud operators,
both `google_cloud_storage_default` and `google_cloud_default`. This can be confusing and
therefore the `google_cloud_storage_default` connection id has been replaced with `google_cloud_default`
to make the connection id consistent across Airflow.
+
 ## Airflow 1.9
 
 ### SSH Hook updates, along with new SSH Operator & SFTP Operator

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/hooks/gcs_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/gcs_hook.py b/airflow/contrib/hooks/gcs_hook.py
index 8c1e7bb..6a53b1d 100644
--- a/airflow/contrib/hooks/gcs_hook.py
+++ b/airflow/contrib/hooks/gcs_hook.py
@@ -27,7 +27,7 @@ class GoogleCloudStorageHook(GoogleCloudBaseHook):
     """
 
     def __init__(self,
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None):
         super(GoogleCloudStorageHook, self).__init__(google_cloud_storage_conn_id,
                                                      delegate_to)

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/bigquery_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/bigquery_operator.py b/airflow/contrib/operators/bigquery_operator.py
index ef1ada0..6f1cc6c 100644
--- a/airflow/contrib/operators/bigquery_operator.py
+++ b/airflow/contrib/operators/bigquery_operator.py
@@ -242,7 +242,7 @@ class BigQueryCreateEmptyTableOperator(BaseOperator):
                  gcs_schema_object=None,
                  time_partitioning={},
                  bigquery_conn_id='bigquery_default',
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  *args, **kwargs):
 
@@ -376,7 +376,7 @@ class BigQueryCreateExternalTableOperator(BaseOperator):
                  allow_quoted_newlines=False,
                  allow_jagged_rows=False,
                  bigquery_conn_id='bigquery_default',
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  src_fmt_configs={},
                  *args, **kwargs):

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/file_to_gcs.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/file_to_gcs.py b/airflow/contrib/operators/file_to_gcs.py
index b9f68f9..eaf8c9d 100644
--- a/airflow/contrib/operators/file_to_gcs.py
+++ b/airflow/contrib/operators/file_to_gcs.py
@@ -36,13 +36,13 @@ class FileToGoogleCloudStorageOperator(BaseOperator):
     :type delegate_to: string
     """
     template_fields = ('src', 'dst', 'bucket')
-    
+
     @apply_defaults
     def __init__(self,
                  src,
                  dst,
                  bucket,
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  mime_type='application/octet-stream',
                  delegate_to=None,
                  *args,

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_copy_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_copy_operator.py b/airflow/contrib/operators/gcs_copy_operator.py
index 4922708..842d6dc 100644
--- a/airflow/contrib/operators/gcs_copy_operator.py
+++ b/airflow/contrib/operators/gcs_copy_operator.py
@@ -67,7 +67,7 @@ class GoogleCloudStorageCopyOperator(BaseOperator):
                  source_files_delimiter=None,
                  destination_bucket=None,
                  destination_directory='',
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  *args,
                  **kwargs):

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_download_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_download_operator.py b/airflow/contrib/operators/gcs_download_operator.py
index 75de608..f6276b5 100644
--- a/airflow/contrib/operators/gcs_download_operator.py
+++ b/airflow/contrib/operators/gcs_download_operator.py
@@ -53,7 +53,7 @@ class GoogleCloudStorageDownloadOperator(BaseOperator):
                  object,
                  filename=None,
                  store_to_xcom_key=None,
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  *args,
                  **kwargs):

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_list_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_list_operator.py b/airflow/contrib/operators/gcs_list_operator.py
index 3c49cc1..a708524 100644
--- a/airflow/contrib/operators/gcs_list_operator.py
+++ b/airflow/contrib/operators/gcs_list_operator.py
@@ -60,7 +60,7 @@ class GoogleCloudStorageListOperator(BaseOperator):
                  bucket,
                  prefix=None,
                  delimiter=None,
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  *args,
                  **kwargs):

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_operator.py b/airflow/contrib/operators/gcs_operator.py
index d92d125..f03b5ec 100644
--- a/airflow/contrib/operators/gcs_operator.py
+++ b/airflow/contrib/operators/gcs_operator.py
@@ -85,7 +85,7 @@ class GoogleCloudStorageCreateBucketOperator(BaseOperator):
                  location='US',
                  project_id=None,
                  labels=None,
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  *args,
                  **kwargs):

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_to_bq.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_to_bq.py b/airflow/contrib/operators/gcs_to_bq.py
index be15e52..3126ae0 100644
--- a/airflow/contrib/operators/gcs_to_bq.py
+++ b/airflow/contrib/operators/gcs_to_bq.py
@@ -135,7 +135,7 @@ class GoogleCloudStorageToBigQueryOperator(BaseOperator):
                  allow_jagged_rows=False,
                  max_id_key=None,
                  bigquery_conn_id='bigquery_default',
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  schema_update_options=(),
                  src_fmt_configs={},

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_to_gcs.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_to_gcs.py b/airflow/contrib/operators/gcs_to_gcs.py
index 8db3067..b6e49dc 100644
--- a/airflow/contrib/operators/gcs_to_gcs.py
+++ b/airflow/contrib/operators/gcs_to_gcs.py
@@ -65,7 +65,7 @@ class GoogleCloudStorageToGoogleCloudStorageOperator(BaseOperator):
                  destination_bucket=None,
                  destination_object=None,
                  move_object=False,
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  *args,
                  **kwargs):

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/mysql_to_gcs.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/mysql_to_gcs.py b/airflow/contrib/operators/mysql_to_gcs.py
index 9ba84c7..2249b9c 100644
--- a/airflow/contrib/operators/mysql_to_gcs.py
+++ b/airflow/contrib/operators/mysql_to_gcs.py
@@ -45,7 +45,7 @@ class MySqlToGoogleCloudStorageOperator(BaseOperator):
                  schema_filename=None,
                  approx_max_file_size_bytes=1900000000,
                  mysql_conn_id='mysql_default',
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  schema=None,
                  delegate_to=None,
                  *args,

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/postgres_to_gcs_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/postgres_to_gcs_operator.py b/airflow/contrib/operators/postgres_to_gcs_operator.py
index ab6fdf4..8224b36 100644
--- a/airflow/contrib/operators/postgres_to_gcs_operator.py
+++ b/airflow/contrib/operators/postgres_to_gcs_operator.py
@@ -44,7 +44,7 @@ class PostgresToGoogleCloudStorageOperator(BaseOperator):
                  schema_filename=None,
                  approx_max_file_size_bytes=1900000000,
                  postgres_conn_id='postgres_default',
-                 google_cloud_storage_conn_id='google_cloud_storage_default',
+                 google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  parameters=None,
                  *args,

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/sensors/gcs_sensor.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/sensors/gcs_sensor.py b/airflow/contrib/sensors/gcs_sensor.py
index d91e2b1..aa7afd3 100644
--- a/airflow/contrib/sensors/gcs_sensor.py
+++ b/airflow/contrib/sensors/gcs_sensor.py
@@ -28,7 +28,7 @@ class GoogleCloudStorageObjectSensor(BaseSensorOperator):
             self,
             bucket,
             object,  # pylint:disable=redefined-builtin
-            google_cloud_conn_id='google_cloud_storage_default',
+            google_cloud_conn_id='google_cloud_default',
             delegate_to=None,
             *args,
             **kwargs):
@@ -84,7 +84,7 @@ class GoogleCloudStorageObjectUpdatedSensor(BaseSensorOperator):
             bucket,
             object,  # pylint:disable=redefined-builtin
             ts_func=ts_function,
-            google_cloud_conn_id='google_cloud_storage_default',
+            google_cloud_conn_id='google_cloud_default',
             delegate_to=None,
             *args,
             **kwargs):
@@ -135,7 +135,7 @@ class GoogleCloudStoragePrefixSensor(BaseSensorOperator):
         self,
         bucket,
         prefix,
-        google_cloud_conn_id='google_cloud_storage_default',
+        google_cloud_conn_id='google_cloud_default',
         delegate_to=None,
         *args,
         **kwargs):


Mime
View raw message