Mailing-List: contact commits-help@airflow.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@airflow.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: criccomini@apache.org
To: commits@airflow.incubator.apache.org
Message-Id: <974f5da84b40480d91fad6a6308f5628@git.apache.org>
Subject: incubator-airflow git commit: [AIRFLOW-1635] Allow creating GCP
 connection without requiring a JSON file
Date: Thu, 28 Sep 2017 16:19:49 +0000 (UTC)
archived-at: Thu, 28 Sep 2017 16:19:54 -0000

Repository: incubator-airflow
Updated Branches:
  refs/heads/master a87ced4c1 -> b3e985a31


[AIRFLOW-1635] Allow creating GCP connection without requiring a JSON file

Closes #2640 from barrywhart/airflow-1635-gcp-
json-data-master


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/b3e985a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/b3e985a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/b3e985a3

Branch: refs/heads/master
Commit: b3e985a3146272ecfd3ceaaa0d8567e4e9e117d4
Parents: a87ced4
Author: Barry Hart <barrywhart@yahoo.com>
Authored: Thu Sep 28 09:19:43 2017 -0700
Committer: Chris Riccomini <criccomini@apache.org>
Committed: Thu Sep 28 09:19:43 2017 -0700

----------------------------------------------------------------------
 airflow/contrib/hooks/gcp_api_base_hook.py | 27 +++++++++++++++++++++++--
 airflow/www/views.py                       |  2 ++
 2 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/b3e985a3/airflow/contrib/hooks/gcp_api_base_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/gcp_api_base_hook.py b/airflow/contrib/hooks/gcp_api_base_hook.py
index 28721d3..e6ca240 100644
--- a/airflow/contrib/hooks/gcp_api_base_hook.py
+++ b/airflow/contrib/hooks/gcp_api_base_hook.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import json
+
 import httplib2
 from oauth2client.client import GoogleCredentials
 from oauth2client.service_account import ServiceAccountCredentials
@@ -59,20 +61,23 @@ class GoogleCloudBaseHook(BaseHook, LoggingMixin):
         Returns the Credentials object for Google API
         """
         key_path = self._get_field('key_path', False)
+        keyfile_dict = self._get_field('keyfile_dict', False)
         scope = self._get_field('scope', False)
 
         kwargs = {}
         if self.delegate_to:
             kwargs['sub'] = self.delegate_to
 
-        if not key_path:
+        if not key_path and not keyfile_dict:
             self.log.info('Getting connection using `gcloud auth` user, since no key file '
                          'is defined for hook.')
             credentials = GoogleCredentials.get_application_default()
-        else:
+        elif key_path:
             if not scope:
                 raise AirflowException('Scope should be defined when using a key file.')
             scopes = [s.strip() for s in scope.split(',')]
+
+            # Get credentials from a JSON file.
             if key_path.endswith('.json'):
                 self.log.info('Getting connection using a JSON key file.')
                 credentials = ServiceAccountCredentials\
@@ -82,6 +87,24 @@ class GoogleCloudBaseHook(BaseHook, LoggingMixin):
                                        'use a JSON key file.')
             else:
                 raise AirflowException('Unrecognised extension for key file.')
+        else:
+            if not scope:
+                raise AirflowException('Scope should be defined when using key JSON.')
+            scopes = [s.strip() for s in scope.split(',')]
+
+            # Get credentials from JSON data provided in the UI.
+            try:
+                keyfile_dict = json.loads(keyfile_dict)
+
+                # Depending on how the JSON was formatted, it may contain
+                # escaped newlines. Convert those to actual newlines.
+                keyfile_dict['private_key'] = keyfile_dict['private_key'].replace(
+                    '\\n', '\n')
+
+                credentials = ServiceAccountCredentials\
+                    .from_json_keyfile_dict(keyfile_dict, scopes)
+            except json.decoder.JSONDecodeError:
+                raise AirflowException('Invalid key JSON.')
         return credentials
 
     def _get_access_token(self):

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/b3e985a3/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index a0a0999..ad27238 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2582,6 +2582,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView):
         'extra__jdbc__drv_clsname',
         'extra__google_cloud_platform__project',
         'extra__google_cloud_platform__key_path',
+        'extra__google_cloud_platform__keyfile_dict',
         'extra__google_cloud_platform__scope',
     )
     verbose_name = "Connection"
@@ -2603,6 +2604,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView):
         'extra__jdbc__drv_clsname': StringField('Driver Class'),
         'extra__google_cloud_platform__project': StringField('Project Id'),
         'extra__google_cloud_platform__key_path': StringField('Keyfile Path'),
+        'extra__google_cloud_platform__keyfile_dict': PasswordField('Keyfile JSON'),
         'extra__google_cloud_platform__scope': StringField('Scopes (comma seperated)'),
 
     }