airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (AIRFLOW-3371) BigQueryHook's Ability to Create View
Date Wed, 21 Nov 2018 23:17:00 GMT

    [ https://issues.apache.org/jira/browse/AIRFLOW-3371?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16695349#comment-16695349
] 

ASF GitHub Bot commented on AIRFLOW-3371:
-----------------------------------------

kaxil closed pull request #4213: [AIRFLOW-3371] BigQueryHook's Ability to Create View
URL: https://github.com/apache/incubator-airflow/pull/4213
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/airflow/contrib/hooks/bigquery_hook.py b/airflow/contrib/hooks/bigquery_hook.py
index a03429e155..c7324adde4 100644
--- a/airflow/contrib/hooks/bigquery_hook.py
+++ b/airflow/contrib/hooks/bigquery_hook.py
@@ -218,10 +218,11 @@ def create_empty_table(self,
                            table_id,
                            schema_fields=None,
                            time_partitioning=None,
-                           labels=None
-                           ):
+                           labels=None,
+                           view=None):
         """
         Creates a new, empty table in the dataset.
+        To create a view, which is defined by a SQL query, parse a dictionary to 'view' kwarg
 
         :param project_id: The project to create the table into.
         :type project_id: str
@@ -246,6 +247,17 @@ def create_empty_table(self,
             .. seealso::
             https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning
         :type time_partitioning: dict
+        :param view: [Optional] A dictionary containing definition for the view.
+            If set, it will create a view instead of a table:
+            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#view
+        :type view: dict
+
+        **Example**: ::
+
+            view = {
+                "query": "SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*`
LIMIT 1000",
+                "useLegacySql": False
+            }
 
         :return:
         """
@@ -267,6 +279,9 @@ def create_empty_table(self,
         if labels:
             table_resource['labels'] = labels
 
+        if view:
+            table_resource['view'] = view
+
         self.log.info('Creating Table %s:%s.%s',
                       project_id, dataset_id, table_id)
 
diff --git a/tests/contrib/hooks/test_bigquery_hook.py b/tests/contrib/hooks/test_bigquery_hook.py
index 9099dcbbb7..8c59116c85 100644
--- a/tests/contrib/hooks/test_bigquery_hook.py
+++ b/tests/contrib/hooks/test_bigquery_hook.py
@@ -22,6 +22,7 @@
 
 from google.auth.exceptions import GoogleAuthError
 import mock
+from apiclient.errors import HttpError
 
 from airflow.contrib.hooks import bigquery_hook as hook
 from airflow.contrib.hooks.bigquery_hook import _cleanse_time_partitioning, \
@@ -344,6 +345,48 @@ def test_insert_all_fail(self, run_with_config):
             cursor.insert_all(project_id, dataset_id, table_id,
                               rows, fail_on_error=True)
 
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_create_view_fails_on_exception(self, run_with_config):
+        project_id = 'bq-project'
+        dataset_id = 'bq_dataset'
+        table_id = 'bq_table_view'
+        view = {
+            'incorrect_key': 'SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*`',
+            "useLegacySql": False
+        }
+
+        mock_service = mock.Mock()
+        method = (mock_service.tables.return_value.insert)
+        method.return_value.execute.side_effect = HttpError(
+            resp={'status': '400'}, content=b'Query is required for views')
+        cursor = hook.BigQueryBaseCursor(mock_service, project_id)
+        with self.assertRaises(Exception):
+            cursor.create_empty_table(project_id, dataset_id, table_id,
+                                      view=view)
+
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_create_view(self, run_with_config):
+        project_id = 'bq-project'
+        dataset_id = 'bq_dataset'
+        table_id = 'bq_table_view'
+        view = {
+            'query': 'SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*`',
+            "useLegacySql": False
+        }
+
+        mock_service = mock.Mock()
+        method = (mock_service.tables.return_value.insert)
+        cursor = hook.BigQueryBaseCursor(mock_service, project_id)
+        cursor.create_empty_table(project_id, dataset_id, table_id,
+                                  view=view)
+        body = {
+            'tableReference': {
+                'tableId': table_id
+            },
+            'view': view
+        }
+        method.assert_called_once_with(projectId=project_id, datasetId=dataset_id, body=body)
+
 
 class TestBigQueryCursor(unittest.TestCase):
     @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> BigQueryHook's Ability to Create View
> -------------------------------------
>
>                 Key: AIRFLOW-3371
>                 URL: https://issues.apache.org/jira/browse/AIRFLOW-3371
>             Project: Apache Airflow
>          Issue Type: New Feature
>            Reporter: Ryan Yuan
>            Assignee: Ryan Yuan
>            Priority: Major
>
> Modify *BigQueryBaseCursor.create_empty_table()* to take in an optional 'view' parameter
to create view in BigQuery.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message