beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rober...@apache.org
Subject [1/2] incubator-beam git commit: Allow for passing format so that we can migrate to BQ Avro export later
Date Tue, 15 Nov 2016 16:53:17 GMT
Repository: incubator-beam
Updated Branches:
  refs/heads/python-sdk 6ac6e420f -> c4208a899


Allow for passing format so that we can migrate to BQ Avro export later


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d0e31218
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d0e31218
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d0e31218

Branch: refs/heads/python-sdk
Commit: d0e312184e319050baa02abff2c08348b6cfb651
Parents: 6ac6e42
Author: Sourabh Bajaj <sourabhbajaj@google.com>
Authored: Mon Nov 7 18:15:17 2016 -0800
Committer: Robert Bradshaw <robertwb@gmail.com>
Committed: Tue Nov 15 08:53:06 2016 -0800

----------------------------------------------------------------------
 sdks/python/apache_beam/runners/dataflow_runner.py | 1 +
 sdks/python/apache_beam/utils/names.py             | 1 +
 2 files changed, 2 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d0e31218/sdks/python/apache_beam/runners/dataflow_runner.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow_runner.py
index 57867fa..00b466b 100644
--- a/sdks/python/apache_beam/runners/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow_runner.py
@@ -515,6 +515,7 @@ class DataflowPipelineRunner(PipelineRunner):
     elif transform.source.format == 'text':
       step.add_property(PropertyNames.FILE_PATTERN, transform.source.path)
     elif transform.source.format == 'bigquery':
+      step.add_property(PropertyNames.BIGQUERY_EXPORT_FORMAT, 'FORMAT_JSON')
       # TODO(silviuc): Add table validation if transform.source.validate.
       if transform.source.table_reference is not None:
         step.add_property(PropertyNames.BIGQUERY_DATASET,

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d0e31218/sdks/python/apache_beam/utils/names.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/names.py b/sdks/python/apache_beam/utils/names.py
index be8c92a..3edde3c 100644
--- a/sdks/python/apache_beam/utils/names.py
+++ b/sdks/python/apache_beam/utils/names.py
@@ -46,6 +46,7 @@ class PropertyNames(object):
   BIGQUERY_DATASET = 'dataset'
   BIGQUERY_QUERY = 'bigquery_query'
   BIGQUERY_USE_LEGACY_SQL = 'bigquery_use_legacy_sql'
+  BIGQUERY_EXPORT_FORMAT = 'bigquery_export_format'
   BIGQUERY_TABLE = 'table'
   BIGQUERY_PROJECT = 'project'
   BIGQUERY_SCHEMA = 'schema'


Mime
View raw message