beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bchamb...@apache.org
Subject [1/2] incubator-beam git commit: Display data keys in Python should be snake_case
Date Wed, 07 Dec 2016 02:26:37 GMT
Repository: incubator-beam
Updated Branches:
  refs/heads/python-sdk f19f767b0 -> d5e8c79a3


Display data keys in Python should be snake_case

This fixes existing mis-cased display data keys.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0dcc9bc0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0dcc9bc0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0dcc9bc0

Branch: refs/heads/python-sdk
Commit: 0dcc9bc03772d8bd5ab1e0d138c445e9418ebf66
Parents: f19f767
Author: Pablo <pabloem@google.com>
Authored: Mon Nov 28 13:05:47 2016 -0800
Committer: bchambers <bchambers@google.com>
Committed: Tue Dec 6 17:17:16 2016 -0800

----------------------------------------------------------------------
 sdks/python/apache_beam/io/avroio.py            |  8 +++-----
 sdks/python/apache_beam/io/avroio_test.py       |  8 ++++----
 sdks/python/apache_beam/io/filebasedsource.py   |  3 ++-
 .../apache_beam/io/filebasedsource_test.py      |  4 ++--
 sdks/python/apache_beam/io/fileio.py            |  8 ++++----
 sdks/python/apache_beam/io/fileio_test.py       | 12 ++++++------
 sdks/python/apache_beam/io/pubsub.py            |  2 +-
 sdks/python/apache_beam/io/pubsub_test.py       |  2 +-
 sdks/python/apache_beam/io/textio.py            | 20 ++++++++++----------
 sdks/python/apache_beam/io/textio_test.py       |  8 +++++---
 .../apache_beam/transforms/combiners_test.py    |  8 ++++----
 sdks/python/apache_beam/transforms/core.py      |  8 ++++----
 12 files changed, 46 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/avroio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index 6cba12d..35d0e94 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -75,8 +75,7 @@ class ReadFromAvro(PTransform):
       **kwargs: Additional keyword arguments to be passed to the base class.
     """
     super(ReadFromAvro, self).__init__()
-    self._args = (file_pattern, min_bundle_size)
-    self._source = _AvroSource(*self._args, validate=validate)
+    self._source = _AvroSource(file_pattern, min_bundle_size, validate=validate)
 
   def apply(self, pvalue):
     return pvalue.pipeline | Read(self._source)
@@ -292,9 +291,8 @@ class WriteToAvro(beam.transforms.PTransform):
     Returns:
       A WriteToAvro transform usable for writing.
     """
-    self._args = (file_path_prefix, schema, codec, file_name_suffix, num_shards,
-                  shard_name_template, mime_type)
-    self._sink = _AvroSink(*self._args)
+    self._sink = _AvroSink(file_path_prefix, schema, codec, file_name_suffix,
+                           num_shards, shard_name_template, mime_type)
 
   def apply(self, pcoll):
     return pcoll | beam.io.iobase.Write(self._sink)

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/avroio_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py
index e8fb12b..e5db196 100644
--- a/sdks/python/apache_beam/io/avroio_test.py
+++ b/sdks/python/apache_beam/io/avroio_test.py
@@ -157,7 +157,7 @@ class TestAvro(unittest.TestCase):
     # No extra avro parameters for AvroSource.
     expected_items = [
         DisplayDataItemMatcher('compression', 'auto'),
-        DisplayDataItemMatcher('filePattern', file_name)]
+        DisplayDataItemMatcher('file_pattern', file_name)]
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
   def test_read_display_data(self):
@@ -168,7 +168,7 @@ class TestAvro(unittest.TestCase):
     # No extra avro parameters for AvroSource.
     expected_items = [
         DisplayDataItemMatcher('compression', 'auto'),
-        DisplayDataItemMatcher('filePattern', file_name)]
+        DisplayDataItemMatcher('file_pattern', file_name)]
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
   def test_sink_display_data(self):
@@ -186,7 +186,7 @@ class TestAvro(unittest.TestCase):
             'schema',
             str(self.SCHEMA)),
         DisplayDataItemMatcher(
-            'filePattern',
+            'file_pattern',
             'some_avro_sink-%(shard_num)05d-of-%(num_shards)05d.end'),
         DisplayDataItemMatcher(
             'shards',
@@ -209,7 +209,7 @@ class TestAvro(unittest.TestCase):
             'schema',
             str(self.SCHEMA)),
         DisplayDataItemMatcher(
-            'filePattern',
+            'file_pattern',
             'some_avro_sink-%(shard_num)05d-of-%(num_shards)05d'),
         DisplayDataItemMatcher(
             'shards',

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/filebasedsource.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py
index 8921801..1bfde25 100644
--- a/sdks/python/apache_beam/io/filebasedsource.py
+++ b/sdks/python/apache_beam/io/filebasedsource.py
@@ -99,7 +99,8 @@ class FileBasedSource(iobase.BoundedSource):
       self._validate()
 
   def display_data(self):
-    return {'filePattern': DisplayDataItem(self._pattern, label="File Pattern"),
+    return {'file_pattern': DisplayDataItem(self._pattern,
+                                            label="File Pattern"),
             'compression': DisplayDataItem(str(self._compression_type),
                                            label='Compression Type')}
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/filebasedsource_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py
index a455cd3..ab7c69f 100644
--- a/sdks/python/apache_beam/io/filebasedsource_test.py
+++ b/sdks/python/apache_beam/io/filebasedsource_test.py
@@ -253,7 +253,7 @@ class TestFileBasedSource(unittest.TestCase):
     fbs = LineSource(file_name)
     dd = DisplayData.create_from(fbs)
     expected_items = [
-        DisplayDataItemMatcher('filePattern', file_name),
+        DisplayDataItemMatcher('file_pattern', file_name),
         DisplayDataItemMatcher('compression', 'auto')]
     hc.assert_that(dd.items,
                    hc.contains_inanyorder(*expected_items))
@@ -586,7 +586,7 @@ class TestSingleFileSource(unittest.TestCase):
     dd = DisplayData.create_from(fbs)
     expected_items = [
         DisplayDataItemMatcher('compression', 'auto'),
-        DisplayDataItemMatcher('filePattern', file_name)]
+        DisplayDataItemMatcher('file_pattern', file_name)]
     hc.assert_that(dd.items,
                    hc.contains_inanyorder(*expected_items))
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/fileio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 82e7813..d7ff35b 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -168,8 +168,8 @@ class NativeFileSource(dataflow_io.NativeSource):
     self.mime_type = mime_type
 
   def display_data(self):
-    return {'filePattern': DisplayDataItem(self.file_path,
-                                           label="File Pattern"),
+    return {'file_pattern': DisplayDataItem(self.file_path,
+                                            label="File Pattern"),
             'compression': DisplayDataItem(str(self.compression_type),
                                            label='Compression')}
 
@@ -845,7 +845,7 @@ class FileSink(iobase.Sink):
             DisplayDataItem(self.num_shards, label='Number of Shards'),
             'compression':
             DisplayDataItem(str(self.compression_type)),
-            'filePattern':
+            'file_pattern':
             DisplayDataItem('{}{}{}'.format(self.file_path_prefix,
                                             self.shard_name_format,
                                             self.file_name_suffix),
@@ -1136,7 +1136,7 @@ class NativeFileSink(dataflow_io.NativeSink):
     return {'shards':
             DisplayDataItem(self.num_shards,
                             label='Number of Shards'),
-            'filePattern':
+            'file_pattern':
             DisplayDataItem(file_name_pattern,
                             label='File Name Pattern'),
             'compression':

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/fileio_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index a68d484..dbf0e69 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -66,7 +66,7 @@ class TestTextFileSource(unittest.TestCase):
     self.assertEqual(read_lines, output_lines)
     dd = DisplayData.create_from(source)
     expected_items = [
-        DisplayDataItemMatcher('filePattern', file_name),
+        DisplayDataItemMatcher('file_pattern', file_name),
         DisplayDataItemMatcher('compression', 'auto')]
     hc.assert_that(dd.items,
                    hc.contains_inanyorder(*expected_items))
@@ -647,7 +647,7 @@ class TestNativeTextFileSink(unittest.TestCase):
     dd = DisplayData.create_from(sink)
     expected_items = [
         DisplayDataItemMatcher(
-            'filePattern',
+            'file_pattern',
             '{}{}'.format(self.path, '-SSSSS-of-NNNNN')),
         DisplayDataItemMatcher(
             'compression',
@@ -662,7 +662,7 @@ class TestNativeTextFileSink(unittest.TestCase):
     dd = DisplayData.create_from(sink)
     expected_items = [
         DisplayDataItemMatcher(
-            'filePattern',
+            'file_pattern',
             '{}{}{}'.format(self.path, '-SSSSS-of-NNNNN', '.pdf')),
         DisplayDataItemMatcher(
             'compression',
@@ -693,7 +693,7 @@ class TestNativeTextFileSink(unittest.TestCase):
     dd = DisplayData.create_from(sink)
     expected_items = [
         DisplayDataItemMatcher(
-            'filePattern',
+            'file_pattern',
             '{}{}'.format(self.path, '-SSSSS-of-NNNNN')),
         DisplayDataItemMatcher(
             'compression',
@@ -733,7 +733,7 @@ class TestNativeTextFileSink(unittest.TestCase):
     dd = DisplayData.create_from(sink)
     expected_items = [
         DisplayDataItemMatcher(
-            'filePattern',
+            'file_pattern',
             '{}{}'.format(self.path, '-SSSSS-of-NNNNN')),
         DisplayDataItemMatcher(
             'compression',
@@ -870,7 +870,7 @@ class TestFileSink(unittest.TestCase):
         DisplayDataItemMatcher(
             'compression', 'auto'),
         DisplayDataItemMatcher(
-            'filePattern',
+            'file_pattern',
             '{}{}'.format(temp_path,
                           '-%(shard_num)05d-of-%(num_shards)05d.foo'))]
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/pubsub.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/pubsub.py b/sdks/python/apache_beam/io/pubsub.py
index 40bd368..efc628d 100644
--- a/sdks/python/apache_beam/io/pubsub.py
+++ b/sdks/python/apache_beam/io/pubsub.py
@@ -56,7 +56,7 @@ class PubSubSource(dataflow_io.NativeSource):
     return 'pubsub'
 
   def display_data(self):
-    return {'idLabel':
+    return {'id_label':
             DisplayDataItem(self.id_label,
                             label='ID Label Attribute').drop_if_none(),
             'topic':

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/pubsub_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/pubsub_test.py b/sdks/python/apache_beam/io/pubsub_test.py
index 828d233..11b5d50 100644
--- a/sdks/python/apache_beam/io/pubsub_test.py
+++ b/sdks/python/apache_beam/io/pubsub_test.py
@@ -35,7 +35,7 @@ class TestPubSubSource(unittest.TestCase):
     expected_items = [
         DisplayDataItemMatcher('topic', 'a_topic'),
         DisplayDataItemMatcher('subscription', 'a_subscription'),
-        DisplayDataItemMatcher('idLabel', 'a_label')]
+        DisplayDataItemMatcher('id_label', 'a_label')]
 
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/textio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py
index bb664d1..ebadf69 100644
--- a/sdks/python/apache_beam/io/textio.py
+++ b/sdks/python/apache_beam/io/textio.py
@@ -234,11 +234,7 @@ class ReadFromText(PTransform):
     """
 
     super(ReadFromText, self).__init__(**kwargs)
-    self._file_pattern = file_pattern
-    self._min_bundle_size = min_bundle_size
-    self._compression_type = compression_type
     self._strip_trailing_newlines = strip_trailing_newlines
-    self._coder = coder
     self._source = _TextSource(file_pattern, min_bundle_size, compression_type,
                                strip_trailing_newlines, coder,
                                validate=validate)
@@ -248,8 +244,8 @@ class ReadFromText(PTransform):
 
   def display_data(self):
     return {'source_dd': self._source,
-            'strip_nwln': DisplayDataItem(self._strip_trailing_newlines,
-                                          label='Strip Trailing New Lines')}
+            'strip_newline': DisplayDataItem(self._strip_trailing_newlines,
+                                             label='Strip Trailing New Lines')}
 
 
 class WriteToText(PTransform):
@@ -296,12 +292,16 @@ class WriteToText(PTransform):
           compression.
     """
 
-    self._args = (file_path_prefix, file_name_suffix, append_trailing_newlines,
-                  num_shards, shard_name_template, coder, compression_type)
-    self._sink = _TextSink(*self._args)
+    self._append_trailing_newlines = append_trailing_newlines
+    self._sink = _TextSink(file_path_prefix, file_name_suffix,
+                           append_trailing_newlines, num_shards,
+                           shard_name_template, coder, compression_type)
 
   def apply(self, pcoll):
     return pcoll | Write(self._sink)
 
   def display_data(self):
-    return {'sink_dd': self._sink}
+    return {'sink_dd': self._sink,
+            'append_newline': DisplayDataItem(
+                self._append_trailing_newlines,
+                label='Append Trailing New Lines')}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/textio_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/textio_test.py b/sdks/python/apache_beam/io/textio_test.py
index b1d3fb0..5784fef 100644
--- a/sdks/python/apache_beam/io/textio_test.py
+++ b/sdks/python/apache_beam/io/textio_test.py
@@ -257,8 +257,8 @@ class TextSourceTest(unittest.TestCase):
     dd = DisplayData.create_from(read)
     expected_items = [
         DisplayDataItemMatcher('compression', 'auto'),
-        DisplayDataItemMatcher('filePattern', 'prefix'),
-        DisplayDataItemMatcher('strip_nwln', True)]
+        DisplayDataItemMatcher('file_pattern', 'prefix'),
+        DisplayDataItemMatcher('strip_newline', True)]
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
   def test_dataflow_single_file(self):
@@ -469,11 +469,13 @@ class TextSinkTest(unittest.TestCase):
     dd = DisplayData.create_from(write)
     expected_items = [
         DisplayDataItemMatcher(
+            'append_newline', True),
+        DisplayDataItemMatcher(
             'compression', 'auto'),
         DisplayDataItemMatcher(
             'shards', 0),
         DisplayDataItemMatcher(
-            'filePattern',
+            'file_pattern',
             '{}{}'.format('prefix',
                           '-%(shard_num)05d-of-%(num_shards)05d'))]
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/transforms/combiners_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py
index d28c63f..8dc274e 100644
--- a/sdks/python/apache_beam/transforms/combiners_test.py
+++ b/sdks/python/apache_beam/transforms/combiners_test.py
@@ -146,7 +146,7 @@ class CombineTest(unittest.TestCase):
       transform = beam.CombinePerKey(combineFn)
       dd = DisplayData.create_from(transform)
       expected_items = [
-          DisplayDataItemMatcher('combineFn', combineFn.__class__),
+          DisplayDataItemMatcher('combine_fn', combineFn.__class__),
           DisplayDataItemMatcher('n', combineFn._n),
           DisplayDataItemMatcher('compare', combineFn._compare.__name__)]
       hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
@@ -164,7 +164,7 @@ class CombineTest(unittest.TestCase):
         dd = DisplayData.create_from(transform)
         expected_items = [
             DisplayDataItemMatcher('fn', sampleFn.fn.__name__),
-            DisplayDataItemMatcher('combineFn',
+            DisplayDataItemMatcher('combine_fn',
                                    transform.fn.__class__)]
         if len(args) > 0:
           expected_items.append(
@@ -185,7 +185,7 @@ class CombineTest(unittest.TestCase):
     transform = beam.CombineGlobally(combine.Smallest(5))
     dd = DisplayData.create_from(transform)
     expected_items = [
-        DisplayDataItemMatcher('combineFn', combine.Smallest),
+        DisplayDataItemMatcher('combine_fn', combine.Smallest),
         DisplayDataItemMatcher('n', 5),
         DisplayDataItemMatcher('compare', 'gt')]
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
@@ -195,7 +195,7 @@ class CombineTest(unittest.TestCase):
         combine.TupleCombineFn(max, combine.MeanCombineFn(), sum))
     dd = DisplayData.create_from(transform)
     expected_items = [
-        DisplayDataItemMatcher('combineFn', combine.TupleCombineFn),
+        DisplayDataItemMatcher('combine_fn', combine.TupleCombineFn),
         DisplayDataItemMatcher('combiners',
                                "['max', 'MeanCombineFn', 'sum']")]
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/transforms/core.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index ffcdd10..523c5a6 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -832,9 +832,9 @@ class CombineGlobally(PTransform):
     self.kwargs = kwargs
 
   def display_data(self):
-    return {'combineFn':
+    return {'combine_fn':
             DisplayDataItem(self.fn.__class__, label='Combine Function'),
-            'combineFn_dd':
+            'combine_fn_dd':
             self.fn}
 
   def default_label(self):
@@ -924,9 +924,9 @@ class CombinePerKey(PTransformWithSideInputs):
     A PObject holding the result of the combine operation.
   """
   def display_data(self):
-    return {'combineFn':
+    return {'combine_fn':
             DisplayDataItem(self.fn.__class__, label='Combine Function'),
-            'combineFn_dd':
+            'combine_fn_dd':
             self.fn}
 
   def make_fn(self, fn):


Mime
View raw message