beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rober...@apache.org
Subject [2/4] beam git commit: Add a --fix option to the standard coder test that populates encodings.
Date Fri, 20 Jan 2017 22:48:53 GMT
Add a --fix option to the standard coder test that populates encodings.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5c054aef
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5c054aef
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5c054aef

Branch: refs/heads/python-sdk
Commit: 5c054aeffd6b5eba8cb62065dd626d33cae0312a
Parents: 1503682
Author: Robert Bradshaw <robertwb@gmail.com>
Authored: Tue Jan 10 16:33:23 2017 -0800
Committer: Robert Bradshaw <robertwb@gmail.com>
Committed: Fri Jan 20 14:48:07 2017 -0800

----------------------------------------------------------------------
 sdks/python/apache_beam/coders/coder_impl.py    |  3 +
 .../apache_beam/coders/standard_coders.yaml     | 28 +++++++
 .../apache_beam/coders/standard_coders_test.py  | 78 +++++++++++++-------
 3 files changed, 84 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/5c054aef/sdks/python/apache_beam/coders/coder_impl.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py
index fcdc441..96d0a3e 100644
--- a/sdks/python/apache_beam/coders/coder_impl.py
+++ b/sdks/python/apache_beam/coders/coder_impl.py
@@ -37,6 +37,9 @@ try:
   from stream import OutputStream as create_OutputStream
   from stream import ByteCountingOutputStream
   from stream import get_varint_size
+  globals()['create_InputStream'] = create_InputStream
+  globals()['create_OutputStream'] = create_OutputStream
+  globals()['ByteCountingOutputStream'] = ByteCountingOutputStream
 except ImportError:
   from slow_stream import InputStream as create_InputStream
   from slow_stream import OutputStream as create_OutputStream

http://git-wip-us.apache.org/repos/asf/beam/blob/5c054aef/sdks/python/apache_beam/coders/standard_coders.yaml
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/standard_coders.yaml b/sdks/python/apache_beam/coders/standard_coders.yaml
index 77db379..3193883 100644
--- a/sdks/python/apache_beam/coders/standard_coders.yaml
+++ b/sdks/python/apache_beam/coders/standard_coders.yaml
@@ -1,3 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
 coder:
   urn: "beam:coders:bytes:0.1"
 nested: false
@@ -23,3 +41,13 @@ coder:
 examples:
   "\u0003abc\0": {key: abc, value: 0}
   "\u0004ab\0c\u000A": {key: "ab\0c", value: 10}
+
+---
+
+coder:
+  urn: "beam:coders:kv:0.1"
+  components: [{urn: "beam:coders:bytes:0.1"},
+               {urn: "beam:coders:bytes:0.1"}]
+examples:
+  "\u0003abc\u0003def": {key: abc, value: def}
+  "\u0004ab\0c\u0004de\0f": {key: "ab\0c", value: "de\0f"}

http://git-wip-us.apache.org/repos/asf/beam/blob/5c054aef/sdks/python/apache_beam/coders/standard_coders_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/standard_coders_test.py b/sdks/python/apache_beam/coders/standard_coders_test.py
index 18dc0a7..d451132 100644
--- a/sdks/python/apache_beam/coders/standard_coders_test.py
+++ b/sdks/python/apache_beam/coders/standard_coders_test.py
@@ -15,17 +15,16 @@
 # limitations under the License.
 #
 
-"""Unit tests for coders that must be consistent across all beam SDKs.
+"""Unit tests for coders that must be consistent across all Beam SDKs.
 """
 
-import collections
+import json
 import os.path
-import pickle
+import sys
 import unittest
 
 import yaml
 
-import apache_beam as beam
 from apache_beam import coders
 from apache_beam.coders import coder_impl
 
@@ -33,17 +32,17 @@ from apache_beam.coders import coder_impl
 class StandardCodersTest(unittest.TestCase):
 
   _urn_to_coder_class = {
-    'beam:coders:bytes:0.1': coders.BytesCoder,
-    'beam:coders:varint:0.1': coders.VarIntCoder,
-    'beam:coders:kv:0.1': lambda k, v: coders.TupleCoder((k, v))
+      'beam:coders:bytes:0.1': coders.BytesCoder,
+      'beam:coders:varint:0.1': coders.VarIntCoder,
+      'beam:coders:kv:0.1': lambda k, v: coders.TupleCoder((k, v))
   }
 
   _urn_to_json_value_parser = {
-    'beam:coders:bytes:0.1': lambda x: x,
-    'beam:coders:varint:0.1': lambda x: x,
-    'beam:coders:kv:0.1':
-        lambda x, key_parser, value_parser: (key_parser(x['key']),
-                                             value_parser(x['value']))
+      'beam:coders:bytes:0.1': lambda x: x,
+      'beam:coders:varint:0.1': lambda x: x,
+      'beam:coders:kv:0.1':
+          lambda x, key_parser, value_parser: (key_parser(x['key']),
+                                               value_parser(x['value']))
   }
 
   # We must prepend an underscore to this name so that the open-source unittest
@@ -62,20 +61,25 @@ class StandardCodersTest(unittest.TestCase):
   # runner does not execute this method directly as a test.
   @classmethod
   def _create_tests(cls, coder_test_specs):
-    for spec in yaml.load_all(open(coder_test_specs)):
+    for ix, spec in enumerate(yaml.load_all(open(coder_test_specs))):
+      spec['index'] = ix
       cls._create_test(spec)
 
   def _run_coder_test(self, spec):
     coder = self.parse_coder(spec['coder'])
     parse_value = self.json_value_parser(spec['coder'])
-    for encoded, json_value in spec['examples'].items():
-      value = parse_value(json_value)
-      if spec.get('nested', True):
-        self.assertEqual(decode_nested(coder, encoded), value)
-        self.assertEqual(encoded, encode_nested(coder, value))
-      if not spec.get('nested', False):
-        self.assertEqual(coder.decode(encoded), value)
-        self.assertEqual(encoded, coder.encode(value))
+    nested_list = [spec['nested']] if 'nested' in spec else [True, False]
+    for nested in nested_list:
+      for expected_encoded, json_value in spec['examples'].items():
+        value = parse_value(json_value)
+        expected_encoded = expected_encoded.encode('latin1')
+        actual_encoded = encode_nested(coder, value, nested)
+        if self.fix and actual_encoded != expected_encoded:
+          self.to_fix[spec['index'], expected_encoded] = actual_encoded
+        else:
+          self.assertEqual(decode_nested(coder, expected_encoded, nested),
+                           value)
+          self.assertEqual(expected_encoded, actual_encoded)
 
   def parse_coder(self, spec):
     return self._urn_to_coder_class[spec['urn']](
@@ -87,6 +91,26 @@ class StandardCodersTest(unittest.TestCase):
     return lambda x: self._urn_to_json_value_parser[coder_spec['urn']](
         x, *component_parsers)
 
+  # Used when --fix is passed.
+
+  fix = False
+  to_fix = {}
+
+  @classmethod
+  def tearDownClass(cls):
+    if cls.fix and cls.to_fix:
+      print "FIXING", len(cls.to_fix), "TESTS"
+      doc_sep = '\n---\n'
+      docs = open(STANDARD_CODERS_YAML).read().split(doc_sep)
+
+      def quote(s):
+        return json.dumps(s.decode('latin1')).replace(r'\u0000', r'\0')
+      for (doc_ix, expected_encoded), actual_encoded in cls.to_fix.items():
+        print quote(expected_encoded), "->", quote(actual_encoded)
+        docs[doc_ix] = docs[doc_ix].replace(
+            quote(expected_encoded) + ':', quote(actual_encoded) + ':')
+      open(STANDARD_CODERS_YAML, 'w').write(doc_sep.join(docs))
+
 
 STANDARD_CODERS_YAML = os.path.join(
     os.path.dirname(__file__), 'standard_coders.yaml')
@@ -94,15 +118,19 @@ if os.path.exists(STANDARD_CODERS_YAML):
   StandardCodersTest._create_tests(STANDARD_CODERS_YAML)
 
 
-def encode_nested(coder, value):
+def encode_nested(coder, value, nested=True):
   out = coder_impl.create_OutputStream()
-  coder.get_impl().encode_to_stream(value, out, True)
+  coder.get_impl().encode_to_stream(value, out, nested)
   return out.get()
 
-def decode_nested(coder, encoded):
+
+def decode_nested(coder, encoded, nested=True):
   return coder.get_impl().decode_from_stream(
-      coder_impl.create_InputStream(encoded), True)
+      coder_impl.create_InputStream(encoded), nested)
 
 
 if __name__ == '__main__':
+  if '--fix' in sys.argv:
+    StandardCodersTest.fix = True
+    sys.argv.remove('--fix')
   unittest.main()


Mime
View raw message