beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhalp...@apache.org
Subject [2/2] incubator-beam git commit: Pylint integration for Python SDK
Date Mon, 27 Jun 2016 19:30:20 GMT
Pylint integration for Python SDK

Runs pylint on modified files under sdks/python/**/*.py.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f71c1ddd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f71c1ddd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f71c1ddd

Branch: refs/heads/python-sdk
Commit: f71c1dddb10d4a8b0f8c8ed8722e6662781d19f9
Parents: 21c819f
Author: Ahmet Altay <altay@google.com>
Authored: Fri Jun 24 14:04:27 2016 -0700
Committer: Dan Halperin <dhalperi@google.com>
Committed: Mon Jun 27 12:30:02 2016 -0700

----------------------------------------------------------------------
 sdks/python/.pylintrc                           | 174 +++++++++++++++++++
 sdks/python/apache_beam/coders/coders.py        |   2 +-
 sdks/python/apache_beam/coders/stream_test.py   |   2 +-
 .../apache_beam/examples/streaming_wordcount.py |   2 +-
 sdks/python/apache_beam/examples/wordcount.py   |   4 +-
 .../apache_beam/examples/wordcount_debugging.py |   6 +-
 sdks/python/apache_beam/io/bigquery.py          |   2 +-
 .../apache_beam/transforms/combiners_test.py    |   7 +-
 sdks/python/apache_beam/transforms/core.py      |  17 +-
 .../apache_beam/transforms/ptransform_test.py   |  43 +++--
 .../apache_beam/transforms/trigger_test.py      |  25 +--
 .../apache_beam/typehints/trivial_inference.py  |   3 +-
 sdks/python/apache_beam/typehints/typehints.py  |   5 +-
 sdks/python/run_pylint.sh                       |  48 +++++
 sdks/python/tox.ini                             |   2 +
 15 files changed, 292 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/.pylintrc
----------------------------------------------------------------------
diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc
new file mode 100644
index 0000000..edf13ee
--- /dev/null
+++ b/sdks/python/.pylintrc
@@ -0,0 +1,174 @@
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+[MASTER]
+# Ignore auto-generated files.
+ignore=clients,windmill_pb2.py,windmill_service_pb2.py
+
+[BASIC]
+# Regular expression which should only match the name
+# of functions or classes which do not require a docstring.
+no-docstring-rgx=(__.*__|main)
+
+# Min length in lines of a function that requires a docstring.
+docstring-min-length=10
+
+# Regular expression which should only match correct module names. The
+# leading underscore is sanctioned for private modules by Google's style
+# guide.
+#
+# There are exceptions to the basic rule (_?[a-z][a-z0-9_]*) to cover
+# requirements of Python's module system and of the presubmit framework.
+module-rgx=^(_?[a-z][a-z0-9_]*)|__init__|PRESUBMIT|PRESUBMIT_unittest$
+
+# Regular expression which should only match correct module level names
+const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
+
+# Regular expression which should only match correct class attribute
+class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
+
+# Regular expression which should only match correct class names
+class-rgx=^_?[A-Z][a-zA-Z0-9]*$
+
+# Regular expression which should only match correct function names.
+# 'camel_case' and 'snake_case' group names are used for consistency of naming
+# styles across functions and methods.
+function-rgx=^(?:(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$
+
+# Regular expression which should only match correct method names.
+# 'camel_case' and 'snake_case' group names are used for consistency of naming
+# styles across functions and methods. 'exempt' indicates a name which is
+# consistent with all naming styles.
+method-rgx=^(?:(?P<exempt>__[a-z0-9_]+__|next)|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$
+
+# Regular expression which should only match correct instance attribute names
+attr-rgx=^_{0,2}[a-z][a-z0-9_]*$
+
+# Regular expression which should only match correct argument names
+argument-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression which should only match correct variable names
+variable-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression which should only match correct list comprehension /
+# generator expression variable names
+inlinevar-rgx=^[a-z][a-z0-9_]*$
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=input,apply,reduce
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=main,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=
+
+[MESSAGES CONTROL]
+disable =
+  abstract-method,
+  arguments-differ,
+  attribute-defined-outside-init,
+  bad-builtin,
+  bad-option-value,
+  bad-super-call,
+  broad-except,
+  consider-using-enumerate,
+  cyclic-import,
+  design,
+  expression-not-assigned,
+  fixme,
+  function-redefined,
+  global-statement,
+  import-error,
+  import-self,
+  invalid-name,
+  locally-disabled,
+  locally-enabled,
+  misplaced-bare-raise,
+  missing-docstring,
+  multiple-statements,
+  no-member,
+  no-name-in-module,
+  no-self-argument,
+  no-self-use,
+  no-value-for-parameter,
+  not-callable,
+  pointless-statement,
+  protected-access,
+  raising-non-exception,
+  redefined-builtin,
+  redefined-outer-name,
+  redefined-variable-type,
+  redundant-keyword-arg,
+  reimported,
+  relative-import,
+  similarities,
+  simplifiable-if-statement,
+  super-init-not-called,
+  undefined-variable,
+  unexpected-keyword-arg,
+  ungrouped-imports,
+  unidiomatic-typecheck,
+  unnecessary-lambda,
+  unneeded-not,
+  unused-argument,
+  unused-import,
+  unused-variable,
+  unused-wildcard-import,
+  used-before-assignment,
+  wildcard-import,
+  wrong-import-order,
+  wrong-import-position,
+
+
+[REPORTS]
+# Tells whether to display a full report or only the messages
+reports=no
+
+[CLASSES]
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# "class_" is also a valid for the first argument to a class method.
+valid-classmethod-first-arg=cls,class_
+
+[FORMAT]
+# Maximum number of characters on a single line.
+max-line-length=80
+
+# Maximum number of lines in a module
+max-module-lines=99999
+
+# String used as indentation unit. (2 spaces.)
+indent-string='  '
+
+# Number of spaces of indent required.
+indent-after-paren=4
+
+# Regexp for a line that is allowed to be longer than the limit.
+# Long import lines or URLs in comments or pydocs. 
+ignore-long-lines=(?x)
+  (^\s*(import|from)\s
+   |^\s*(\#\ )?<?(https?|ftp):\/\/[^\s\/$.?#].[^\s]*>?$
+   )
+
+[VARIABLES]
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching names used for dummy variables (i.e. not used).
+dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_|args|kwargs)

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/coders/coders.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py
index 24a1f3e..6d5b10a 100644
--- a/sdks/python/apache_beam/coders/coders.py
+++ b/sdks/python/apache_beam/coders/coders.py
@@ -152,7 +152,7 @@ class Coder(object):
   def __eq__(self, other):
     # pylint: disable=protected-access
     return (self.__class__ == other.__class__
-       and self._dict_without_impl() == other._dict_without_impl())
+            and self._dict_without_impl() == other._dict_without_impl())
     # pylint: enable=protected-access
 
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/coders/stream_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/stream_test.py b/sdks/python/apache_beam/coders/stream_test.py
index 463ccf7..f2163ca 100644
--- a/sdks/python/apache_beam/coders/stream_test.py
+++ b/sdks/python/apache_beam/coders/stream_test.py
@@ -83,7 +83,7 @@ class StreamTest(unittest.TestCase):
     base = -1.7
     self.run_read_write_var_int64(
         [int(base**pow)
-          for pow in range(1, int(63 * math.log(2) / math.log(-base)))])
+         for pow in range(1, int(63 * math.log(2) / math.log(-base)))])
 
   def test_large_var_int64(self):
     self.run_read_write_var_int64([0, 2**63 - 1, -2**63, 2**63 - 3])

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/examples/streaming_wordcount.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/examples/streaming_wordcount.py b/sdks/python/apache_beam/examples/streaming_wordcount.py
index 685e742..eda74dd 100644
--- a/sdks/python/apache_beam/examples/streaming_wordcount.py
+++ b/sdks/python/apache_beam/examples/streaming_wordcount.py
@@ -53,7 +53,7 @@ def run(argv=None):
   # Capitalize the characters in each line.
   transformed = (lines
                  | (beam.FlatMap('split',
-                               lambda x: re.findall(r'[A-Za-z\']+', x))
+                                 lambda x: re.findall(r'[A-Za-z\']+', x))
                     .with_output_types(unicode))
                  | beam.Map('pair_with_one', lambda x: (x, 1))
                  | beam.WindowInto(window.FixedWindows(15, 0))

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/examples/wordcount.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/examples/wordcount.py b/sdks/python/apache_beam/examples/wordcount.py
index b369294..fc02e91 100644
--- a/sdks/python/apache_beam/examples/wordcount.py
+++ b/sdks/python/apache_beam/examples/wordcount.py
@@ -28,8 +28,8 @@ import apache_beam as beam
 
 empty_line_aggregator = beam.Aggregator('emptyLines')
 average_word_size_aggregator = beam.Aggregator('averageWordLength',
-                                             beam.combiners.MeanCombineFn(),
-                                             float)
+                                               beam.combiners.MeanCombineFn(),
+                                               float)
 
 
 class WordExtractingDoFn(beam.DoFn):

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/examples/wordcount_debugging.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/examples/wordcount_debugging.py b/sdks/python/apache_beam/examples/wordcount_debugging.py
index c06ab04..c989b6a 100644
--- a/sdks/python/apache_beam/examples/wordcount_debugging.py
+++ b/sdks/python/apache_beam/examples/wordcount_debugging.py
@@ -122,7 +122,8 @@ def run(argv=None):
   # each word and filter by a list of words.
   filtered_words = (
       p | beam.io.Read('read', beam.io.TextFileSource(known_args.input))
-      | CountWords() | beam.ParDo('FilterText', FilterTextFn('Flourish|stomach')))
+      | CountWords()
+      | beam.ParDo('FilterText', FilterTextFn('Flourish|stomach')))
 
   # assert_that is a convenient PTransform that checks a PCollection has an
   # expected value. Asserts are best used in unit tests with small data sets but
@@ -132,7 +133,8 @@ def run(argv=None):
   # of the Pipeline implies that the expectations were  met. Learn more at
   # https://cloud.google.com/dataflow/pipelines/testing-your-pipeline on how to
   # test your pipeline.
-  beam.assert_that(filtered_words, beam.equal_to([('Flourish', 3), ('stomach', 1)]))
+  beam.assert_that(
+      filtered_words, beam.equal_to([('Flourish', 3), ('stomach', 1)]))
 
   # Format the counts into a PCollection of strings and write the output using a
   # "Write" transform that has side effects.

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/io/bigquery.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/bigquery.py b/sdks/python/apache_beam/io/bigquery.py
index 030f821..9d33134 100644
--- a/sdks/python/apache_beam/io/bigquery.py
+++ b/sdks/python/apache_beam/io/bigquery.py
@@ -46,7 +46,7 @@ of the side table. The execution framework may use some caching techniques
to
 share the side inputs between calls in order to avoid excessive reading::
 
   main_table = pipeline | beam.io.Read(beam.io.BigQuerySource('very_big_table')
-  side_table = pipeline | beam.io.Read(beam.io.BigQuerySource('not_so_big_table')
+  side_table = pipeline | beam.io.Read(beam.io.BigQuerySource('not_big_table')
   results = (
       main_table
       | beam.Map('process data',

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/transforms/combiners_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py
index b753a17..112c591 100644
--- a/sdks/python/apache_beam/transforms/combiners_test.py
+++ b/sdks/python/apache_beam/transforms/combiners_test.py
@@ -109,7 +109,8 @@ class CombineTest(unittest.TestCase):
     pcoll = pipeline | Create(
         'start-perkey', [('a', x) for x in [6, 3, 1, 1, 9, 1, 5, 2, 0, 6]])
     result_ktop = pcoll | beam.CombinePerKey('top-perkey', combiners.Largest(5))
-    result_kbot = pcoll | beam.CombinePerKey('bot-perkey', combiners.Smallest(4))
+    result_kbot = pcoll | beam.CombinePerKey(
+        'bot-perkey', combiners.Smallest(4))
     assert_that(result_ktop, equal_to([('a', [9, 6, 6, 5, 3])]), label='k:top')
     assert_that(result_kbot, equal_to([('a', [0, 1, 1, 1])]), label='k:bot')
     pipeline.run()
@@ -158,8 +159,8 @@ class CombineTest(unittest.TestCase):
         p
         | Create([('a', 100, 0.0), ('b', 10, -1), ('c', 1, 100)])
         | beam.CombineGlobally(combine.TupleCombineFn(max,
-                                                    combine.MeanCombineFn(),
-                                                    sum)).without_defaults())
+                                                      combine.MeanCombineFn(),
+                                                      sum)).without_defaults())
     assert_that(result, equal_to([('c', 111.0 / 3, 99.0)]))
     p.run()
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/transforms/core.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index 3a0eb30..445367f 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -437,7 +437,7 @@ class CallableWrapperCombineFn(CombineFn):
             'All functions for a Combine PTransform must accept a '
             'single argument compatible with: Iterable[Any]. '
             'Instead a function with input type: %s was received.'
-             % input_args[0])
+            % input_args[0])
       input_args = (element_type(input_args[0]),) + input_args[1:]
       # TODO(robertwb): Assert output type is consistent with input type?
       hints = fn_hints.copy()
@@ -813,10 +813,12 @@ class CombineGlobally(PTransform):
         return transform
 
     combined = (pcoll
-            | add_input_types(Map('KeyWithVoid', lambda v: (None, v))
-               .with_output_types(KV[None, pcoll.element_type]))
-            | CombinePerKey('CombinePerKey', self.fn, *self.args, **self.kwargs)
-            | Map('UnKey', lambda (k, v): v))
+                | add_input_types(Map('KeyWithVoid', lambda v: (None, v))
+                                  .with_output_types(
+                                      KV[None, pcoll.element_type]))
+                | CombinePerKey(
+                    'CombinePerKey', self.fn, *self.args, **self.kwargs)
+                | Map('UnKey', lambda (k, v): v))
 
     if not self.has_defaults and not self.as_view:
       return combined
@@ -884,7 +886,7 @@ class CombineValues(PTransformWithSideInputs):
 
   def apply(self, pcoll):
     args, kwargs = util.insert_values_in_args(
-      self.args, self.kwargs, self.side_inputs)
+        self.args, self.kwargs, self.side_inputs)
 
     input_type = pcoll.element_type
     key_type = None
@@ -1027,7 +1029,8 @@ class GroupByKey(PTransform):
           'GroupByKey operation "%s"' % self.label)
 
       reify_output_type = KV[key_type, typehints.WindowedValue[value_type]]
-      gbk_input_type = KV[key_type, Iterable[typehints.WindowedValue[value_type]]]
+      gbk_input_type = (
+          KV[key_type, Iterable[typehints.WindowedValue[value_type]]])
       gbk_output_type = KV[key_type, Iterable[value_type]]
 
       return (pcoll

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/transforms/ptransform_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index 3244473..8ae7a37 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -286,7 +286,7 @@ class PTransformTest(unittest.TestCase):
     vals_2 = [2, 4, 6, 8, 10, 12, 14]
     pipeline = Pipeline('DirectPipelineRunner')
     pcoll = pipeline | beam.Create('start', ([('a', x) for x in vals_1] +
-                                           [('b', x) for x in vals_2]))
+                                             [('b', x) for x in vals_2]))
     result = pcoll | beam.CombinePerKey('mean', self._MeanCombineFn())
     assert_that(result, equal_to([('a', sum(vals_1) / len(vals_1)),
                                   ('b', sum(vals_2) / len(vals_2))]))
@@ -297,7 +297,7 @@ class PTransformTest(unittest.TestCase):
     vals_2 = [2, 4, 6, 8, 10, 12, 14]
     pipeline = Pipeline('DirectPipelineRunner')
     pcoll = pipeline | beam.Create('start', ([('a', x) for x in vals_1] +
-                                           [('b', x) for x in vals_2]))
+                                             [('b', x) for x in vals_2]))
     result = pcoll | beam.CombinePerKey(sum)
     assert_that(result, equal_to([('a', sum(vals_1)), ('b', sum(vals_2))]))
     pipeline.run()
@@ -307,7 +307,7 @@ class PTransformTest(unittest.TestCase):
     vals_2 = [2, 4, 6, 8, 10, 12, 14]
     pipeline = Pipeline('DirectPipelineRunner')
     pcoll = pipeline | beam.Create('start', ([('a', x) for x in vals_1] +
-                                           [('b', x) for x in vals_2]))
+                                             [('b', x) for x in vals_2]))
     divisor = pipeline | beam.Create('divisor', [2])
     result = pcoll | beam.CombinePerKey(
         lambda vals, d: max(v for v in vals if v % d == 0),
@@ -790,9 +790,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
     d = (self.p
          | beam.Create('s', ['t', 'e', 's', 't']).with_output_types(str)
          | beam.FlatMap('dup', lambda x: [x + x])
-            .with_input_types(str).with_output_types(str)
+         .with_input_types(str).with_output_types(str)
          | beam.FlatMap('upper', lambda x: [x.upper()])
-            .with_input_types(str).with_output_types(str))
+         .with_input_types(str).with_output_types(str))
 
     assert_that(d, equal_to(['TT', 'EE', 'SS', 'TT']))
     self.p.run()
@@ -803,7 +803,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
     with self.assertRaises(typehints.TypeCheckError) as e:
       d = (self.p
            | beam.Create('s', [1, 2, 3, 4]).with_output_types(int)
-           | beam.Map('upper', lambda x: x.upper()).with_input_types(str).with_output_types(str))
+           | beam.Map('upper', lambda x: x.upper())
+           .with_input_types(str).with_output_types(str))
 
     self.assertEqual("Type hint violation for 'upper': "
                      "requires <type 'str'> but got <type 'int'> for x",
@@ -813,7 +814,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
     # No error should be raised if this type-checks properly.
     d = (self.p
          | beam.Create('s', [1, 2, 3, 4]).with_output_types(int)
-         | beam.Map('to_str', lambda x: str(x)).with_input_types(int).with_output_types(str))
+         | beam.Map('to_str', lambda x: str(x))
+         .with_input_types(int).with_output_types(str))
     assert_that(d, equal_to(['1', '2', '3', '4']))
     self.p.run()
 
@@ -853,7 +855,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
     with self.assertRaises(typehints.TypeCheckError) as e:
       (self.p
        | beam.Create('strs', ['1', '2', '3', '4', '5']).with_output_types(str)
-       | beam.Map('lower', lambda x: x.lower()).with_input_types(str).with_output_types(str)
+       | beam.Map('lower', lambda x: x.lower())
+       .with_input_types(str).with_output_types(str)
        | beam.Filter('below 3', lambda x: x < 3).with_input_types(int))
 
     self.assertEqual("Type hint violation for 'below 3': "
@@ -864,7 +867,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
     # No error should be raised if this type-checks properly.
     d = (self.p
          | beam.Create('strs', ['1', '2', '3', '4', '5']).with_output_types(str)
-         | beam.Map('to int', lambda x: int(x)).with_input_types(str).with_output_types(int)
+         | beam.Map('to int', lambda x: int(x))
+         .with_input_types(str).with_output_types(int)
          | beam.Filter('below 3', lambda x: x < 3).with_input_types(int))
     assert_that(d, equal_to([1, 2]))
     self.p.run()
@@ -894,7 +898,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
     (self.p
      | beam.Create('str', range(5)).with_output_types(int)
      | beam.Filter('half', half)
-     | beam.Map('to bool', lambda x: bool(x)).with_input_types(int).with_output_types(bool))
+     | beam.Map('to bool', lambda x: bool(x))
+     .with_input_types(int).with_output_types(bool))
 
   def test_group_by_key_only_output_type_deduction(self):
     d = (self.p
@@ -982,7 +987,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
     # checking was disabled above.
     (self.p
      | beam.Create('t', [1, 2, 3]).with_output_types(int)
-     | beam.Map('lower', lambda x: x.lower()).with_input_types(str).with_output_types(str))
+     | beam.Map('lower', lambda x: x.lower())
+     .with_input_types(str).with_output_types(str))
 
   def test_run_time_type_checking_enabled_type_violation(self):
     self.p.options.view_as(TypeOptions).pipeline_type_check = False
@@ -1020,7 +1026,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
     # Pipeline checking is off, but the above function should satisfy types at
     # run-time.
     result = (self.p
-              | beam.Create('t', ['t', 'e', 's', 't', 'i', 'n', 'g']).with_output_types(str)
+              | beam.Create('t', ['t', 'e', 's', 't', 'i', 'n', 'g'])
+              .with_output_types(str)
               | beam.Map('gen keys', group_with_upper_ord)
               | beam.GroupByKey('O'))
 
@@ -1232,9 +1239,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
 
     self.assertEqual(
         "All functions for a Combine PTransform must accept a "
-         "single argument compatible with: Iterable[Any]. "
-         "Instead a function with input type: <type 'int'> was received.",
-         e.exception.message)
+        "single argument compatible with: Iterable[Any]. "
+        "Instead a function with input type: <type 'int'> was received.",
+        e.exception.message)
 
   def test_combine_pipeline_type_propagation_using_decorators(self):
     @with_output_types(int)
@@ -1532,7 +1539,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
 
     d = (self.p
          | beam.Create('c', ['t', 'e', 's', 't']).with_output_types(str)
-         | beam.Map('dup key', lambda x: (x, x)).with_output_types(typehints.KV[str, str])
+         | beam.Map('dup key', lambda x: (x, x))
+         .with_output_types(typehints.KV[str, str])
          | combine.Count.PerKey('count dups'))
 
     self.assertCompatible(typehints.KV[str, int], d.element_type)
@@ -1565,7 +1573,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
     self.p.options.view_as(TypeOptions).runtime_type_check = True
 
     d = (self.p
-         | beam.Create('w', [True, True, False, True, True]).with_output_types(bool)
+         | beam.Create('w', [True, True, False, True, True])
+         .with_output_types(bool)
          | combine.Count.PerElement('count elems'))
 
     self.assertCompatible(typehints.KV[bool, int], d.element_type)

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/transforms/trigger_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py
index 5d4d7a8..fee9b1b 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -175,17 +175,18 @@ class TriggerTest(unittest.TestCase):
                        late=AfterCount(1)),
         AccumulationMode.ACCUMULATING,
         [(1, 'a'), (15, 'b'), (7, 'c'), (30, 'd')],
-        {IntervalWindow(1, 25): [
-            set('abc'),                # early
-            set('abc'),                # on time
-            set('abcxy')               # late
-         ],
-         IntervalWindow(30, 40): [
-             set('d'),                  # on time
-         ],
-         IntervalWindow(1, 40): [
-             set('abcdxyz')             # late
-         ],
+        {
+            IntervalWindow(1, 25): [
+                set('abc'),                # early
+                set('abc'),                # on time
+                set('abcxy')               # late
+            ],
+            IntervalWindow(30, 40): [
+                set('d'),                  # on time
+            ],
+            IntervalWindow(1, 40): [
+                set('abcdxyz')             # late
+            ],
         },
         2,
         late_data=[(1, 'x'), (2, 'y'), (21, 'z')])
@@ -374,7 +375,7 @@ class TriggerPipelineTest(unittest.TestCase):
               | beam.FlatMap(lambda t: [('A', t), ('B', t + 5)])
               | beam.Map(lambda (k, t): TimestampedValue((k, t), t))
               | beam.WindowInto(FixedWindows(10), trigger=AfterCount(3),
-                              accumulation_mode=AccumulationMode.DISCARDING)
+                                accumulation_mode=AccumulationMode.DISCARDING)
               | beam.GroupByKey()
               | beam.Map(lambda (k, v): ('%s-%s' % (k, len(v)), set(v))))
     assert_that(result, equal_to(

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/typehints/trivial_inference.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py
index 55d580d..82ce765 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference.py
@@ -237,7 +237,8 @@ def infer_return_type(c, input_types, debug=False, depth=5):
       return infer_return_type_func(c.im_func, input_types, debug, depth)
     elif isinstance(c, BoundMethod):
       input_types = [c.unbound.im_class] + input_types
-      return infer_return_type_func(c.unbound.im_func, input_types, debug, depth)
+      return infer_return_type_func(
+          c.unbound.im_func, input_types, debug, depth)
     elif isinstance(c, (type, types.ClassType)):
       if c in typehints.DISALLOWED_PRIMITIVE_TYPES:
         return {

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/typehints/typehints.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py
index c109dc6..5e31fd1 100644
--- a/sdks/python/apache_beam/typehints/typehints.py
+++ b/sdks/python/apache_beam/typehints/typehints.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-"""Syntax and semantics for type-hinting custom-functions/PTransforms in the SDK.
+"""Syntax & semantics for type-hinting custom-functions/PTransforms in the SDK.
 
 This module defines type-hinting objects and the corresponding syntax for
 type-hinting function arguments, function return types, or PTransform object
@@ -322,7 +322,8 @@ def _unified_repr(o):
   Returns:
     A qualified name for the passed Python object fit for string formatting.
   """
-  return repr(o) if isinstance(o, (TypeConstraint, types.NoneType)) else o.__name__
+  return repr(o) if isinstance(
+      o, (TypeConstraint, types.NoneType)) else o.__name__
 
 
 def check_constraint(type_constraint, object_instance):

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/run_pylint.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh
new file mode 100755
index 0000000..e8062c0
--- /dev/null
+++ b/sdks/python/run_pylint.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# This script will run pylint on files that changed compared to the current
+# HEAD of the branch.
+#
+# Use "pylint apache_beam" to run pylint all files.
+#
+# The exit-code of the script indicates success or a failure.
+
+BASE_BRANCH=python-sdk
+
+set -e
+set -o pipefail
+
+# Retrieve base branch for comparison. Travis does not fetch it by default.
+git remote set-branches --add origin $BASE_BRANCH
+git fetch
+
+# Get the name of the files that changed compared to the HEAD of the branch.
+# Filter the output to .py files only. Rewrite the paths relative to the
+# sdks/python folder.
+CHANGED_FILES=$(git diff --name-only origin/$BASE_BRANCH apache_beam \
+                | { grep ".py$" || true; }  \
+                | sed 's/sdks\/python\///g')
+
+if test "$CHANGED_FILES"; then
+  echo "Running pylint on changed files:"
+  echo "$CHANGED_FILES"
+  pylint $CHANGED_FILES
+else
+  echo "Not running pylint. No eligible files."
+fi

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/tox.ini
----------------------------------------------------------------------
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index c0208b5..356de57 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -19,6 +19,8 @@
 envlist = py27
 
 [testenv:py27]
+deps=pylint
 commands =
   python setup.py test
+  {toxinidir}/run_pylint.sh
 passenv = TRAVIS*


Mime
View raw message