beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (BEAM-3143) Fix type inference in Python 3 for generators
Date Fri, 01 Dec 2017 00:28:00 GMT

    [ https://issues.apache.org/jira/browse/BEAM-3143?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16273721#comment-16273721
] 

ASF GitHub Bot commented on BEAM-3143:
--------------------------------------

holdenk closed pull request #4079: [BEAM-3143][WIP] Py3 type inferance
URL: https://github.com/apache/beam/pull/4079
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py
index dcca6d02644..76dc25c41c3 100644
--- a/sdks/python/apache_beam/typehints/opcodes.py
+++ b/sdks/python/apache_beam/typehints/opcodes.py
@@ -30,6 +30,7 @@
 
 import types
 from functools import reduce
+import inspect
 
 from . import typehints
 from .trivial_inference import BoundMethod
@@ -265,9 +266,12 @@ def load_attr(state, arg):
   name = state.get_name(arg)
   if isinstance(o, Const) and hasattr(o.value, name):
     state.stack.append(Const(getattr(o.value, name)))
-  elif (isinstance(o, type)
-        and isinstance(getattr(o, name, None), types.MethodType)):
-    state.stack.append(Const(BoundMethod(getattr(o, name))))
+  elif ((isinstance(o, type) or inspect.isclass(o))):
+    elem = getattr(o, name, None)
+    if callable(elem):
+      state.stack.append(Const(BoundMethod(elem)))
+    else:
+      state.stack.append(Any)
   else:
     state.stack.append(Any)
 
diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py
index a68bd18b1c3..ce3b9647602 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference.py
@@ -21,8 +21,13 @@
 """
 from __future__ import absolute_import
 from __future__ import print_function
+from __future__ import division
 
-import __builtin__
+from builtins import zip
+from builtins import str
+from past.utils import old_div
+from builtins import object
+import builtins
 import collections
 import dis
 import pprint
@@ -133,8 +138,8 @@ def get_global(self, i):
     name = self.get_name(i)
     if name in self.f.__globals__:
       return Const(self.f.__globals__[name])
-    if name in __builtin__.__dict__:
-      return Const(__builtin__.__dict__[name])
+    if name in builtins.__dict__:
+      return Const(builtins.__dict__[name])
     return Any
 
   def get_name(self, i):
@@ -238,9 +243,13 @@ def infer_return_type(c, input_types, debug=False, depth=5):
         input_types = [Const(c.__self__)] + input_types
       return infer_return_type_func(c.__func__, input_types, debug, depth)
     elif isinstance(c, BoundMethod):
-      input_types = [c.unbound.__self__.__class__] + input_types
+      unbound = c.unbound
+      # Python 2/3 compatibility.
+      method_class = getattr(c.unbound, "__self__", c.unbound).__class__
+      func = getattr(c.unbound, "__func__", c.unbound)
+      input_types = [method_class] + input_types
       return infer_return_type_func(
-          c.unbound.__func__, input_types, debug, depth)
+          func, input_types, debug, depth)
     elif isinstance(c, type):
       if c in typehints.DISALLOWED_PRIMITIVE_TYPES:
         return {
@@ -294,8 +303,13 @@ def infer_return_type_func(f, input_types, debug=False, depth=0):
   yields = set()
   returns = set()
   # TODO(robertwb): Default args via inspect module.
-  local_vars = list(input_types) + [typehints.Union[()]] * (len(co.co_varnames)
-                                                            - len(input_types))
+  try:
+    input_types_list = list(input_types)
+  except TypeError:
+    input_types_list = [input_types]
+  typehints_union = [typehints.Union[()]]
+  target_length = len(co.co_varnames) - len(input_types_list)
+  local_vars = input_types_list + typehints_union * target_length
   state = FrameState(f, local_vars)
   states = collections.defaultdict(lambda: None)
   jumps = collections.defaultdict(int)
@@ -303,7 +317,10 @@ def infer_return_type_func(f, input_types, debug=False, depth=0):
   last_pc = -1
   while pc < end:
     start = pc
-    op = ord(code[pc])
+    if sys.version_info[0] >= 3:
+      op = code[pc]
+    else:
+      op = ord(code[pc])
 
     if debug:
       print('-->' if pc == last_pc else '    ', end=' ')
@@ -311,7 +328,10 @@ def infer_return_type_func(f, input_types, debug=False, depth=0):
       print(dis.opname[op].ljust(20), end=' ')
     pc += 1
     if op >= dis.HAVE_ARGUMENT:
-      arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg
+      if sys.version_info[0] >= 3:
+        arg = code[pc] + code[pc + 1] * 256 + extended_arg
+      else:
+        arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg
       extended_arg = 0
       pc += 2
       if op == dis.EXTENDED_ARG:
@@ -344,7 +364,7 @@ def infer_return_type_func(f, input_types, debug=False, depth=0):
     opname = dis.opname[op]
     jmp = jmp_state = None
     if opname.startswith('CALL_FUNCTION'):
-      standard_args = (arg & 0xF) + (arg & 0xF0) / 8
+      standard_args = (arg & 0xF) + old_div((arg & 0xF0), 8)
       var_args = 'VAR' in opname
       kw_args = 'KW' in opname
       pop_count = standard_args + var_args + kw_args + 1
@@ -353,6 +373,10 @@ def infer_return_type_func(f, input_types, debug=False, depth=0):
       elif arg & 0xF0:
         # TODO(robertwb): Handle this case.
         return_type = Any
+      elif (isinstance(state.stack[-pop_count], Const) and
+            state.stack[-pop_count].value == list):
+        # TODO(robertwb + holden): Handle this better.
+        return_type = typehints.List[element_type(state.stack[1])]
       elif isinstance(state.stack[-pop_count], Const):
         # TODO(robertwb): Handle this better.
         if var_args or kw_args:
@@ -380,7 +404,8 @@ def infer_return_type_func(f, input_types, debug=False, depth=0):
         state.stack.append(Any)
     elif opname in simple_ops:
       if debug:
-        print("Executing simple op " + opname)
+        print("Executing simple %s op with state %s and arg %s" %
+              (opname, state, arg))
       simple_ops[opname](state, arg)
     elif opname == 'RETURN_VALUE':
       returns.add(state.stack[-1])
@@ -408,6 +433,11 @@ def infer_return_type_func(f, input_types, debug=False, depth=0):
       jmp_state = state.copy()
       jmp_state.stack.pop()
       state.stack.append(element_type(state.stack[-1]))
+    elif opname == 'BUILD_LIST':
+      jmp = pc + arg
+      jmp_state = state.copy()
+      jmp_state.stack.pop()
+      state.stack.append(element_type(state.stack[-1]))
     else:
       raise TypeInferenceError('unable to handle %s' % opname)
 
diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py
index 37b22584723..ab031833e40 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference_test.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py
@@ -16,6 +16,7 @@
 #
 
 """Tests for apache_beam.typehints.trivial_inference."""
+import sys
 import unittest
 
 from apache_beam.typehints import trivial_inference
@@ -72,6 +73,14 @@ def func(a):
       return None
     self.assertReturnType(typehints.Union[int, type(None)], func, [int])
 
+  @unittest.skipIf(sys.version_info[0] < 3, "List inference test py3 only")
+  def testSimpleList(self):
+    self.assertReturnType(
+        typehints.List[int],
+        lambda xs: [1, 2],
+        [typehints.Tuple[int, ...]])
+
+
   def testListComprehension(self):
     self.assertReturnType(
         typehints.List[int],
@@ -137,6 +146,17 @@ def m(self, x):
     self.assertReturnType(int, lambda: A().m(3))
     self.assertReturnType(float, lambda: A.m(A(), 3.0))
 
+  def testMethod2(self):
+
+    class A(object):
+
+      def m(self, x, y):
+        return x
+
+    self.assertReturnType(int, lambda: A().m(3, "a"))
+    self.assertReturnType(float, lambda: A.m(A(), 3.0, "e"))
+
+
   def testAlwaysReturnsEarly(self):
 
     def some_fn(v):


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> Fix type inference in Python 3 for generators
> ---------------------------------------------
>
>                 Key: BEAM-3143
>                 URL: https://issues.apache.org/jira/browse/BEAM-3143
>             Project: Beam
>          Issue Type: Sub-task
>          Components: sdk-py-core
>            Reporter: holdenk
>




--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Mime
View raw message