This is an automated email from the ASF dual-hosted git repository. robertwb pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new a424c64 [BEAM-3143] Type Inference Python 3 Compatibility (#4183) a424c64 is described below commit a424c64a3234a178a2a86161ee200c8be79d8f53 Author: Luke Zhu <luke.l....@gmail.com> AuthorDate: Mon Dec 18 14:06:53 2017 -0500 [BEAM-3143] Type Inference Python 3 Compatibility (#4183) --- sdks/python/apache_beam/typehints/opcodes.py | 34 ++++++++++++-- .../apache_beam/typehints/trivial_inference.py | 53 ++++++++++++++-------- .../typehints/trivial_inference_test.py | 18 +++++++- 3 files changed, 79 insertions(+), 26 deletions(-) diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py index dcca6d0..ccf0195 100644 --- a/sdks/python/apache_beam/typehints/opcodes.py +++ b/sdks/python/apache_beam/typehints/opcodes.py @@ -28,9 +28,13 @@ For internal use only; no backwards-compatibility guarantees. """ from __future__ import absolute_import +import inspect +import sys import types from functools import reduce +import six + from . import typehints from .trivial_inference import BoundMethod from .trivial_inference import Const @@ -147,7 +151,7 @@ binary_subtract = inplace_subtract = symmetric_binary_op def binary_subscr(state, unused_arg): tos = state.stack.pop() - if tos in (str, unicode): + if tos in (str, six.text_type): out = tos else: out = element_type(tos) @@ -261,13 +265,22 @@ build_map = push_value(Dict[Any, Any]) def load_attr(state, arg): + """Replaces the top of the stack, TOS, with + getattr(TOS, co_names[arg]) + """ o = state.stack.pop() name = state.get_name(arg) if isinstance(o, Const) and hasattr(o.value, name): state.stack.append(Const(getattr(o.value, name))) - elif (isinstance(o, type) - and isinstance(getattr(o, name, None), types.MethodType)): - state.stack.append(Const(BoundMethod(getattr(o, name)))) + elif (inspect.isclass(o) and + isinstance(getattr(o, name, None), + (types.MethodType, types.FunctionType))): + # TODO(luke-zhu): Support other callable objects + if sys.version_info[0] == 2: + func = getattr(o, name).__func__ + else: + func = getattr(o, name) # Python 3 has no unbound methods + state.stack.append(Const(BoundMethod(func, o))) else: state.stack.append(Any) @@ -327,7 +340,18 @@ def call_function(state, arg, has_var=False, has_kw=False): def make_function(state, arg): - state.stack[-arg - 1:] = [Any] # a callable + """Creates a function with the arguments at the top of the stack. + """ + # TODO(luke-zhu): Handle default argument types + globals = state.f.__globals__ # Inherits globals from the current frame + if sys.version_info[0] == 2: + func_code = state.stack[-1].value + func = types.FunctionType(func_code, globals) + else: + func_name = state.stack[-1].value + func_code = state.stack[-2].value + func = types.FunctionType(func_code, globals, name=func_name) + state.stack.append(Const(func)) def make_closure(state, arg): diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index a68bd18..28bf8f5 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -22,9 +22,9 @@ For internal use only; no backwards-compatibility guarantees. from __future__ import absolute_import from __future__ import print_function -import __builtin__ import collections import dis +import inspect import pprint import sys import types @@ -32,6 +32,8 @@ from functools import reduce from apache_beam.typehints import Any from apache_beam.typehints import typehints +from six.moves import builtins +from six.moves import zip class TypeInferenceError(ValueError): @@ -46,9 +48,9 @@ def instance_to_type(o): if o is None: return type(None) elif t not in typehints.DISALLOWED_PRIMITIVE_TYPES: - if t == types.InstanceType: + if sys.version_info[0] == 2 and t == types.InstanceType: return o.__class__ - elif t == BoundMethod: + if t == BoundMethod: return types.MethodType return t elif t == tuple: @@ -107,15 +109,12 @@ class FrameState(object): def __init__(self, f, local_vars=None, stack=()): self.f = f - if sys.version_info[0] >= 3: - self.co = f.__code__ - else: - self.co = f.func_code + self.co = f.__code__ self.vars = list(local_vars) self.stack = list(stack) def __eq__(self, other): - return self.__dict__ == other.__dict__ + return isinstance(other, FrameState) and self.__dict__ == other.__dict__ def copy(self): return FrameState(self.f, self.vars, self.stack) @@ -133,8 +132,8 @@ class FrameState(object): name = self.get_name(i) if name in self.f.__globals__: return Const(self.f.__globals__[name]) - if name in __builtin__.__dict__: - return Const(__builtin__.__dict__[name]) + if name in builtins.__dict__: + return Const(builtins.__dict__[name]) return Any def get_name(self, i): @@ -203,8 +202,15 @@ class BoundMethod(object): """Used to create a bound method when we only know the type of the instance. """ - def __init__(self, unbound): - self.unbound = unbound + def __init__(self, func, type): + """Instantiates a bound method object. + + Args: + func (types.FunctionType): The method's underlying function + type (type): The class of the method. + """ + self.func = func + self.type = type def hashable(c): @@ -238,10 +244,9 @@ def infer_return_type(c, input_types, debug=False, depth=5): input_types = [Const(c.__self__)] + input_types return infer_return_type_func(c.__func__, input_types, debug, depth) elif isinstance(c, BoundMethod): - input_types = [c.unbound.__self__.__class__] + input_types - return infer_return_type_func( - c.unbound.__func__, input_types, debug, depth) - elif isinstance(c, type): + input_types = [c.type] + input_types + return infer_return_type_func(c.func, input_types, debug, depth) + elif inspect.isclass(c): if c in typehints.DISALLOWED_PRIMITIVE_TYPES: return { list: typehints.List[Any], @@ -303,15 +308,23 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): last_pc = -1 while pc < end: start = pc - op = ord(code[pc]) - + if sys.version_info[0] == 2: + op = ord(code[pc]) + else: + op = code[pc] if debug: print('-->' if pc == last_pc else ' ', end=' ') print(repr(pc).rjust(4), end=' ') print(dis.opname[op].ljust(20), end=' ') + pc += 1 if op >= dis.HAVE_ARGUMENT: - arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg + if sys.version_info[0] == 2: + arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg + elif sys.version_info[0] == 3 and sys.version_info[1] < 6: + arg = code[pc] + code[pc + 1] * 256 + extended_arg + else: + pass # TODO(luke-zhu): Python 3.6 bytecode to wordcode changes extended_arg = 0 pc += 2 if op == dis.EXTENDED_ARG: @@ -344,7 +357,7 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): opname = dis.opname[op] jmp = jmp_state = None if opname.startswith('CALL_FUNCTION'): - standard_args = (arg & 0xF) + (arg & 0xF0) / 8 + standard_args = (arg & 0xF) + (arg & 0xF0) // 8 var_args = 'VAR' in opname kw_args = 'KW' in opname pop_count = standard_args + var_args + kw_args + 1 diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index 37b2258..b017e8a 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -72,6 +72,17 @@ class TrivialInferenceTest(unittest.TestCase): return None self.assertReturnType(typehints.Union[int, type(None)], func, [int]) + def testSimpleList(self): + self.assertReturnType( + typehints.List[int], + lambda xs: [1, 2], + [typehints.Tuple[int, ...]]) + + self.assertReturnType( + typehints.List[typehints.Any], + lambda xs: list(xs), # List is a disallowed builtin + [typehints.Tuple[int, ...]]) + def testListComprehension(self): self.assertReturnType( typehints.List[int], @@ -98,6 +109,12 @@ class TrivialInferenceTest(unittest.TestCase): self.assertReturnType( typehints.Iterable[typehints.Union[int, float]], foo, [int, float]) + def testGeneratorComprehension(self): + self.assertReturnType( + typehints.Iterable[int], + lambda xs: (x for x in xs), + [typehints.Tuple[int, ...]]) + def testBinOp(self): self.assertReturnType(int, lambda a, b: a + b, [int, int]) self.assertReturnType( @@ -130,7 +147,6 @@ class TrivialInferenceTest(unittest.TestCase): def testMethod(self): class A(object): - def m(self, x): return x -- To stop receiving notification emails like this one, please contact ['"commits@beam.apache.org" <commits@beam.apache.org>'].