Author: Alex Gaynor <alex.gay...@gmail.com> Branch: Changeset: r44872:9db17fbc1fd9 Date: 2011-06-10 10:51 +0200 http://bitbucket.org/pypy/pypy/changeset/9db17fbc1fd9/
Log: merged upstream. diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -91,7 +91,7 @@ Remove the GIL -------------- -This is a major task that requiers lots of thinking. However, few subprojects +This is a major task that requires lots of thinking. However, few subprojects can be potentially specified, unless a better plan can be thought out: * A thread-aware garbage collector diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -191,9 +191,15 @@ # of the operation. It must inherit from AbstractDescr. The # backend provides it with cpu.fielddescrof(), cpu.arraydescrof(), # cpu.calldescrof(), and cpu.typedescrof(). + self._check_descr(descr) + self._descr = descr + + def _check_descr(self, descr): + if not we_are_translated() and getattr(descr, 'I_am_a_descr', False): + return # needed for the mock case in oparser_model from pypy.jit.metainterp.history import check_descr check_descr(descr) - self._descr = descr + class GuardResOp(ResOpWithDescr): diff --git a/pypy/jit/metainterp/test/test_logger.py b/pypy/jit/metainterp/test/test_logger.py --- a/pypy/jit/metainterp/test/test_logger.py +++ b/pypy/jit/metainterp/test/test_logger.py @@ -53,7 +53,7 @@ def make_metainterp_sd(self): class FakeJitDriver(object): class warmstate(object): - get_location_str = staticmethod(lambda args: args[0]._get_str()) + get_location_str = staticmethod(lambda args: "dupa") class FakeMetaInterpSd: cpu = AbstractCPU() @@ -116,10 +116,10 @@ def test_debug_merge_point(self): inp = ''' [] - debug_merge_point(0, 0, "dupa") + debug_merge_point(0, 0) ''' _, loop, oloop = self.reparse(inp, check_equal=False) - assert loop.operations[0].getarg(2)._get_str() == "dupa" + assert loop.operations[0].getarg(1).getint() == 0 assert oloop.operations[0].getarg(1)._get_str() == "dupa" def test_floats(self): diff --git a/pypy/jit/tool/oparser.py b/pypy/jit/tool/oparser.py --- a/pypy/jit/tool/oparser.py +++ b/pypy/jit/tool/oparser.py @@ -3,24 +3,15 @@ in a nicer fashion """ -from pypy.jit.metainterp.history import TreeLoop, BoxInt, ConstInt,\ - ConstObj, ConstPtr, Box, BasicFailDescr, BoxFloat, ConstFloat,\ - LoopToken, get_const_ptr_for_string, get_const_ptr_for_unicode +from pypy.jit.tool.oparser_model import get_model + from pypy.jit.metainterp.resoperation import rop, ResOperation, \ ResOpWithDescr, N_aryOp, \ UnaryOp, PlainResOp -from pypy.jit.metainterp.typesystem import llhelper -from pypy.jit.codewriter.heaptracker import adr2int -from pypy.jit.codewriter import longlong -from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.ootypesystem import ootype class ParseError(Exception): pass -class Boxes(object): - pass - class ESCAPE_OP(N_aryOp, ResOpWithDescr): OPNUM = -123 @@ -54,37 +45,15 @@ def clone(self): return FORCE_SPILL(self.OPNUM, self.getarglist()[:]) -class ExtendedTreeLoop(TreeLoop): - def getboxes(self): - def opboxes(operations): - for op in operations: - yield op.result - for box in op.getarglist(): - yield box - def allboxes(): - for box in self.inputargs: - yield box - for box in opboxes(self.operations): - yield box - - boxes = Boxes() - for box in allboxes(): - if isinstance(box, Box): - name = str(box) - setattr(boxes, name, box) - return boxes - - def setvalues(self, **kwds): - boxes = self.getboxes() - for name, value in kwds.iteritems(): - getattr(boxes, name).value = value - -def default_fail_descr(fail_args=None): - return BasicFailDescr() +def default_fail_descr(model, fail_args=None): + return model.BasicFailDescr() class OpParser(object): + + use_mock_model = False + def __init__(self, input, cpu, namespace, type_system, boxkinds, invent_fail_descr=default_fail_descr, nonstrict=False): @@ -100,7 +69,8 @@ self._cache = {} self.invent_fail_descr = invent_fail_descr self.nonstrict = nonstrict - self.looptoken = LoopToken() + self.model = get_model(self.use_mock_model) + self.looptoken = self.model.LoopToken() def get_const(self, name, typ): if self._consts is None: @@ -108,16 +78,16 @@ obj = self._consts[name] if self.type_system == 'lltype': if typ == 'ptr': - return ConstPtr(obj) + return self.model.ConstPtr(obj) else: assert typ == 'class' - return ConstInt(adr2int(llmemory.cast_ptr_to_adr(obj))) + return self.model.ConstInt(self.model.ptr_to_int(obj)) else: if typ == 'ptr': - return ConstObj(obj) + return self.model.ConstObj(obj) else: assert typ == 'class' - return ConstObj(ootype.cast_to_object(obj)) + return self.model.ConstObj(ootype.cast_to_object(obj)) def get_descr(self, poss_descr): if poss_descr.startswith('<'): @@ -132,16 +102,16 @@ pass if elem.startswith('i'): # integer - box = BoxInt() - _box_counter_more_than(elem[1:]) + box = self.model.BoxInt() + _box_counter_more_than(self.model, elem[1:]) elif elem.startswith('f'): - box = BoxFloat() - _box_counter_more_than(elem[1:]) + box = self.model.BoxFloat() + _box_counter_more_than(self.model, elem[1:]) elif elem.startswith('p'): # pointer - ts = getattr(self.cpu, 'ts', llhelper) + ts = getattr(self.cpu, 'ts', self.model.llhelper) box = ts.BoxRef() - _box_counter_more_than(elem[1:]) + _box_counter_more_than(self.model, elem[1:]) else: for prefix, boxclass in self.boxkinds.iteritems(): if elem.startswith(prefix): @@ -175,21 +145,21 @@ def getvar(self, arg): if not arg: - return ConstInt(0) + return self.model.ConstInt(0) try: - return ConstInt(int(arg)) + return self.model.ConstInt(int(arg)) except ValueError: if self.is_float(arg): - return ConstFloat(longlong.getfloatstorage(float(arg))) + return self.model.ConstFloat(self.model.convert_to_floatstorage(arg)) if (arg.startswith('"') or arg.startswith("'") or arg.startswith('s"')): # XXX ootype info = arg[1:].strip("'\"") - return get_const_ptr_for_string(info) + return self.model.get_const_ptr_for_string(info) if arg.startswith('u"'): # XXX ootype info = arg[1:].strip("'\"") - return get_const_ptr_for_unicode(info) + return self.model.get_const_ptr_for_unicode(info) if arg.startswith('ConstClass('): name = arg[len('ConstClass('):-1] return self.get_const(name, 'class') @@ -197,9 +167,9 @@ return None elif arg == 'NULL': if self.type_system == 'lltype': - return ConstPtr(ConstPtr.value) + return self.model.ConstPtr(self.model.ConstPtr.value) else: - return ConstObj(ConstObj.value) + return self.model.ConstObj(self.model.ConstObj.value) elif arg.startswith('ConstPtr('): name = arg[len('ConstPtr('):-1] return self.get_const(name, 'ptr') @@ -212,7 +182,7 @@ descr = None if argspec.strip(): if opname == 'debug_merge_point': - allargs = argspec.split(',', 2) + allargs = argspec.split(',', 1) else: allargs = [arg for arg in argspec.split(",") if arg != ''] @@ -266,14 +236,14 @@ "Unknown var in fail_args: %s" % arg) fail_args.append(fail_arg) if descr is None and self.invent_fail_descr: - descr = self.invent_fail_descr(fail_args) + descr = self.invent_fail_descr(self.model, fail_args) if hasattr(descr, '_oparser_uses_descr_of_guard'): descr._oparser_uses_descr_of_guard(self, fail_args) else: fail_args = None if opnum == rop.FINISH: if descr is None and self.invent_fail_descr: - descr = self.invent_fail_descr() + descr = self.invent_fail_descr(self.model) elif opnum == rop.JUMP: if descr is None and self.invent_fail_descr: descr = self.looptoken @@ -338,7 +308,7 @@ num, ops, last_offset = self.parse_ops(base_indent, newlines, 0) if num < len(newlines): raise ParseError("unexpected dedent at line: %s" % newlines[num]) - loop = ExtendedTreeLoop("loop") + loop = self.model.ExtendedTreeLoop("loop") loop.comment = first_comment loop.token = self.looptoken loop.operations = ops @@ -394,7 +364,7 @@ def parse(input, cpu=None, namespace=None, type_system='lltype', boxkinds=None, invent_fail_descr=default_fail_descr, - no_namespace=False, nonstrict=False): + no_namespace=False, nonstrict=False, OpParser=OpParser): if namespace is None and not no_namespace: namespace = {} return OpParser(input, cpu, namespace, type_system, boxkinds, @@ -405,6 +375,6 @@ return parse(*args, **kwds) -def _box_counter_more_than(s): +def _box_counter_more_than(model, s): if s.isdigit(): - Box._counter = max(Box._counter, int(s)+1) + model.Box._counter = max(model.Box._counter, int(s)+1) diff --git a/pypy/jit/tool/oparser_model.py b/pypy/jit/tool/oparser_model.py new file mode 100644 --- /dev/null +++ b/pypy/jit/tool/oparser_model.py @@ -0,0 +1,148 @@ +class Boxes(object): + pass + +def get_real_model(): + class LoopModel(object): + from pypy.jit.metainterp.history import TreeLoop, LoopToken + from pypy.jit.metainterp.history import Box, BoxInt, BoxFloat + from pypy.jit.metainterp.history import ConstInt, ConstObj, ConstPtr, ConstFloat + from pypy.jit.metainterp.history import BasicFailDescr + from pypy.jit.metainterp.typesystem import llhelper + + from pypy.jit.metainterp.history import get_const_ptr_for_string + from pypy.jit.metainterp.history import get_const_ptr_for_unicode + get_const_ptr_for_string = staticmethod(get_const_ptr_for_string) + get_const_ptr_for_unicode = staticmethod(get_const_ptr_for_unicode) + + @staticmethod + def convert_to_floatstorage(arg): + from pypy.jit.codewriter import longlong + return longlong.getfloatstorage(float(arg)) + + @staticmethod + def ptr_to_int(obj): + from pypy.jit.codewriter.heaptracker import adr2int + from pypy.rpython.lltypesystem import llmemory + return adr2int(llmemory.cast_ptr_to_adr(obj)) + + @staticmethod + def ootype_cast_to_object(obj): + from pypy.rpython.ootypesystem import ootype + return ootype.cast_to_object(obj) + + return LoopModel + +def get_mock_model(): + class LoopModel(object): + + class TreeLoop(object): + def __init__(self, name): + self.name = name + + class LoopToken(object): + I_am_a_descr = True + + class BasicFailDescr(object): + I_am_a_descr = True + + class Box(object): + _counter = 0 + type = 'b' + + def __init__(self, value=0): + self.value = value + + def __repr__(self): + result = str(self) + result += '(%s)' % self.value + return result + + def __str__(self): + if not hasattr(self, '_str'): + self._str = '%s%d' % (self.type, Box._counter) + Box._counter += 1 + return self._str + + class BoxInt(Box): + type = 'i' + + class BoxFloat(Box): + type = 'f' + + class BoxRef(Box): + type = 'p' + + class Const(object): + def __init__(self, value=None): + self.value = value + + def _get_str(self): + return str(self.value) + + class ConstInt(Const): + pass + + class ConstPtr(Const): + pass + + class ConstFloat(Const): + pass + + @classmethod + def get_const_ptr_for_string(cls, s): + return cls.ConstPtr(s) + + @classmethod + def get_const_ptr_for_unicode(cls, s): + return cls.ConstPtr(s) + + @staticmethod + def convert_to_floatstorage(arg): + return float(arg) + + @staticmethod + def ptr_to_int(obj): + return id(obj) + + class llhelper(object): + pass + + LoopModel.llhelper.BoxRef = LoopModel.BoxRef + + return LoopModel + + +def get_model(use_mock): + if use_mock: + model = get_mock_model() + else: + model = get_real_model() + + class ExtendedTreeLoop(model.TreeLoop): + + def getboxes(self): + def opboxes(operations): + for op in operations: + yield op.result + for box in op.getarglist(): + yield box + def allboxes(): + for box in self.inputargs: + yield box + for box in opboxes(self.operations): + yield box + + boxes = Boxes() + for box in allboxes(): + if isinstance(box, model.Box): + name = str(box) + setattr(boxes, name, box) + return boxes + + def setvalues(self, **kwds): + boxes = self.getboxes() + for name, value in kwds.iteritems(): + getattr(boxes, name).value = value + + model.ExtendedTreeLoop = ExtendedTreeLoop + return model diff --git a/pypy/jit/tool/test/test_oparser.py b/pypy/jit/tool/test/test_oparser.py --- a/pypy/jit/tool/test/test_oparser.py +++ b/pypy/jit/tool/test/test_oparser.py @@ -1,227 +1,274 @@ import py +import sys from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.jit.tool.oparser import parse, ParseError +from pypy.jit.tool.oparser import parse, OpParser from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken,\ - BoxFloat +from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken -def test_basic_parse(): - x = """ - [i0, i1] - # a comment - i2 = int_add(i0, i1) - i3 = int_sub(i2, 3) # another comment - finish() # (tricky) - """ - loop = parse(x) - assert len(loop.operations) == 3 - assert [op.getopnum() for op in loop.operations] == [rop.INT_ADD, rop.INT_SUB, - rop.FINISH] - assert len(loop.inputargs) == 2 - assert loop.operations[-1].getdescr() +class BaseTestOparser(object): -def test_const_ptr_subops(): - x = """ - [p0] - guard_class(p0, ConstClass(vtable)) [] - """ - S = lltype.Struct('S') - vtable = lltype.nullptr(S) - loop = parse(x, None, locals()) - assert len(loop.operations) == 1 - assert loop.operations[0].getdescr() - assert loop.operations[0].getfailargs() == [] + OpParser = None -def test_descr(): - class Xyz(AbstractDescr): - pass - - x = """ - [p0] - i1 = getfield_gc(p0, descr=stuff) - """ - stuff = Xyz() - loop = parse(x, None, locals()) - assert loop.operations[0].getdescr() is stuff + def parse(self, *args, **kwds): + kwds['OpParser'] = self.OpParser + return parse(*args, **kwds) -def test_after_fail(): - x = """ - [i0] - guard_value(i0, 3) [] - i1 = int_add(1, 2) - """ - loop = parse(x, None, {}) - assert len(loop.operations) == 2 + def test_basic_parse(self): + x = """ + [i0, i1] + # a comment + i2 = int_add(i0, i1) + i3 = int_sub(i2, 3) # another comment + finish() # (tricky) + """ + loop = self.parse(x) + assert len(loop.operations) == 3 + assert [op.getopnum() for op in loop.operations] == [rop.INT_ADD, rop.INT_SUB, + rop.FINISH] + assert len(loop.inputargs) == 2 + assert loop.operations[-1].getdescr() -def test_descr_setfield(): - class Xyz(AbstractDescr): - pass - - x = """ - [p0] - setfield_gc(p0, 3, descr=stuff) - """ - stuff = Xyz() - loop = parse(x, None, locals()) - assert loop.operations[0].getdescr() is stuff + def test_const_ptr_subops(self): + x = """ + [p0] + guard_class(p0, ConstClass(vtable)) [] + """ + S = lltype.Struct('S') + vtable = lltype.nullptr(S) + loop = self.parse(x, None, locals()) + assert len(loop.operations) == 1 + assert loop.operations[0].getdescr() + assert loop.operations[0].getfailargs() == [] -def test_boxname(): - x = """ - [i42] - i50 = int_add(i42, 1) - """ - loop = parse(x, None, {}) - assert str(loop.inputargs[0]) == 'i42' - assert str(loop.operations[0].result) == 'i50' + def test_descr(self): + class Xyz(AbstractDescr): + I_am_a_descr = True # for the mock case -def test_getboxes(): - x = """ - [i0] - i1 = int_add(i0, 10) - """ - loop = parse(x, None, {}) - boxes = loop.getboxes() - assert boxes.i0 is loop.inputargs[0] - assert boxes.i1 is loop.operations[0].result - -def test_setvalues(): - x = """ - [i0] - i1 = int_add(i0, 10) - """ - loop = parse(x, None, {}) - loop.setvalues(i0=32, i1=42) - assert loop.inputargs[0].value == 32 - assert loop.operations[0].result.value == 42 + x = """ + [p0] + i1 = getfield_gc(p0, descr=stuff) + """ + stuff = Xyz() + loop = self.parse(x, None, locals()) + assert loop.operations[0].getdescr() is stuff -def test_boxkind(): - x = """ - [sum0] - """ - loop = parse(x, None, {}, boxkinds={'sum': BoxInt}) - b = loop.getboxes() - assert isinstance(b.sum0, BoxInt) - -def test_getvar_const_ptr(): - x = ''' - [] - call(ConstPtr(func_ptr)) + def test_after_fail(self): + x = """ + [i0] + guard_value(i0, 3) [] + i1 = int_add(1, 2) + """ + loop = self.parse(x, None, {}) + assert len(loop.operations) == 2 + + def test_descr_setfield(self): + class Xyz(AbstractDescr): + I_am_a_descr = True # for the mock case + + x = """ + [p0] + setfield_gc(p0, 3, descr=stuff) + """ + stuff = Xyz() + loop = self.parse(x, None, locals()) + assert loop.operations[0].getdescr() is stuff + + def test_boxname(self): + x = """ + [i42] + i50 = int_add(i42, 1) + """ + loop = self.parse(x, None, {}) + assert str(loop.inputargs[0]) == 'i42' + assert str(loop.operations[0].result) == 'i50' + + def test_getboxes(self): + x = """ + [i0] + i1 = int_add(i0, 10) + """ + loop = self.parse(x, None, {}) + boxes = loop.getboxes() + assert boxes.i0 is loop.inputargs[0] + assert boxes.i1 is loop.operations[0].result + + def test_setvalues(self): + x = """ + [i0] + i1 = int_add(i0, 10) + """ + loop = self.parse(x, None, {}) + loop.setvalues(i0=32, i1=42) + assert loop.inputargs[0].value == 32 + assert loop.operations[0].result.value == 42 + + def test_getvar_const_ptr(self): + x = ''' + [] + call(ConstPtr(func_ptr)) + ''' + TP = lltype.GcArray(lltype.Signed) + NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP)) + loop = self.parse(x, None, {'func_ptr' : NULL}) + assert loop.operations[0].getarg(0).value == NULL + + def test_jump_target(self): + x = ''' + [] + jump() + ''' + loop = self.parse(x) + assert loop.operations[0].getdescr() is loop.token + + def test_jump_target_other(self): + looptoken = LoopToken() + looptoken.I_am_a_descr = True # for the mock case + x = ''' + [] + jump(descr=looptoken) + ''' + loop = self.parse(x, namespace=locals()) + assert loop.operations[0].getdescr() is looptoken + + def test_floats(self): + x = ''' + [f0] + f1 = float_add(f0, 3.5) + ''' + loop = self.parse(x) + box = loop.operations[0].getarg(0) + # we cannot use isinstance, because in case of mock the class will be + # constructed on the fly + assert box.__class__.__name__ == 'BoxFloat' + + def test_debug_merge_point(self): + x = ''' + [] + debug_merge_point(0, "info") + debug_merge_point(0, 'info') + debug_merge_point(1, '<some ('other,')> info') + debug_merge_point(0, '(stuff) #1') + ''' + loop = self.parse(x) + assert loop.operations[0].getarg(1)._get_str() == 'info' + assert loop.operations[1].getarg(1)._get_str() == 'info' + assert loop.operations[2].getarg(1)._get_str() == "<some ('other,')> info" + assert loop.operations[3].getarg(1)._get_str() == "(stuff) #1" + + + def test_descr_with_obj_print(self): + x = ''' + [p0] + setfield_gc(p0, 1, descr=<SomeDescr>) + ''' + loop = self.parse(x) + # assert did not explode + + example_loop_log = '''\ + # bridge out of Guard12, 6 ops + [i0, i1, i2] + i4 = int_add(i0, 2) + i6 = int_sub(i1, 1) + i8 = int_gt(i6, 3) + guard_true(i8, descr=<Guard15>) [i4, i6] + debug_merge_point('(no jitdriver.get_printable_location!)', 0) + jump(i6, i4, descr=<Loop0>) ''' - TP = lltype.GcArray(lltype.Signed) - NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP)) - loop = parse(x, None, {'func_ptr' : NULL}) - assert loop.operations[0].getarg(0).value == NULL -def test_jump_target(): - x = ''' - [] - jump() - ''' - loop = parse(x) - assert loop.operations[0].getdescr() is loop.token + def test_parse_no_namespace(self): + loop = self.parse(self.example_loop_log, no_namespace=True) -def test_jump_target_other(): - looptoken = LoopToken() - x = ''' - [] - jump(descr=looptoken) - ''' - loop = parse(x, namespace=locals()) - assert loop.operations[0].getdescr() is looptoken + def test_attach_comment_to_loop(self): + loop = self.parse(self.example_loop_log, no_namespace=True) + assert loop.comment == ' # bridge out of Guard12, 6 ops' -def test_floats(): - x = ''' - [f0] - f1 = float_add(f0, 3.5) - ''' - loop = parse(x) - assert isinstance(loop.operations[0].getarg(0), BoxFloat) - -def test_debug_merge_point(): - x = ''' - [] - debug_merge_point(0, "info") - debug_merge_point(0, 'info') - debug_merge_point(1, '<some ('other,')> info') - debug_merge_point(0, '(stuff) #1') - ''' - loop = parse(x) - assert loop.operations[0].getarg(1)._get_str() == 'info' - assert loop.operations[1].getarg(1)._get_str() == 'info' - assert loop.operations[2].getarg(1)._get_str() == "<some ('other,')> info" - assert loop.operations[3].getarg(1)._get_str() == "(stuff) #1" - + def test_parse_new_with_comma(self): + # this is generated by PYPYJITLOG, check that we can handle it + x = ''' + [] + p0 = new(, descr=<SizeDescr 12>) + ''' + loop = self.parse(x) + assert loop.operations[0].getopname() == 'new' -def test_descr_with_obj_print(): - x = ''' - [p0] - setfield_gc(p0, 1, descr=<SomeDescr>) - ''' - loop = parse(x) - # assert did not explode + def test_no_fail_args(self): + x = ''' + [i0] + guard_true(i0, descr=<Guard0>) + ''' + loop = self.parse(x, nonstrict=True) + assert loop.operations[0].getfailargs() == [] -example_loop_log = '''\ -# bridge out of Guard12, 6 ops -[i0, i1, i2] -i4 = int_add(i0, 2) -i6 = int_sub(i1, 1) -i8 = int_gt(i6, 3) -guard_true(i8, descr=<Guard15>) [i4, i6] -debug_merge_point('(no jitdriver.get_printable_location!)', 0) -jump(i6, i4, descr=<Loop0>) -''' + def test_no_inputargs(self): + x = ''' + i2 = int_add(i0, i1) + ''' + loop = self.parse(x, nonstrict=True) + assert loop.inputargs == [] + assert loop.operations[0].getopname() == 'int_add' -def test_parse_no_namespace(): - loop = parse(example_loop_log, no_namespace=True) + def test_offsets(self): + x = """ + [i0, i1] + +10: i2 = int_add(i0, i1) + i3 = int_add(i2, 3) + """ + # +30: --end of the loop-- + loop = self.parse(x) + assert loop.operations[0].offset == 10 + assert not hasattr(loop.operations[1], 'offset') -def test_attach_comment_to_loop(): - loop = parse(example_loop_log, no_namespace=True) - assert loop.comment == '# bridge out of Guard12, 6 ops' + def test_last_offset(self): + x = """ + [i0, i1] + +10: i2 = int_add(i0, i1) + i3 = int_add(i2, 3) + +30: --end of the loop-- + """ + loop = self.parse(x) + assert len(loop.operations) == 2 + assert loop.last_offset == 30 -def test_parse_new_with_comma(): - # this is generated by PYPYJITLOG, check that we can handle it - x = ''' - [] - p0 = new(, descr=<SizeDescr 12>) - ''' - loop = parse(x) - assert loop.operations[0].getopname() == 'new' -def test_no_fail_args(): - x = ''' - [i0] - guard_true(i0, descr=<Guard0>) - ''' - loop = parse(x, nonstrict=True) - assert loop.operations[0].getfailargs() == [] +class TestOpParser(BaseTestOparser): -def test_no_inputargs(): - x = ''' - i2 = int_add(i0, i1) - ''' - loop = parse(x, nonstrict=True) - assert loop.inputargs == [] - assert loop.operations[0].getopname() == 'int_add' + OpParser = OpParser -def test_offsets(): - x = """ - [i0, i1] - +10: i2 = int_add(i0, i1) - i3 = int_add(i2, 3) - """ - # +30: --end of the loop-- - loop = parse(x) - assert loop.operations[0].offset == 10 - assert not hasattr(loop.operations[1], 'offset') + def test_boxkind(self): + x = """ + [sum0] + """ + loop = self.parse(x, None, {}, boxkinds={'sum': BoxInt}) + b = loop.getboxes() + assert isinstance(b.sum0, BoxInt) -def test_last_offset(): - x = """ - [i0, i1] - +10: i2 = int_add(i0, i1) - i3 = int_add(i2, 3) - +30: --end of the loop-- - """ - loop = parse(x) - assert len(loop.operations) == 2 - assert loop.last_offset == 30 + +class ForbiddenModule(object): + def __init__(self, name, old_mod): + self.name = name + self.old_mod = old_mod + + def __getattr__(self, attr): + assert False, "You should not import module %s" % self.name + + +class TestOpParserWithMock(BaseTestOparser): + + class OpParser(OpParser): + use_mock_model = True + + def setup_class(cls): + forbidden_mods = [ + 'pypy.jit.metainterp.history', + 'pypy.rpython.lltypesystem.lltype', + ] + for modname in forbidden_mods: + if modname in sys.modules: + newmod = ForbiddenModule(modname, sys.modules[modname]) + sys.modules[modname] = newmod + + def teardown_class(cls): + for modname, mod in sys.modules.iteritems(): + if isinstance(mod, ForbiddenModule): + sys.modules[modname] = mod.old_mod diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -107,6 +107,9 @@ def tmpnam(): """Return an absolute pathname of a file that did not exist at the time the call is made.""" + from warnings import warn + warn(RuntimeWarning("tmpnam is a potential security risk to your program")) + import tempfile return tempfile.mktemp() @@ -114,6 +117,9 @@ """Return an absolute pathname of a file that did not exist at the time the call is made. The directory and a prefix may be specified as strings; they may be omitted or None if not needed.""" + from warnings import warn + warn(RuntimeWarning("tempnam is a potential security risk to your program")) + import tempfile return tempfile.mktemp('', prefix or 'tmp', dir) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -847,6 +847,21 @@ assert os.path.basename(s1).startswith(prefix or 'tmp') assert os.path.basename(s2).startswith(prefix or 'tmp') + def test_tmpnam_warning(self): + import warnings, os + # + def f_tmpnam_warning(): os.tmpnam() # a single line + # + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + f_tmpnam_warning() + assert len(w) == 1 + assert issubclass(w[-1].category, RuntimeWarning) + assert "potential security risk" in str(w[-1].message) + # check that the warning points to the call to os.tmpnam(), + # not to some code inside app_posix.py + assert w[-1].lineno == f_tmpnam_warning.func_code.co_firstlineno + class AppTestEnvironment(object): def setup_class(cls): diff --git a/pypy/rlib/rrandom.py b/pypy/rlib/rrandom.py --- a/pypy/rlib/rrandom.py +++ b/pypy/rlib/rrandom.py @@ -24,8 +24,7 @@ def __init__(self, seed=r_uint(0)): self.state = [r_uint(0)] * N self.index = 0 - if seed: - self.init_genrand(seed) + self.init_genrand(seed) def init_genrand(self, s): mt = self.state diff --git a/pypy/rlib/test/test_rrandom.py b/pypy/rlib/test/test_rrandom.py --- a/pypy/rlib/test/test_rrandom.py +++ b/pypy/rlib/test/test_rrandom.py @@ -3,6 +3,12 @@ # the numbers were created by using CPython's _randommodule.c +def test_init_from_zero(): + rnd = Random(0) + assert rnd.state[:14] == [0, 1, 1812433255, 1900727105, 1208447044, + 2481403966, 4042607538, 337614300, 3232553940, + 1018809052, 3202401494, 1775180719, 3192392114, 594215549] + def test_init_from_seed(): rnd = Random(1000) assert rnd.state[:14] == [1000, 4252021385, 1724402292, 571538732, diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -253,7 +253,7 @@ if hasattr(callable, '_errorcode_'): errorcode = callable._errorcode_ else: - errorcode = TP.TO.RESULT._example() + errorcode = TP.TO.RESULT._defl() callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True diff --git a/pypy/tool/gcc_cache.py b/pypy/tool/gcc_cache.py --- a/pypy/tool/gcc_cache.py +++ b/pypy/tool/gcc_cache.py @@ -39,7 +39,12 @@ data = '' if not (data.startswith('True') or data.startswith('FAIL\n')): try: - platform.compile(c_files, eci) + _previous = platform.log_errors + try: + platform.log_errors = False + platform.compile(c_files, eci) + finally: + platform.log_errors = _previous data = 'True' path.write(data) except CompilationError, e: diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -1,4 +1,5 @@ import re, sys + from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.tool.oparser import OpParser @@ -51,6 +52,7 @@ # factory method Op = Op + use_mock_model = True @classmethod def parse_from_input(cls, input): diff --git a/pypy/tool/test/test_gcc_cache.py b/pypy/tool/test/test_gcc_cache.py --- a/pypy/tool/test/test_gcc_cache.py +++ b/pypy/tool/test/test_gcc_cache.py @@ -1,11 +1,13 @@ - +import sys from pypy.tool.gcc_cache import * from pypy.tool.udir import udir -import md5 +import md5, cStringIO from pypy.translator.tool.cbuild import ExternalCompilationInfo +localudir = udir.join('test_gcc_cache').ensure(dir=1) + def test_gcc_exec(): - f = udir.join("x.c") + f = localudir.join("x.c") f.write(""" #include <stdio.h> #include <test_gcc_exec.h> @@ -15,8 +17,8 @@ return 0; } """) - dir1 = udir.join('test_gcc_exec_dir1').ensure(dir=1) - dir2 = udir.join('test_gcc_exec_dir2').ensure(dir=1) + dir1 = localudir.join('test_gcc_exec_dir1').ensure(dir=1) + dir2 = localudir.join('test_gcc_exec_dir2').ensure(dir=1) dir1.join('test_gcc_exec.h').write('#define ANSWER 3\n') dir2.join('test_gcc_exec.h').write('#define ANSWER 42\n') eci = ExternalCompilationInfo(include_dirs=[str(dir1)]) @@ -36,7 +38,7 @@ print '>>>' def test_gcc_ask(): - f = udir.join("y.c") + f = localudir.join("y.c") f.write(""" #include <stdio.h> #include <test_gcc_ask.h> @@ -46,8 +48,8 @@ return 0; } """) - dir1 = udir.join('test_gcc_ask_dir1').ensure(dir=1) - dir2 = udir.join('test_gcc_ask_dir2').ensure(dir=1) + dir1 = localudir.join('test_gcc_ask_dir1').ensure(dir=1) + dir2 = localudir.join('test_gcc_ask_dir2').ensure(dir=1) dir1.join('test_gcc_ask.h').write('/* hello world */\n') dir2.join('test_gcc_ask.h').write('#error boom\n') eci = ExternalCompilationInfo(include_dirs=[str(dir1)]) @@ -63,3 +65,15 @@ print '<<<' print err print '>>>' + +def test_gcc_ask_doesnt_log_errors(): + f = localudir.join('z.c') + f.write("""this file is not valid C code\n""") + eci = ExternalCompilationInfo() + oldstderr = sys.stderr + try: + sys.stderr = capture = cStringIO.StringIO() + py.test.raises(CompilationError, try_compile_cache, [f], eci) + finally: + sys.stderr = oldstderr + assert 'ERROR' not in capture.getvalue().upper() diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py --- a/pypy/translator/c/gc.py +++ b/pypy/translator/c/gc.py @@ -297,6 +297,13 @@ gc_startup_code = RefcountingGcPolicy.gc_startup_code.im_func + def compilation_info(self): + eci = BasicGcPolicy.compilation_info(self) + eci = eci.merge(ExternalCompilationInfo( + post_include_bits=['#define USING_NO_GC_AT_ALL'], + )) + return eci + class FrameworkGcPolicy(BasicGcPolicy): transformerclass = framework.FrameworkGCTransformer diff --git a/pypy/translator/c/gcc/test/elf/track12.s b/pypy/translator/c/gcc/test/elf/track12.s new file mode 100644 --- /dev/null +++ b/pypy/translator/c/gcc/test/elf/track12.s @@ -0,0 +1,9 @@ + .type pypy_f, @function +pypy_f: + pushl 4(%esp) + call pypy_other + ;; expected {4(%esp) | %ebx, %esi, %edi, %ebp | (%esp)} + popl %eax + /* GCROOT %eax */ + ret + .size pypy_f, .-pypy_f diff --git a/pypy/translator/c/gcc/test/elf/track13.s b/pypy/translator/c/gcc/test/elf/track13.s new file mode 100644 --- /dev/null +++ b/pypy/translator/c/gcc/test/elf/track13.s @@ -0,0 +1,9 @@ + .type pypy_f, @function +pypy_f: + call pypy_other + ;; expected {(%esp) | %ebx, %esi, %edi, %ebp | 8(%esp)} + pushl 8(%esp) + popl %eax + /* GCROOT %eax */ + ret + .size pypy_f, .-pypy_f diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py --- a/pypy/translator/c/gcc/trackgcroot.py +++ b/pypy/translator/c/gcc/trackgcroot.py @@ -271,7 +271,8 @@ match = self.r_localvar_esp.match(localvar) if match: - if localvar == self.TOP_OF_STACK: # for pushl and popl, by + if localvar == self.TOP_OF_STACK_MINUS_WORD: + # for pushl and popl, by hint = None # default ebp addressing is else: # a bit nicer hint = 'esp' @@ -591,10 +592,12 @@ def _visit_push(self, line): match = self.r_unaryinsn.match(line) source = match.group(1) - return [InsnStackAdjust(-self.WORD)] + self.insns_for_copy(source, self.TOP_OF_STACK) + return self.insns_for_copy(source, self.TOP_OF_STACK_MINUS_WORD) + \ + [InsnStackAdjust(-self.WORD)] def _visit_pop(self, target): - return self.insns_for_copy(self.TOP_OF_STACK, target) + [InsnStackAdjust(+self.WORD)] + return [InsnStackAdjust(+self.WORD)] + \ + self.insns_for_copy(self.TOP_OF_STACK_MINUS_WORD, target) def _visit_prologue(self): # for the prologue of functions that use %ebp as frame pointer @@ -986,15 +989,15 @@ OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' OFFSET_LABELS = 2**30 - TOP_OF_STACK = '0(%esp)' + TOP_OF_STACK_MINUS_WORD = '-4(%esp)' r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") - LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" + LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|-?\d*[(]%esp[)]" LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" r_localvarnofp = re.compile(LOCALVAR) r_localvarfp = re.compile(LOCALVARFP) - r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") + r_localvar_esp = re.compile(r"(-?\d*)[(]%esp[)]") r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") r_rel_label = re.compile(r"(\d+):\s*$") @@ -1047,7 +1050,7 @@ OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' OFFSET_LABELS = 2**30 - TOP_OF_STACK = '0(%rsp)' + TOP_OF_STACK_MINUS_WORD = '-8(%rsp)' r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") @@ -1143,7 +1146,7 @@ CALLEE_SAVE_REGISTERS = ['ebx', 'esi', 'edi', 'ebp'] REG2LOC = dict((_reg, LOC_REG | ((_i+1)<<2)) for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) - TOP_OF_STACK = 'DWORD PTR [esp]' + TOP_OF_STACK_MINUS_WORD = 'DWORD PTR [esp-4]' OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?:@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' LABEL = r'([a-zA-Z_$@.][a-zA-Z0-9_$@.]*)' diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py --- a/pypy/translator/c/genc.py +++ b/pypy/translator/c/genc.py @@ -900,8 +900,9 @@ print >> f, '}' def commondefs(defines): - from pypy.rlib.rarithmetic import LONG_BIT + from pypy.rlib.rarithmetic import LONG_BIT, LONGLONG_BIT defines['PYPY_LONG_BIT'] = LONG_BIT + defines['PYPY_LONGLONG_BIT'] = LONGLONG_BIT def add_extra_files(eci): srcdir = py.path.local(autopath.pypydir).join('translator', 'c', 'src') diff --git a/pypy/translator/c/src/int.h b/pypy/translator/c/src/int.h --- a/pypy/translator/c/src/int.h +++ b/pypy/translator/c/src/int.h @@ -73,15 +73,28 @@ /* NB. shifting has same limitations as C: the shift count must be >= 0 and < LONG_BITS. */ -#define OP_INT_RSHIFT(x,y,r) r = Py_ARITHMETIC_RIGHT_SHIFT(long, x, y) -#define OP_UINT_RSHIFT(x,y,r) r = (x) >> (y) -#define OP_LLONG_RSHIFT(x,y,r) r = Py_ARITHMETIC_RIGHT_SHIFT(PY_LONG_LONG,x,y) -#define OP_ULLONG_RSHIFT(x,y,r) r = (x) >> (y) +#define CHECK_SHIFT_RANGE(y, bits) RPyAssert(y >= 0 && y < bits, \ + "The shift count is outside of the supported range") -#define OP_INT_LSHIFT(x,y,r) r = (x) << (y) -#define OP_UINT_LSHIFT(x,y,r) r = (x) << (y) -#define OP_LLONG_LSHIFT(x,y,r) r = (x) << (y) -#define OP_ULLONG_LSHIFT(x,y,r) r = (x) << (y) + +#define OP_INT_RSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \ + r = Py_ARITHMETIC_RIGHT_SHIFT(long, x, (y)) +#define OP_UINT_RSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \ + r = (x) >> (y) +#define OP_LLONG_RSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \ + r = Py_ARITHMETIC_RIGHT_SHIFT(PY_LONG_LONG,x, (y)) +#define OP_ULLONG_RSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \ + r = (x) >> (y) + + +#define OP_INT_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \ + r = (x) << (y) +#define OP_UINT_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \ + r = (x) << (y) +#define OP_LLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \ + r = (x) << (y) +#define OP_ULLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \ + r = (x) << (y) #define OP_INT_LSHIFT_OVF(x,y,r) \ OP_INT_LSHIFT(x,y,r); \ diff --git a/pypy/translator/c/src/mem.h b/pypy/translator/c/src/mem.h --- a/pypy/translator/c/src/mem.h +++ b/pypy/translator/c/src/mem.h @@ -222,6 +222,15 @@ #endif /* USING_BOEHM_GC */ + +#ifdef USING_NO_GC_AT_ALL +#define OP_BOEHM_ZERO_MALLOC(size, r, restype, is_atomic, is_varsize) \ + r = (restype) calloc(1, size); +#define OP_BOEHM_DISAPPEARING_LINK(link, obj, r) /* nothing */ +#define OP_GC__DISABLE_FINALIZERS(r) /* nothing */ +#define OP_GC__ENABLE_FINALIZERS(r) /* nothing */ +#endif + /************************************************************/ /* weakref support */ diff --git a/pypy/translator/c/test/test_standalone.py b/pypy/translator/c/test/test_standalone.py --- a/pypy/translator/c/test/test_standalone.py +++ b/pypy/translator/c/test/test_standalone.py @@ -596,6 +596,42 @@ # The traceback stops at f() because it's the first function that # captures the AssertionError, which makes the program abort. + def test_int_lshift_too_large(self): + from pypy.rlib.rarithmetic import LONG_BIT, LONGLONG_BIT + def entry_point(argv): + a = int(argv[1]) + b = int(argv[2]) + print a << b + return 0 + + t, cbuilder = self.compile(entry_point, debug=True) + out = cbuilder.cmdexec("10 2", expect_crash=False) + assert out.strip() == str(10 << 2) + cases = [-4, LONG_BIT, LONGLONG_BIT] + for x in cases: + out, err = cbuilder.cmdexec("%s %s" % (1, x), expect_crash=True) + lines = err.strip() + assert 'The shift count is outside of the supported range' in lines + + def test_llong_rshift_too_large(self): + from pypy.rlib.rarithmetic import LONG_BIT, LONGLONG_BIT + def entry_point(argv): + a = r_longlong(int(argv[1])) + b = r_longlong(int(argv[2])) + print a >> b + return 0 + + t, cbuilder = self.compile(entry_point, debug=True) + out = cbuilder.cmdexec("10 2", expect_crash=False) + assert out.strip() == str(10 >> 2) + out = cbuilder.cmdexec("%s %s" % (-42, LONGLONG_BIT - 1), expect_crash=False) + assert out.strip() == '-1' + cases = [-4, LONGLONG_BIT] + for x in cases: + out, err = cbuilder.cmdexec("%s %s" % (1, x), expect_crash=True) + lines = err.strip() + assert 'The shift count is outside of the supported range' in lines + def test_ll_assert_error_debug(self): def entry_point(argv): ll_assert(len(argv) != 1, "foobar") diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py --- a/pypy/translator/goal/translate.py +++ b/pypy/translator/goal/translate.py @@ -149,6 +149,9 @@ log.ERROR("Could not find target %r" % (arg, )) sys.exit(1) + # apply the platform settings + set_platform(config) + targetspec = translateconfig.targetspec targetspec_dic = load_target(targetspec) @@ -164,9 +167,6 @@ existing_config=config, translating=True) - # apply the platform settings - set_platform(config) - # apply the optimization level settings set_opt_level(config, translateconfig.opt) diff --git a/pypy/translator/platform/__init__.py b/pypy/translator/platform/__init__.py --- a/pypy/translator/platform/__init__.py +++ b/pypy/translator/platform/__init__.py @@ -38,6 +38,7 @@ c_environ = None relevant_environ = () + log_errors = True so_prefixes = ('',) @@ -120,11 +121,12 @@ if returncode != 0: errorfile = outname.new(ext='errors') errorfile.write(stderr, 'wb') - stderrlines = stderr.splitlines() - for line in stderrlines: - log.Error(line) - # ^^^ don't use ERROR, because it might actually be fine. - # Also, ERROR confuses lib-python/conftest.py. + if self.log_errors: + stderrlines = stderr.splitlines() + for line in stderrlines: + log.Error(line) + # ^^^ don't use ERROR, because it might actually be fine. + # Also, ERROR confuses lib-python/conftest.py. raise CompilationError(stdout, stderr) else: for line in stderr.splitlines(): _______________________________________________ pypy-commit mailing list pypy-commit@python.org http://mail.python.org/mailman/listinfo/pypy-commit