Author: Richard Plangger <[email protected]>
Branch: new-jit-log
Changeset: r85809:755792b503d9
Date: 2016-07-22 10:46 +0200
http://bitbucket.org/pypy/pypy/changeset/755792b503d9/
Log: refactor jitlog and unwind it into a new module
diff --git a/pypy/module/_vmprof/interp_vmprof.py
b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -73,7 +73,7 @@
def enable_jitlog(space, fileno):
""" Enable PyPy's logging facility. """
try:
- rvmprof.enable_jitlog(fileno)
+ rjitlog.enable_jitlog(fileno)
except rvmprof.VMProfError, e:
raise VMProfError(space, e)
diff --git a/rpython/jit/backend/llsupport/assembler.py
b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -13,7 +13,7 @@
from rpython.rlib.jitlog import _log_jit_counter
from rpython.rtyper.annlowlevel import cast_instance_to_gcref, llhelper
from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib.rvmprof.rvmprof import _get_vmprof
+from rpython.rlib.rjitlog import rjitlog
DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER',
# 'b'ridge, 'l'abel or # 'e'ntry point
@@ -347,7 +347,7 @@
@specialize.argtype(1)
def _inject_debugging_code(self, looptoken, operations, tp, number):
- if self._debug or _get_vmprof().cintf.jitlog_enabled():
+ if self._debug or rjitlog.jitlog_enabled():
newoperations = []
self._append_debugging_code(newoperations, tp, number, None)
for op in operations:
diff --git a/rpython/jit/metainterp/compile.py
b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -6,7 +6,7 @@
from rpython.rlib.rarithmetic import r_uint, intmask
from rpython.rlib import rstack
from rpython.rlib.jit import JitDebugInfo, Counters, dont_look_inside
-from rpython.rlib.rvmprof.rvmprof import _get_vmprof
+from rpython.rlib.rjitlog import rjitlog
from rpython.conftest import option
from rpython.jit.metainterp.resoperation import ResOperation, rop,\
@@ -545,7 +545,7 @@
operations = get_deep_immutable_oplist(loop.operations)
metainterp_sd.profiler.start_backend()
debug_start("jit-backend")
- log = have_debug_prints() or _get_vmprof().cintf.jitlog_enabled()
+ log = have_debug_prints() or rjitlog.jitlog_enabled()
try:
loopname = jitdriver_sd.warmstate.get_location_str(greenkey)
unique_id = jitdriver_sd.warmstate.get_unique_id(greenkey)
@@ -597,7 +597,7 @@
operations = get_deep_immutable_oplist(operations)
metainterp_sd.profiler.start_backend()
debug_start("jit-backend")
- log = have_debug_prints() or _get_vmprof().cintf.jitlog_enabled()
+ log = have_debug_prints() or rjitlog.jitlog_enabled()
try:
asminfo = do_compile_bridge(metainterp_sd, faildescr, inputargs,
operations,
diff --git a/rpython/rlib/jitlog.py b/rpython/rlib/jitlog.py
deleted file mode 100644
--- a/rpython/rlib/jitlog.py
+++ /dev/null
@@ -1,551 +0,0 @@
-import sys
-import weakref
-import struct
-import os
-
-from rpython.rlib.rvmprof.rvmprof import _get_vmprof
-from rpython.jit.metainterp import resoperation as resoperations
-from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.metainterp.history import ConstInt, ConstFloat
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rlib.objectmodel import compute_unique_id, always_inline
-from rpython.rlib.objectmodel import we_are_translated, specialize
-from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.jit_hooks import register_helper
-from rpython.annotator import model as annmodel
-
-@register_helper(None)
-def stats_flush_trace_counts(warmrunnerdesc):
- warmrunnerdesc.metainterp_sd.cpu.assembler.flush_trace_counters()
-
-def commonprefix(a,b):
- "Given a list of pathnames, returns the longest common leading component"
- assert a is not None
- assert b is not None
- la = len(a)
- lb = len(b)
- c = min(la,lb)
- if c == 0:
- return ""
- for i in range(c):
- if a[i] != b[i]:
- return a[:i] # partly matching
- return a # full match
-
-@always_inline
-def encode_str(string):
- return encode_le_32bit(len(string)) + string
-
-@always_inline
-def encode_le_16bit(val):
- return chr((val >> 0) & 0xff) + chr((val >> 8) & 0xff)
-
-@always_inline
-def encode_le_32bit(val):
- return ''.join([chr((val >> 0) & 0xff),
- chr((val >> 8) & 0xff),
- chr((val >> 16) & 0xff),
- chr((val >> 24) & 0xff)])
-
-@always_inline
-def encode_le_64bit(val):
- return ''.join([chr((val >> 0) & 0xff),
- chr((val >> 8) & 0xff),
- chr((val >> 16) & 0xff),
- chr((val >> 24) & 0xff),
- chr((val >> 32) & 0xff),
- chr((val >> 40) & 0xff),
- chr((val >> 48) & 0xff),
- chr((val >> 56)& 0xff)])
-
-@always_inline
-def encode_le_addr(val):
- if IS_32_BIT:
- return encode_le_32bit(val)
- else:
- return encode_le_64bit(val)
-
-def encode_type(type, value):
- if type == "s":
- return encode_str(value)
- elif type == "q":
- return encode_le_64bit(value)
- elif type == "i":
- return encode_le_32bit(value)
- elif type == "h":
- return encode_le_32bit(value)
- else:
- raise NotImplementedError
-
-
-# more variable parameters
-MP_STR = (0x0, "s")
-MP_INT = (0x0, "i")
-
-# concrete parameters
-MP_FILENAME = (0x1, "s")
-MP_LINENO = (0x2, "i")
-MP_INDEX = (0x4, "i")
-MP_SCOPE = (0x8, "s")
-MP_OPCODE = (0x10, "s")
-
-class WrappedValue(object):
- def encode(self, log, i, compressor):
- raise NotImplementedError
-
-class StringValue(WrappedValue):
- def __init__(self, sem_type, gen_type, value):
- self.value = value
-
- def encode(self, log, i, compressor):
- str_value = self.value
- last_prefix = compressor.get_last_written(i)
- cp = compressor.compress(i, str_value)
- if cp is None:
- return b'\xff' + encode_str(str_value)
-
- else:
- cp_len = len(cp)
- if cp == last_prefix:
- # we have the same prefix
- pass
- else:
- compressor.write(log, i, cp)
- if len(str_value) == len(cp):
- return b'\xef'
- return b'\x00' + encode_str(str_value[len(cp):])
-
-class IntValue(WrappedValue):
- def __init__(self, sem_type, gen_type, value):
- self.value = value
-
- def encode(self, log, i, prefixes):
- return b'\x00' + encode_le_64bit(self.value)
-
-# note that a ...
-# "semantic_type" is an integer denoting which meaning does a type at a merge
point have
-# there are very common ones that are predefined. E.g.
MP_FILENAME
-# "generic_type" is one of the primitive types supported (string,int)
-
[email protected](2)
-def wrap(sem_type, gen_type, value):
- if isinstance(value, int):
- return IntValue(sem_type, gen_type, value)
- elif isinstance(value, str):
- return StringValue(sem_type, gen_type, value)
- raise NotImplementedError
-
-def returns(*args):
- """ Decorate your get_location function to specify the types.
- Use MP_* constant as parameters. An example impl for get_location
- would return the following:
-
- @returns(MP_FILENAME, MP_LINENO)
- def get_location(...):
- return ("a.py", 0)
- """
- def decor(method):
- method._loc_types = args
- return method
- return decor
-
-JITLOG_VERSION = 1
-JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
-
-marks = [
- ('INPUT_ARGS',),
- ('RESOP_META',),
- ('RESOP',),
- ('RESOP_DESCR',),
- ('ASM_ADDR',),
- ('ASM',),
-
- # which type of trace is logged after this
- # the trace as it is recorded by the tracer
- ('TRACE',),
- # the trace that has passed the optimizer
- ('TRACE_OPT',),
- # the trace assembled to machine code (after rewritten)
- ('TRACE_ASM',),
-
- # the machine code was patched (e.g. guard)
- ('STITCH_BRIDGE',),
-
- ('START_TRACE',),
-
- ('JITLOG_COUNTER',),
- ('INIT_MERGE_POINT',),
-
- ('JITLOG_HEADER',),
- ('MERGE_POINT',),
- ('COMMON_PREFIX',),
- ('ABORT_TRACE',),
- ('SOURCE_CODE',),
-]
-
-start = 0x11
-for mark, in marks:
- globals()['MARK_' + mark] = chr(start)
- start += 1
-
-if __name__ == "__main__":
- print("# generated constants from rpython/rlib/jitlog.py")
- print 'MARK_JITLOG_START = struct.pack("b", "%s")' % hex(0x10)
- for mark, in marks:
- nmr = globals()['MARK_' + mark]
- h = hex(ord(nmr))
- print '%s = struct.pack("b", "%s")' % ('MARK_' + mark, h)
- print 'MARK_JITLOG_END = struct.pack("b", "%s")' % hex(start)
- for key,value in locals().items():
- if key.startswith("MP_"):
- print '%s = (%s,"%s")' % (key, hex(value[0]), value[1])
- print 'SEM_TYPE_NAMES = {'
- for key,value in locals().items():
- if key.startswith("MP_") and value[0] != 0:
- print ' %s: "%s",' % (hex(value[0]), key[3:].lower())
- print '}'
-
-MP_STR = (0x0, "s")
-MP_INT = (0x0, "i")
-
-# concrete parameters
-MP_FILENAME = (0x1, "s")
-MP_LINENO = (0x2, "i")
-MP_INDEX = (0x4, "i")
-MP_SCOPE = (0x8, "s")
-MP_OPCODE = (0x10, "s")
-
-del marks
-del start
-
-IS_32_BIT = sys.maxint == 2**31-1
-
-def assemble_header():
- version = JITLOG_VERSION_16BIT_LE
- count = len(resoperations.opname)
- content = [version, MARK_RESOP_META,
- encode_le_16bit(count)]
- for opnum, opname in resoperations.opname.items():
- content.append(encode_le_16bit(opnum))
- content.append(encode_str(opname.lower()))
- return ''.join(content)
-
-def _log_jit_counter(struct):
- cintf = _get_vmprof().cintf
- if not cintf.jitlog_enabled():
- return
- # addr is either a number (trace_id), or the address
- # of the descriptor. for entries it is a the trace_id,
- # for any label/bridge entry the addr is the address
- list = [MARK_JITLOG_COUNTER, encode_le_addr(struct.number),
- struct.type, encode_le_64bit(struct.i)]
- content = ''.join(list)
- cintf.jitlog_write_marked(content, len(content))
-
-class VMProfJitLogger(object):
- def __init__(self, cpu=None):
- self.cpu = cpu
- self.memo = {}
- self.trace_id = -1
- self.metainterp_sd = None
- self.cintf = _get_vmprof().cintf
- # legacy
- self.logger_ops = None
- self.logger_noopt = None
-
- def setup_once(self):
- if self.cintf.jitlog_enabled():
- return
- self.cintf.jitlog_try_init_using_env()
- if not self.cintf.jitlog_enabled():
- return
- blob = assemble_header()
- self.cintf.jitlog_write_marked(MARK_JITLOG_HEADER + blob, len(blob) +
1)
-
- def finish(self):
- self.cintf.jitlog_teardown()
-
- def start_new_trace(self, metainterp_sd, faildescr=None,
entry_bridge=False):
- # even if the logger is not enabled, increment the trace id
- self.trace_id += 1
- if not self.cintf.jitlog_enabled():
- return
- self.metainterp_sd = metainterp_sd
- content = [encode_le_addr(self.trace_id)]
- if faildescr:
- content.append(encode_str('bridge'))
- descrnmr = compute_unique_id(faildescr)
- content.append(encode_le_addr(descrnmr))
- else:
- content.append(encode_str('loop'))
- content.append(encode_le_addr(int(entry_bridge)))
- self._write_marked(MARK_START_TRACE, ''.join(content))
-
- def trace_aborted(self):
- if not self.cintf.jitlog_enabled():
- return
- self._write_marked(MARK_ABORT_TRACE, encode_le_64bit(self.trace_id))
-
- def _write_marked(self, mark, line):
- if not we_are_translated():
- assert self.cintf.jitlog_enabled()
- self.cintf.jitlog_write_marked(mark + line, len(line) + 1)
-
- def log_jit_counter(self, struct):
- _log_jit_counter(self.cintf, struct)
-
- def log_trace(self, tag, metainterp_sd, mc, memo=None):
- if not self.cintf.jitlog_enabled():
- return EMPTY_TRACE_LOG
- assert self.metainterp_sd is not None
- if memo is None:
- memo = {}
- return LogTrace(tag, memo, self.metainterp_sd, mc, self)
-
- def log_patch_guard(self, descr_number, addr):
- if not self.cintf.jitlog_enabled():
- return
- le_descr_number = encode_le_addr(descr_number)
- le_addr = encode_le_addr(addr)
- lst = [le_descr_number, le_addr]
- self._write_marked(MARK_STITCH_BRIDGE, ''.join(lst))
-
-class BaseLogTrace(object):
- def write_trace(self, trace):
- return None
-
- def write(self, args, ops, ops_offset={}):
- return None
-
-EMPTY_TRACE_LOG = BaseLogTrace()
-
-class PrefixCompressor(object):
- def __init__(self, count):
- self.prefixes = [None] * count
- self.written_prefixes = [None] * count
-
- def get_last(self, index):
- return self.prefixes[index]
-
- def get_last_written(self, index):
- return self.written_prefixes[index]
-
- def compress(self, index, string):
- assert string is not None
- last = self.get_last(index)
- if last is None:
- self.prefixes[index] = string
- return None
- cp = commonprefix(last, string)
- if len(cp) <= 1: # prevent very small common prefixes (like "/")
- self.prefixes[index] = string
- return None
- return cp
-
-
- def write(self, log, index, prefix):
- # we have a new prefix
- log._write_marked(MARK_COMMON_PREFIX, chr(index) \
- + encode_str(prefix))
- self.written_prefixes[index] = prefix
-
-def encode_merge_point(log, compressor, values):
- line = []
- unrolled = unrolling_iterable(values)
- i = 0
- for value in unrolled:
- line.append(value.encode(log,i,compressor))
- i += 1
- return ''.join(line)
-
-
-class LogTrace(BaseLogTrace):
- def __init__(self, tag, memo, metainterp_sd, mc, logger):
- self.memo = memo
- self.metainterp_sd = metainterp_sd
- self.ts = None
- if self.metainterp_sd is not None:
- self.ts = metainterp_sd.cpu.ts
- self.tag = tag
- self.mc = mc
- self.logger = logger
- self.common_prefix = None
-
- def write_trace(self, trace):
- ops = []
- i = trace.get_iter()
- while not i.done():
- ops.append(i.next())
- self.write(i.inputargs, ops)
-
- def write(self, args, ops, ops_offset={}):
- log = self.logger
- log._write_marked(self.tag, encode_le_64bit(self.logger.trace_id))
-
- # input args
- str_args = [self.var_to_str(arg) for arg in args]
- string = encode_str(','.join(str_args))
- log._write_marked(MARK_INPUT_ARGS, string)
-
- # assembler address (to not duplicate it in write_code_dump)
- if self.mc is not None:
- absaddr = self.mc.absolute_addr()
- rel = self.mc.get_relative_pos()
- # packs <start addr> <end addr> as two unsigend longs
- le_addr1 = encode_le_addr(absaddr)
- le_addr2 = encode_le_addr(absaddr + rel)
- log._write_marked(MARK_ASM_ADDR, le_addr1 + le_addr2)
- for i,op in enumerate(ops):
- if rop.DEBUG_MERGE_POINT == op.getopnum():
- self.encode_debug_info(op)
- continue
- mark, line = self.encode_op(op)
- log._write_marked(mark, line)
- self.write_core_dump(ops, i, op, ops_offset)
-
- self.memo = {}
-
- def encode_once(self):
- pass
-
- def encode_debug_info(self, op):
- # the idea is to write the debug merge point as it's own well known
- # tag. Compression for common prefixes is implemented:
-
- log = self.logger
- jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()]
- if not jd_sd.warmstate.get_location:
- return
- values = jd_sd.warmstate.get_location(op.getarglist()[3:])
- if values is None:
- # indicates that this function is not provided to the jit driver
- return
- types = jd_sd.warmstate.get_location_types
-
- if self.common_prefix is None:
- # first time visiting a merge point
- # setup the common prefix
- self.common_prefix = PrefixCompressor(len(types))
- encoded_types = []
- for i, (semantic_type, generic_type) in enumerate(types):
- encoded_types.append(chr(semantic_type))
- encoded_types.append(generic_type)
- count = encode_le_16bit(len(types))
- log._write_marked(MARK_INIT_MERGE_POINT, count +
''.join(encoded_types))
-
- # the types have already been written
- encoded = encode_merge_point(log, self.common_prefix, values)
- log._write_marked(MARK_MERGE_POINT, encoded)
-
- def encode_op(self, op):
- """ an operation is written as follows:
- <marker> <opid (16 bit)> \
- <len (32 bit)> \
- <res_val>,<arg_0>,...,<arg_n>,<descr>
- The marker indicates if the last argument is
- a descr or a normal argument.
- """
- str_args = [self.var_to_str(arg) for arg in op.getarglist()]
- descr = op.getdescr()
- le_opnum = encode_le_16bit(op.getopnum())
- str_res = self.var_to_str(op)
- line = ','.join([str_res] + str_args)
- if descr:
- descr_str = descr.repr_of_descr()
- line = line + ',' + descr_str
- string = encode_str(line)
- descr_number = compute_unique_id(descr)
- le_descr_number = encode_le_addr(descr_number)
- return MARK_RESOP_DESCR, le_opnum + string + le_descr_number
- else:
- string = encode_str(line)
- return MARK_RESOP, le_opnum + string
-
-
- def write_core_dump(self, operations, i, op, ops_offset):
- if self.mc is None:
- return
-
- op2 = None
- j = i+1
- # find the next op that is in the offset hash
- while j < len(operations):
- op2 = operations[j]
- if op in ops_offset:
- break
- j += 1
-
- # this op has no known offset in the machine code (it might be
- # a debug operation)
- if op not in ops_offset:
- return
- # there is no well defined boundary for the end of the
- # next op in the assembler
- if op2 is not None and op2 not in ops_offset:
- return
- dump = []
-
- start_offset = ops_offset[op]
- assert start_offset >= 0
- # end offset is either the last pos in the assembler
- # or the offset of op2
- if op2 is None:
- end_offset = self.mc.get_relative_pos()
- else:
- end_offset = ops_offset[op2]
-
- count = end_offset - start_offset
- dump = self.copy_core_dump(self.mc.absolute_addr(), start_offset,
count)
- offset = encode_le_16bit(start_offset)
- edump = encode_str(dump)
- self.logger._write_marked(MARK_ASM, offset + edump)
-
- def copy_core_dump(self, addr, offset=0, count=-1):
- dump = []
- src = rffi.cast(rffi.CCHARP, addr)
- end = self.mc.get_relative_pos()
- if count != -1:
- end = offset + count
- for p in range(offset, end):
- dump.append(src[p])
- return ''.join(dump)
-
- def var_to_str(self, arg):
- try:
- mv = self.memo[arg]
- except KeyError:
- mv = len(self.memo)
- self.memo[arg] = mv
- if isinstance(arg, ConstInt):
- if self.metainterp_sd and int_could_be_an_address(arg.value):
- addr = arg.getaddr()
- name = self.metainterp_sd.get_name_from_address(addr)
- if name:
- return 'ConstClass(' + name + ')'
- return str(arg.value)
- elif self.ts is not None and isinstance(arg, self.ts.ConstRef):
- if arg.value:
- return 'ConstPtr(ptr' + str(mv) + ')'
- return 'ConstPtr(null)'
- if isinstance(arg, ConstFloat):
- return str(arg.getfloat())
- elif arg is None:
- return 'None'
- elif arg.is_vector():
- return 'v' + str(mv)
- elif arg.type == 'i':
- return 'i' + str(mv)
- elif arg.type == 'r':
- return 'p' + str(mv)
- elif arg.type == 'f':
- return 'f' + str(mv)
- else:
- return '?'
-
-def int_could_be_an_address(x):
- if we_are_translated():
- x = rffi.cast(lltype.Signed, x) # force it
- return not (-32768 <= x <= 32767)
- else:
- return isinstance(x, llmemory.AddressAsInt)
diff --git a/rpython/rlib/rjitlog/__init__.py b/rpython/rlib/rjitlog/__init__.py
new file mode 100644
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -0,0 +1,605 @@
+import py
+import sys
+import sys
+import weakref
+import struct
+import os
+from rpython.rlib import jit
+from rpython.tool.udir import udir
+from rpython.tool.version import rpythonroot
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.jit.metainterp import resoperation as resoperations
+from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.history import ConstInt, ConstFloat
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rlib.objectmodel import compute_unique_id, always_inline
+from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.jit_hooks import register_helper
+from rpython.annotator import model as annmodel
+
+
+ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rjitlog')
+SRC = ROOT.join('src')
+
+_libs = []
+if sys.platform.startswith('linux'):
+ _libs = ['dl']
+eci_kwds = dict(
+ include_dirs = [SRC],
+ includes = ['rjitlog.h'],
+ libraries = _libs,
+ separate_module_files = [SRC.join('rjitlog.c')],
+ post_include_bits=['#define RPYTHON_JITLOG\n'],
+ )
+eci = ExternalCompilationInfo(**eci_kwds)
+
+# jit log functions
+jitlog_init = rffi.llexternal("jitlog_init", [rffi.INT],
+ rffi.CCHARP, compilation_info=eci)
+jitlog_try_init_using_env = rffi.llexternal("jitlog_try_init_using_env",
+ [], lltype.Void, compilation_info=eci)
+jitlog_write_marked = rffi.llexternal("jitlog_write_marked",
+ [rffi.CCHARP, rffi.INT],
+ lltype.Void, compilation_info=eci,
+ releasegil=False)
+jitlog_enabled = rffi.llexternal("jitlog_enabled", [], rffi.INT,
+ compilation_info=eci,
+ releasegil=False)
+jitlog_teardown = rffi.llexternal("jitlog_teardown", [], lltype.Void,
+ compilation_info=eci)
+
+class JitlogError(Exception):
+ def __init__(self, msg):
+ self.msg = msg
+ def __str__(self):
+ return self.msg
+
[email protected]_look_inside
+def enable_jitlog(fileno):
+ # initialize the jit log
+ from rpython.rlib import jitlog as jl
+ p_error = jitlog_init(fileno)
+ if p_error:
+ raise JitlogError(rffi.charp2str(p_error))
+ blob = jl.assemble_header()
+ jitlog_write_marked(jl.MARK_JITLOG_HEADER + blob, len(blob) + 1)
+
+def disable_jitlog():
+ from rpython.rlib.jitlog import stats_flush_trace_counts
+ stats_flush_trace_counts(None)
+ jitlog_teardown()
+
+@register_helper(None)
+def stats_flush_trace_counts(warmrunnerdesc):
+ warmrunnerdesc.metainterp_sd.cpu.assembler.flush_trace_counters()
+
+def commonprefix(a,b):
+ "Given a list of pathnames, returns the longest common leading component"
+ assert a is not None
+ assert b is not None
+ la = len(a)
+ lb = len(b)
+ c = min(la,lb)
+ if c == 0:
+ return ""
+ for i in range(c):
+ if a[i] != b[i]:
+ return a[:i] # partly matching
+ return a # full match
+
+@always_inline
+def encode_str(string):
+ return encode_le_32bit(len(string)) + string
+
+@always_inline
+def encode_le_16bit(val):
+ return chr((val >> 0) & 0xff) + chr((val >> 8) & 0xff)
+
+@always_inline
+def encode_le_32bit(val):
+ return ''.join([chr((val >> 0) & 0xff),
+ chr((val >> 8) & 0xff),
+ chr((val >> 16) & 0xff),
+ chr((val >> 24) & 0xff)])
+
+@always_inline
+def encode_le_64bit(val):
+ return ''.join([chr((val >> 0) & 0xff),
+ chr((val >> 8) & 0xff),
+ chr((val >> 16) & 0xff),
+ chr((val >> 24) & 0xff),
+ chr((val >> 32) & 0xff),
+ chr((val >> 40) & 0xff),
+ chr((val >> 48) & 0xff),
+ chr((val >> 56)& 0xff)])
+
+@always_inline
+def encode_le_addr(val):
+ if IS_32_BIT:
+ return encode_le_32bit(val)
+ else:
+ return encode_le_64bit(val)
+
+def encode_type(type, value):
+ if type == "s":
+ return encode_str(value)
+ elif type == "q":
+ return encode_le_64bit(value)
+ elif type == "i":
+ return encode_le_32bit(value)
+ elif type == "h":
+ return encode_le_32bit(value)
+ else:
+ raise NotImplementedError
+
+# more variable parameters
+MP_STR = (0x0, "s")
+MP_INT = (0x0, "i")
+
+# concrete parameters
+MP_FILENAME = (0x1, "s")
+MP_LINENO = (0x2, "i")
+MP_INDEX = (0x4, "i")
+MP_SCOPE = (0x8, "s")
+MP_OPCODE = (0x10, "s")
+
+class WrappedValue(object):
+ def encode(self, log, i, compressor):
+ raise NotImplementedError
+
+class StringValue(WrappedValue):
+ def __init__(self, sem_type, gen_type, value):
+ self.value = value
+
+ def encode(self, log, i, compressor):
+ str_value = self.value
+ last_prefix = compressor.get_last_written(i)
+ cp = compressor.compress(i, str_value)
+ if cp is None:
+ return b'\xff' + encode_str(str_value)
+
+ else:
+ cp_len = len(cp)
+ if cp == last_prefix:
+ # we have the same prefix
+ pass
+ else:
+ compressor.write(log, i, cp)
+ if len(str_value) == len(cp):
+ return b'\xef'
+ return b'\x00' + encode_str(str_value[len(cp):])
+
+class IntValue(WrappedValue):
+ def __init__(self, sem_type, gen_type, value):
+ self.value = value
+
+ def encode(self, log, i, prefixes):
+ return b'\x00' + encode_le_64bit(self.value)
+
+# note that a ...
+# "semantic_type" is an integer denoting which meaning does a type at a merge
point have
+# there are very common ones that are predefined. E.g.
MP_FILENAME
+# "generic_type" is one of the primitive types supported (string,int)
+
[email protected](2)
+def wrap(sem_type, gen_type, value):
+ if isinstance(value, int):
+ return IntValue(sem_type, gen_type, value)
+ elif isinstance(value, str):
+ return StringValue(sem_type, gen_type, value)
+ raise NotImplementedError
+
+def returns(*args):
+ """ Decorate your get_location function to specify the types.
+ Use MP_* constant as parameters. An example impl for get_location
+ would return the following:
+
+ @returns(MP_FILENAME, MP_LINENO)
+ def get_location(...):
+ return ("a.py", 0)
+ """
+ def decor(method):
+ method._loc_types = args
+ return method
+ return decor
+
+JITLOG_VERSION = 1
+JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
+
+marks = [
+ ('INPUT_ARGS',),
+ ('RESOP_META',),
+ ('RESOP',),
+ ('RESOP_DESCR',),
+ ('ASM_ADDR',),
+ ('ASM',),
+
+ # which type of trace is logged after this
+ # the trace as it is recorded by the tracer
+ ('TRACE',),
+ # the trace that has passed the optimizer
+ ('TRACE_OPT',),
+ # the trace assembled to machine code (after rewritten)
+ ('TRACE_ASM',),
+
+ # the machine code was patched (e.g. guard)
+ ('STITCH_BRIDGE',),
+
+ ('START_TRACE',),
+
+ ('JITLOG_COUNTER',),
+ ('INIT_MERGE_POINT',),
+
+ ('JITLOG_HEADER',),
+ ('MERGE_POINT',),
+ ('COMMON_PREFIX',),
+ ('ABORT_TRACE',),
+ ('SOURCE_CODE',),
+]
+
+start = 0x11
+for mark, in marks:
+ globals()['MARK_' + mark] = chr(start)
+ start += 1
+
+if __name__ == "__main__":
+ print("# generated constants from rpython/rlib/jitlog.py")
+ print 'MARK_JITLOG_START = struct.pack("b", "%s")' % hex(0x10)
+ for mark, in marks:
+ nmr = globals()['MARK_' + mark]
+ h = hex(ord(nmr))
+ print '%s = struct.pack("b", "%s")' % ('MARK_' + mark, h)
+ print 'MARK_JITLOG_END = struct.pack("b", "%s")' % hex(start)
+ for key,value in locals().items():
+ if key.startswith("MP_"):
+ print '%s = (%s,"%s")' % (key, hex(value[0]), value[1])
+ print 'SEM_TYPE_NAMES = {'
+ for key,value in locals().items():
+ if key.startswith("MP_") and value[0] != 0:
+ print ' %s: "%s",' % (hex(value[0]), key[3:].lower())
+ print '}'
+
+MP_STR = (0x0, "s")
+MP_INT = (0x0, "i")
+
+# concrete parameters
+MP_FILENAME = (0x1, "s")
+MP_LINENO = (0x2, "i")
+MP_INDEX = (0x4, "i")
+MP_SCOPE = (0x8, "s")
+MP_OPCODE = (0x10, "s")
+
+del marks
+del start
+
+IS_32_BIT = sys.maxint == 2**31-1
+
+def assemble_header():
+ version = JITLOG_VERSION_16BIT_LE
+ count = len(resoperations.opname)
+ content = [version, MARK_RESOP_META,
+ encode_le_16bit(count)]
+ for opnum, opname in resoperations.opname.items():
+ content.append(encode_le_16bit(opnum))
+ content.append(encode_str(opname.lower()))
+ return ''.join(content)
+
+def _log_jit_counter(struct):
+ if not jitlog_enabled():
+ return
+ # addr is either a number (trace_id), or the address
+ # of the descriptor. for entries it is a the trace_id,
+ # for any label/bridge entry the addr is the address
+ list = [MARK_JITLOG_COUNTER, encode_le_addr(struct.number),
+ struct.type, encode_le_64bit(struct.i)]
+ content = ''.join(list)
+ jitlog_write_marked(content, len(content))
+
+class JitLogger(object):
+ def __init__(self, cpu=None):
+ self.cpu = cpu
+ self.memo = {}
+ self.trace_id = -1
+ self.metainterp_sd = None
+ # legacy
+ self.logger_ops = None
+ self.logger_noopt = None
+
+ def setup_once(self):
+ if jitlog_enabled():
+ return
+ jitlog_try_init_using_env()
+ if not jitlog_enabled():
+ return
+ blob = assemble_header()
+ jitlog_write_marked(MARK_JITLOG_HEADER + blob, len(blob) + 1)
+
+ def finish(self):
+ jitlog_teardown()
+
+ def start_new_trace(self, metainterp_sd, faildescr=None,
entry_bridge=False):
+ # even if the logger is not enabled, increment the trace id
+ self.trace_id += 1
+ if not jitlog_enabled():
+ return
+ self.metainterp_sd = metainterp_sd
+ content = [encode_le_addr(self.trace_id)]
+ if faildescr:
+ content.append(encode_str('bridge'))
+ descrnmr = compute_unique_id(faildescr)
+ content.append(encode_le_addr(descrnmr))
+ else:
+ content.append(encode_str('loop'))
+ content.append(encode_le_addr(int(entry_bridge)))
+ self._write_marked(MARK_START_TRACE, ''.join(content))
+
+ def trace_aborted(self):
+ if not jitlog_enabled():
+ return
+ self._write_marked(MARK_ABORT_TRACE, encode_le_64bit(self.trace_id))
+
+ def _write_marked(self, mark, line):
+ if not we_are_translated():
+ assert jitlog_enabled()
+ jitlog_write_marked(mark + line, len(line) + 1)
+
+ def log_jit_counter(self, struct):
+ _log_jit_counter(struct)
+
+ def log_trace(self, tag, metainterp_sd, mc, memo=None):
+ if not jitlog_enabled():
+ return EMPTY_TRACE_LOG
+ assert self.metainterp_sd is not None
+ if memo is None:
+ memo = {}
+ return LogTrace(tag, memo, self.metainterp_sd, mc, self)
+
+ def log_patch_guard(self, descr_number, addr):
+ if not jitlog_enabled():
+ return
+ le_descr_number = encode_le_addr(descr_number)
+ le_addr = encode_le_addr(addr)
+ lst = [le_descr_number, le_addr]
+ self._write_marked(MARK_STITCH_BRIDGE, ''.join(lst))
+
+class BaseLogTrace(object):
+ def write_trace(self, trace):
+ return None
+
+ def write(self, args, ops, ops_offset={}):
+ return None
+
+EMPTY_TRACE_LOG = BaseLogTrace()
+
+class PrefixCompressor(object):
+ def __init__(self, count):
+ self.prefixes = [None] * count
+ self.written_prefixes = [None] * count
+
+ def get_last(self, index):
+ return self.prefixes[index]
+
+ def get_last_written(self, index):
+ return self.written_prefixes[index]
+
+ def compress(self, index, string):
+ assert string is not None
+ last = self.get_last(index)
+ if last is None:
+ self.prefixes[index] = string
+ return None
+ cp = commonprefix(last, string)
+ if len(cp) <= 1: # prevent very small common prefixes (like "/")
+ self.prefixes[index] = string
+ return None
+ return cp
+
+
+ def write(self, log, index, prefix):
+ # we have a new prefix
+ log._write_marked(MARK_COMMON_PREFIX, chr(index) \
+ + encode_str(prefix))
+ self.written_prefixes[index] = prefix
+
+def encode_merge_point(log, compressor, values):
+ line = []
+ unrolled = unrolling_iterable(values)
+ i = 0
+ for value in unrolled:
+ line.append(value.encode(log,i,compressor))
+ i += 1
+ return ''.join(line)
+
+
+class LogTrace(BaseLogTrace):
+ def __init__(self, tag, memo, metainterp_sd, mc, logger):
+ self.memo = memo
+ self.metainterp_sd = metainterp_sd
+ self.ts = None
+ if self.metainterp_sd is not None:
+ self.ts = metainterp_sd.cpu.ts
+ self.tag = tag
+ self.mc = mc
+ self.logger = logger
+ self.common_prefix = None
+
+ def write_trace(self, trace):
+ ops = []
+ i = trace.get_iter()
+ while not i.done():
+ ops.append(i.next())
+ self.write(i.inputargs, ops)
+
+ def write(self, args, ops, ops_offset={}):
+ log = self.logger
+ log._write_marked(self.tag, encode_le_64bit(self.logger.trace_id))
+
+ # input args
+ str_args = [self.var_to_str(arg) for arg in args]
+ string = encode_str(','.join(str_args))
+ log._write_marked(MARK_INPUT_ARGS, string)
+
+ # assembler address (to not duplicate it in write_code_dump)
+ if self.mc is not None:
+ absaddr = self.mc.absolute_addr()
+ rel = self.mc.get_relative_pos()
+ # packs <start addr> <end addr> as two unsigend longs
+ le_addr1 = encode_le_addr(absaddr)
+ le_addr2 = encode_le_addr(absaddr + rel)
+ log._write_marked(MARK_ASM_ADDR, le_addr1 + le_addr2)
+ for i,op in enumerate(ops):
+ if rop.DEBUG_MERGE_POINT == op.getopnum():
+ self.encode_debug_info(op)
+ continue
+ mark, line = self.encode_op(op)
+ log._write_marked(mark, line)
+ self.write_core_dump(ops, i, op, ops_offset)
+
+ self.memo = {}
+
+ def encode_once(self):
+ pass
+
+ def encode_debug_info(self, op):
+ # the idea is to write the debug merge point as it's own well known
+ # tag. Compression for common prefixes is implemented:
+
+ log = self.logger
+ jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()]
+ if not jd_sd.warmstate.get_location:
+ return
+ values = jd_sd.warmstate.get_location(op.getarglist()[3:])
+ if values is None:
+ # indicates that this function is not provided to the jit driver
+ return
+ types = jd_sd.warmstate.get_location_types
+
+ if self.common_prefix is None:
+ # first time visiting a merge point
+ # setup the common prefix
+ self.common_prefix = PrefixCompressor(len(types))
+ encoded_types = []
+ for i, (semantic_type, generic_type) in enumerate(types):
+ encoded_types.append(chr(semantic_type))
+ encoded_types.append(generic_type)
+ count = encode_le_16bit(len(types))
+ log._write_marked(MARK_INIT_MERGE_POINT, count +
''.join(encoded_types))
+
+ # the types have already been written
+ encoded = encode_merge_point(log, self.common_prefix, values)
+ log._write_marked(MARK_MERGE_POINT, encoded)
+
+ def encode_op(self, op):
+ """ an operation is written as follows:
+ <marker> <opid (16 bit)> \
+ <len (32 bit)> \
+ <res_val>,<arg_0>,...,<arg_n>,<descr>
+ The marker indicates if the last argument is
+ a descr or a normal argument.
+ """
+ str_args = [self.var_to_str(arg) for arg in op.getarglist()]
+ descr = op.getdescr()
+ le_opnum = encode_le_16bit(op.getopnum())
+ str_res = self.var_to_str(op)
+ line = ','.join([str_res] + str_args)
+ if descr:
+ descr_str = descr.repr_of_descr()
+ line = line + ',' + descr_str
+ string = encode_str(line)
+ descr_number = compute_unique_id(descr)
+ le_descr_number = encode_le_addr(descr_number)
+ return MARK_RESOP_DESCR, le_opnum + string + le_descr_number
+ else:
+ string = encode_str(line)
+ return MARK_RESOP, le_opnum + string
+
+
+ def write_core_dump(self, operations, i, op, ops_offset):
+ if self.mc is None:
+ return
+
+ op2 = None
+ j = i+1
+ # find the next op that is in the offset hash
+ while j < len(operations):
+ op2 = operations[j]
+ if op in ops_offset:
+ break
+ j += 1
+
+ # this op has no known offset in the machine code (it might be
+ # a debug operation)
+ if op not in ops_offset:
+ return
+ # there is no well defined boundary for the end of the
+ # next op in the assembler
+ if op2 is not None and op2 not in ops_offset:
+ return
+ dump = []
+
+ start_offset = ops_offset[op]
+ assert start_offset >= 0
+ # end offset is either the last pos in the assembler
+ # or the offset of op2
+ if op2 is None:
+ end_offset = self.mc.get_relative_pos()
+ else:
+ end_offset = ops_offset[op2]
+
+ count = end_offset - start_offset
+ dump = self.copy_core_dump(self.mc.absolute_addr(), start_offset,
count)
+ offset = encode_le_16bit(start_offset)
+ edump = encode_str(dump)
+ self.logger._write_marked(MARK_ASM, offset + edump)
+
+ def copy_core_dump(self, addr, offset=0, count=-1):
+ dump = []
+ src = rffi.cast(rffi.CCHARP, addr)
+ end = self.mc.get_relative_pos()
+ if count != -1:
+ end = offset + count
+ for p in range(offset, end):
+ dump.append(src[p])
+ return ''.join(dump)
+
+ def var_to_str(self, arg):
+ try:
+ mv = self.memo[arg]
+ except KeyError:
+ mv = len(self.memo)
+ self.memo[arg] = mv
+ if isinstance(arg, ConstInt):
+ if self.metainterp_sd and int_could_be_an_address(arg.value):
+ addr = arg.getaddr()
+ name = self.metainterp_sd.get_name_from_address(addr)
+ if name:
+ return 'ConstClass(' + name + ')'
+ return str(arg.value)
+ elif self.ts is not None and isinstance(arg, self.ts.ConstRef):
+ if arg.value:
+ return 'ConstPtr(ptr' + str(mv) + ')'
+ return 'ConstPtr(null)'
+ if isinstance(arg, ConstFloat):
+ return str(arg.getfloat())
+ elif arg is None:
+ return 'None'
+ elif arg.is_vector():
+ return 'v' + str(mv)
+ elif arg.type == 'i':
+ return 'i' + str(mv)
+ elif arg.type == 'r':
+ return 'p' + str(mv)
+ elif arg.type == 'f':
+ return 'f' + str(mv)
+ else:
+ return '?'
+
+def int_could_be_an_address(x):
+ if we_are_translated():
+ x = rffi.cast(lltype.Signed, x) # force it
+ return not (-32768 <= x <= 32767)
+ else:
+ return isinstance(x, llmemory.AddressAsInt)
diff --git a/rpython/rlib/rjitlog/src/rjitlog.c
b/rpython/rlib/rjitlog/src/rjitlog.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rjitlog/src/rjitlog.c
@@ -0,0 +1,86 @@
+#define _GNU_SOURCE 1
+
+#ifdef RPYTHON_LL2CTYPES
+/* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
+#ifndef RPY_EXTERN
+#define RPY_EXTERN RPY_EXPORTED
+#endif
+#ifdef _WIN32
+#define RPY_EXPORTED __declspec(dllexport)
+#else
+#define RPY_EXPORTED extern __attribute__((visibility("default")))
+#endif
+#endif
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+
+static int jitlog_fd = -1;
+static int jitlog_ready = 0;
+
+RPY_EXTERN
+int jitlog_enabled()
+{
+ return jitlog_ready;
+}
+
+RPY_EXTERN
+void jitlog_try_init_using_env(void) {
+ if (jitlog_ready) { return; }
+
+ char *filename = getenv("JITLOG");
+
+ if (filename && filename[0]) {
+ // mode is 775
+ mode_t mode = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH;
+ jitlog_fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, mode);
+ if (jitlog_fd == -1) {
+ dprintf(2, "could not open '%s': ", filename);
+ perror(NULL);
+ exit(-1);
+ }
+ } else {
+ jitlog_ready = 0;
+ return;
+ }
+#ifndef _WIN32
+ unsetenv("JITLOG");
+#else
+ putenv("JITLOG=");
+#endif
+ jitlog_ready = 1;
+}
+
+RPY_EXTERN
+char *jitlog_init(int fd)
+{
+ jitlog_fd = fd;
+ jitlog_ready = 1;
+ return NULL;
+}
+
+RPY_EXTERN
+void jitlog_teardown()
+{
+ jitlog_ready = 0;
+ if (jitlog_fd == -1) {
+ return;
+ }
+ // close the jitlog file descriptor
+ close(jitlog_fd);
+ jitlog_fd = -1;
+}
+
+RPY_EXTERN
+void jitlog_write_marked(char * text, int length)
+{
+ if (!jitlog_ready) { return; }
+
+ write(jitlog_fd, text, length);
+}
diff --git a/rpython/rlib/rjitlog/src/rjitlog.h
b/rpython/rlib/rjitlog/src/rjitlog.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rjitlog/src/rjitlog.h
@@ -0,0 +1,18 @@
+#define _GNU_SOURCE 1
+
+#ifdef RPYTHON_LL2CTYPES
+ /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
+#ifndef RPY_EXTERN
+#define RPY_EXTERN RPY_EXPORTED
+#endif
+#ifdef _WIN32
+#define RPY_EXPORTED __declspec(dllexport)
+#else
+#define RPY_EXPORTED extern __attribute__((visibility("default")))
+#endif
+
+RPY_EXTERN char * jitlog_init(int);
+RPY_EXTERN void jitlog_try_init_using_env(void);
+RPY_EXTERN int jitlog_enabled();
+RPY_EXTERN void jitlog_write_marked(char*, int);
+RPY_EXTERN void jitlog_teardown();
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py
b/rpython/rlib/rjitlog/test/test_jitlog.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -0,0 +1,109 @@
+import py
+from rpython.jit.tool.oparser import pure_parse
+from rpython.jit.metainterp.optimizeopt.util import equaloplists
+from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.backend.model import AbstractCPU
+from rpython.jit.metainterp.history import ConstInt, ConstPtr
+from rpython.rlib.rjitlog import rjitlog as jl
+
+class FakeLog(object):
+ def __init__(self):
+ self.values = []
+
+ def _write_marked(self, id, text):
+ self.values.append(id + text)
+
+def _get_location(greenkey_list):
+ assert len(greenkey_list) == 0
+ return '/home/pypy/jit.py', 0, 'enclosed', 99, 'DEL'
+
+class TestLogger(object):
+
+ def make_metainterp_sd(self):
+ class FakeJitDriver(object):
+ class warmstate(object):
+ get_location_types = [jl.MP_FILENAME,jl.MP_INT,jl.MP_SCOPE,
jl.MP_INT, jl.MP_OPCODE]
+ @staticmethod
+ def get_location(greenkey_list):
+ return [jl.wrap(jl.MP_FILENAME[0],'s','/home/pypy/jit.py'),
+ jl.wrap(jl.MP_INT[0], 'i', 0),
+ jl.wrap(jl.MP_SCOPE[0], 's', 'enclosed'),
+ jl.wrap(jl.MP_INT[0], 'i', 99),
+ jl.wrap(jl.MP_OPCODE[0], 's', 'DEL')
+ ]
+
+
+ class FakeMetaInterpSd:
+ cpu = AbstractCPU()
+ cpu.ts = None
+ jitdrivers_sd = [FakeJitDriver()]
+ def get_name_from_address(self, addr):
+ return 'Name'
+ return FakeMetaInterpSd()
+
+ def test_debug_merge_point(self, tmpdir):
+ logger = jl.JitLogger()
+ file = tmpdir.join('binary_file')
+ file.ensure()
+ fd = file.open('wb')
+ jl.jitlog_init(fd.fileno())
+ logger.start_new_trace(self.make_metainterp_sd())
+ log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
+ op = ResOperation(rop.DEBUG_MERGE_POINT, [ConstInt(0), ConstInt(0),
ConstInt(0)])
+ log_trace.write([], [op])
+ #the next line will close 'fd'
+ fd.close()
+ logger.finish()
+ binary = file.read()
+ assert binary == (jl.MARK_START_TRACE) + jl.encode_le_addr(0) + \
+ jl.encode_str('loop') + jl.encode_le_addr(0) + \
+ (jl.MARK_TRACE) + jl.encode_le_addr(0) + \
+ (jl.MARK_INPUT_ARGS) + jl.encode_str('') + \
+ (jl.MARK_INIT_MERGE_POINT) +
b'\x05\x00\x01s\x00i\x08s\x00i\x10s' + \
+ (jl.MARK_MERGE_POINT) + \
+ b'\xff' + jl.encode_str('/home/pypy/jit.py') + \
+ b'\x00' + jl.encode_le_64bit(0) + \
+ b'\xff' + jl.encode_str('enclosed') + \
+ b'\x00' + jl.encode_le_64bit(99) + \
+ b'\xff' + jl.encode_str('DEL')
+
+ def test_common_prefix(self):
+ fakelog = FakeLog()
+ compressor = jl.PrefixCompressor(1)
+ # nothing to compress yet!
+ result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','hello')])
+ assert result == b"\xff\x05\x00\x00\x00hello"
+ assert fakelog.values == []
+ #
+ result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','hello')])
+ assert result == b"\xef"
+ assert fakelog.values == [(jl.MARK_COMMON_PREFIX) +
"\x00\x05\x00\x00\x00hello"]
+ #
+ fakelog.values = []
+ result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','heiter')])
+ assert result == b"\x00\x04\x00\x00\x00iter"
+ assert fakelog.values == [(jl.MARK_COMMON_PREFIX) +
"\x00\x02\x00\x00\x00he"]
+ #
+ fakelog.values = []
+ result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','heute')])
+ assert result == b"\x00\x03\x00\x00\x00ute"
+ assert fakelog.values == []
+ #
+ fakelog.values = []
+ result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','welt')])
+ assert result == b"\xff\x04\x00\x00\x00welt"
+ assert fakelog.values == []
+ #
+ fakelog.values = []
+ result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','welle')])
+ assert result == b"\x00\x02\x00\x00\x00le"
+ assert fakelog.values == [(jl.MARK_COMMON_PREFIX) +
"\x00\x03\x00\x00\x00wel"]
+
+ def test_common_prefix_func(self):
+ assert jl.commonprefix("","") == ""
+ assert jl.commonprefix("/hello/world","/path/to") == "/"
+ assert jl.commonprefix("pyramid","python") == "py"
+ assert jl.commonprefix("0"*100,"0"*100) == "0"*100
+ with py.test.raises(AssertionError):
+ jl.commonprefix(None,None)
+
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -35,11 +35,5 @@
def enable(fileno, interval):
_get_vmprof().enable(fileno, interval)
-def enable_jitlog(fileno):
- _get_vmprof().enable_jitlog(fileno)
-
def disable():
_get_vmprof().disable()
-
-def disable_jitlog():
- _get_vmprof().disable_jitlog()
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -56,21 +56,6 @@
compilation_info=eci,
_nowrapper=True)
- # jit log functions
- jitlog_init = rffi.llexternal("jitlog_init", [rffi.INT],
- rffi.CCHARP, compilation_info=eci)
- jitlog_try_init_using_env = rffi.llexternal("jitlog_try_init_using_env",
- [], lltype.Void, compilation_info=eci)
- jitlog_write_marked = rffi.llexternal("jitlog_write_marked",
- [rffi.CCHARP, rffi.INT],
- lltype.Void, compilation_info=eci,
- releasegil=False)
- jitlog_enabled = rffi.llexternal("jitlog_enabled", [], rffi.INT,
- compilation_info=eci,
- releasegil=False)
- jitlog_teardown = rffi.llexternal("jitlog_teardown", [], lltype.Void,
- compilation_info=eci)
-
return CInterface(locals())
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -122,21 +122,6 @@
raise VMProfError(os.strerror(rposix.get_saved_errno()))
self.is_enabled = True
- @jit.dont_look_inside
- def enable_jitlog(self, fileno):
- # initialize the jit log
- from rpython.rlib import jitlog as jl
- p_error = self.cintf.jitlog_init(fileno)
- if p_error:
- raise VMProfError(rffi.charp2str(p_error))
- blob = jl.assemble_header()
- self.cintf.jitlog_write_marked(jl.MARK_JITLOG_HEADER + blob, len(blob)
+ 1)
-
- def disable_jitlog(self):
- from rpython.rlib.jitlog import stats_flush_trace_counts
- stats_flush_trace_counts(None)
- self.cintf.jitlog_teardown()
-
def disable(self):
"""Disable vmprof.
Raises VMProfError if something goes wrong.
diff --git a/rpython/rlib/rvmprof/src/jitlog_main.h
b/rpython/rlib/rvmprof/src/jitlog_main.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/jitlog_main.h
+++ /dev/null
@@ -1,69 +0,0 @@
-#include <string.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-static int jitlog_fd = -1;
-static int jitlog_ready = 0;
-
-RPY_EXTERN
-int jitlog_enabled()
-{
- return jitlog_ready;
-}
-
-RPY_EXTERN
-void jitlog_try_init_using_env(void) {
- if (jitlog_ready) { return; }
-
- char *filename = getenv("JITLOG");
-
- if (filename && filename[0]) {
- // mode is 775
- mode_t mode = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH;
- jitlog_fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, mode);
- if (jitlog_fd == -1) {
- dprintf(2, "could not open '%s': ", filename);
- perror(NULL);
- exit(-1);
- }
- } else {
- jitlog_ready = 0;
- return;
- }
-#ifndef _WIN32
- unsetenv("JITLOG");
-#else
- putenv("JITLOG=");
-#endif
- jitlog_ready = 1;
-}
-
-RPY_EXTERN
-char *jitlog_init(int fd)
-{
- jitlog_fd = fd;
- jitlog_ready = 1;
- return NULL;
-}
-
-RPY_EXTERN
-void jitlog_teardown()
-{
- jitlog_ready = 0;
- if (jitlog_fd == -1) {
- return;
- }
- // close the jitlog file descriptor
- close(jitlog_fd);
- jitlog_fd = -1;
-}
-
-RPY_EXTERN
-void jitlog_write_marked(char * text, int length)
-{
- if (!jitlog_ready) { return; }
-
- write(jitlog_fd, text, length);
-}
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h
b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -8,9 +8,3 @@
RPY_EXTERN int vmprof_stack_append(void*, long);
RPY_EXTERN long vmprof_stack_pop(void*);
RPY_EXTERN void vmprof_stack_free(void*);
-
-RPY_EXTERN char * jitlog_init(int);
-RPY_EXTERN void jitlog_try_init_using_env(void);
-RPY_EXTERN int jitlog_enabled();
-RPY_EXTERN void jitlog_write_marked(char*, int);
-RPY_EXTERN void jitlog_teardown();
diff --git a/rpython/rlib/test/test_jitlog.py b/rpython/rlib/test/test_jitlog.py
deleted file mode 100644
--- a/rpython/rlib/test/test_jitlog.py
+++ /dev/null
@@ -1,110 +0,0 @@
-import py
-from rpython.jit.tool.oparser import pure_parse
-from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.metainterp.resoperation import ResOperation, rop
-from rpython.jit.backend.model import AbstractCPU
-from rpython.jit.metainterp.history import ConstInt, ConstPtr
-from rpython.rlib.jitlog import (encode_str, encode_le_16bit, encode_le_64bit)
-from rpython.rlib import jitlog as jl
-
-class FakeLog(object):
- def __init__(self):
- self.values = []
-
- def _write_marked(self, id, text):
- self.values.append(id + text)
-
-def _get_location(greenkey_list):
- assert len(greenkey_list) == 0
- return '/home/pypy/jit.py', 0, 'enclosed', 99, 'DEL'
-
-class TestLogger(object):
-
- def make_metainterp_sd(self):
- class FakeJitDriver(object):
- class warmstate(object):
- get_location_types = [jl.MP_FILENAME,jl.MP_INT,jl.MP_SCOPE,
jl.MP_INT, jl.MP_OPCODE]
- @staticmethod
- def get_location(greenkey_list):
- return [jl.wrap(jl.MP_FILENAME[0],'s','/home/pypy/jit.py'),
- jl.wrap(jl.MP_INT[0], 'i', 0),
- jl.wrap(jl.MP_SCOPE[0], 's', 'enclosed'),
- jl.wrap(jl.MP_INT[0], 'i', 99),
- jl.wrap(jl.MP_OPCODE[0], 's', 'DEL')
- ]
-
-
- class FakeMetaInterpSd:
- cpu = AbstractCPU()
- cpu.ts = None
- jitdrivers_sd = [FakeJitDriver()]
- def get_name_from_address(self, addr):
- return 'Name'
- return FakeMetaInterpSd()
-
- def test_debug_merge_point(self, tmpdir):
- logger = jl.VMProfJitLogger()
- file = tmpdir.join('binary_file')
- file.ensure()
- fd = file.open('wb')
- logger.cintf.jitlog_init(fd.fileno())
- logger.start_new_trace(self.make_metainterp_sd())
- log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
- op = ResOperation(rop.DEBUG_MERGE_POINT, [ConstInt(0), ConstInt(0),
ConstInt(0)])
- log_trace.write([], [op])
- #the next line will close 'fd'
- fd.close()
- logger.finish()
- binary = file.read()
- assert binary == (jl.MARK_START_TRACE) + jl.encode_le_addr(0) + \
- jl.encode_str('loop') + jl.encode_le_addr(0) + \
- (jl.MARK_TRACE) + jl.encode_le_addr(0) + \
- (jl.MARK_INPUT_ARGS) + jl.encode_str('') + \
- (jl.MARK_INIT_MERGE_POINT) +
b'\x05\x00\x01s\x00i\x08s\x00i\x10s' + \
- (jl.MARK_MERGE_POINT) + \
- b'\xff' + encode_str('/home/pypy/jit.py') + \
- b'\x00' + encode_le_64bit(0) + \
- b'\xff' + encode_str('enclosed') + \
- b'\x00' + encode_le_64bit(99) + \
- b'\xff' + encode_str('DEL')
-
- def test_common_prefix(self):
- fakelog = FakeLog()
- compressor = jl.PrefixCompressor(1)
- # nothing to compress yet!
- result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','hello')])
- assert result == b"\xff\x05\x00\x00\x00hello"
- assert fakelog.values == []
- #
- result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','hello')])
- assert result == b"\xef"
- assert fakelog.values == [(jl.MARK_COMMON_PREFIX) +
"\x00\x05\x00\x00\x00hello"]
- #
- fakelog.values = []
- result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','heiter')])
- assert result == b"\x00\x04\x00\x00\x00iter"
- assert fakelog.values == [(jl.MARK_COMMON_PREFIX) +
"\x00\x02\x00\x00\x00he"]
- #
- fakelog.values = []
- result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','heute')])
- assert result == b"\x00\x03\x00\x00\x00ute"
- assert fakelog.values == []
- #
- fakelog.values = []
- result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','welt')])
- assert result == b"\xff\x04\x00\x00\x00welt"
- assert fakelog.values == []
- #
- fakelog.values = []
- result = jl.encode_merge_point(fakelog, compressor,
[jl.StringValue(0x0,'s','welle')])
- assert result == b"\x00\x02\x00\x00\x00le"
- assert fakelog.values == [(jl.MARK_COMMON_PREFIX) +
"\x00\x03\x00\x00\x00wel"]
-
- def test_common_prefix_func(self):
- assert jl.commonprefix("","") == ""
- assert jl.commonprefix("/hello/world","/path/to") == "/"
- assert jl.commonprefix("pyramid","python") == "py"
- assert jl.commonprefix("0"*100,"0"*100) == "0"*100
- with py.test.raises(AssertionError):
- jl.commonprefix(None,None)
-
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
