Author: Maciej Fijalkowski <[email protected]>
Branch: vmprof
Changeset: r75508:a9901f5effb2
Date: 2015-01-24 14:00 +0200
http://bitbucket.org/pypy/pypy/changeset/a9901f5effb2/
Log: Getting somewhere - a more sane way to record all the code objs and
store them in the same file
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,6 +9,7 @@
from rpython.rlib.signature import signature
from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.rweaklist import RWeakListMixin
from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
UserDelAction)
@@ -366,6 +367,10 @@
# ____________________________________________________________
+class CodeObjWeakList(RWeakListMixin):
+ def __init__(self):
+ self.initialize()
+
class ObjSpace(object):
"""Base class for the interpreter-level implementations of object spaces.
http://pypy.readthedocs.org/en/latest/objspace.html""";
@@ -389,6 +394,7 @@
self.check_signal_action = None # changed by the signal module
self.user_del_action = UserDelAction(self)
self._code_of_sys_exc_info = None
+ self.all_code_objs = CodeObjWeakList()
# can be overridden to a subclass
self.initialize()
@@ -666,6 +672,16 @@
assert ec is not None
return ec
+ def register_code_object(self, pycode):
+ callback = self.getexecutioncontext().register_code_callback
+ if callback is not None:
+ callback(self, pycode)
+ self.all_code_objs.add_handle(pycode)
+
+ def set_code_callback(self, callback):
+ ec = self.getexecutioncontext()
+ ec.register_code_callback = callback
+
def _freeze_(self):
return True
diff --git a/pypy/interpreter/executioncontext.py
b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -33,6 +33,7 @@
self.profilefunc = None
self.w_profilefuncarg = None
self.thread_disappeared = False # might be set to True after
os.fork()
+ self.register_code_callback = None
@staticmethod
def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -14,9 +14,10 @@
CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY)
from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, r_longlong
from rpython.rlib.objectmodel import compute_hash
from rpython.rlib import jit
+from rpython.rlib.debug import debug_start, debug_stop, debug_print
class BytecodeCorruption(Exception):
@@ -56,6 +57,11 @@
_immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]",
"co_freevars[*]", "co_cellvars[*]",
"_args_as_cellvars[*]"]
+ if sys.maxint == 2147483647:
+ _unique_id = 0 # XXX this is wrong, it won't work on 32bit
+ else:
+ _unique_id = 0x7000000000000000
+
def __init__(self, space, argcount, nlocals, stacksize, flags,
code, consts, names, varnames, filename,
name, firstlineno, lnotab, freevars, cellvars,
@@ -83,7 +89,7 @@
self.magic = magic
self._signature = cpython_code_signature(self)
self._initialize()
- self._vmprof_setup_maybe()
+ space.register_code_object(self)
def _initialize(self):
if self.co_cellvars:
@@ -125,9 +131,12 @@
from pypy.objspace.std.mapdict import init_mapdict_cache
init_mapdict_cache(self)
- def _vmprof_setup_maybe(self):
- # this is overridden only if _vmprof is enabled
- pass
+ self._unique_id = PyCode._unique_id
+ PyCode._unique_id += 1
+
+ def _get_full_name(self):
+ return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno,
+ self.co_filename)
def _cleanup_(self):
if (self.magic == cpython_magic and
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -14,4 +14,5 @@
def setup_after_space_initialization(self):
# force the __extend__ hacks to occur early
- import pypy.module._vmprof.interp_vmprof
+ from pypy.module._vmprof.interp_vmprof import VMProf
+ self.vmprof = VMProf()
diff --git a/pypy/module/_vmprof/interp_vmprof.py
b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -1,17 +1,13 @@
-import py
+import py, os, struct
from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
from rpython.translator.tool.cbuild import ExternalCompilationInfo
from rpython.rtyper.annlowlevel import cast_instance_to_gcref,
cast_base_ptr_to_instance
-from rpython.rlib.objectmodel import we_are_translated, CDefinedIntSymbolic
-from rpython.rlib import jit, rposix
-from rpython.tool.pairtype import extendabletype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import jit, rposix, entrypoint
from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt, wrap_oserror
+from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
from pypy.interpreter.gateway import unwrap_spec
from pypy.interpreter.pyframe import PyFrame
-from pypy.interpreter.pycode import PyCode
-
-FALSE_BUT_NON_CONSTANT = CDefinedIntSymbolic('0', default=0)
ROOT = py.path.local(__file__).join('..')
SRC = ROOT.join('src')
@@ -63,7 +59,7 @@
pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], lltype.Void,
compilation_info=eci)
vmprof_enable = rffi.llexternal("vmprof_enable",
- [rffi.INT, rffi.INT, rffi.LONG],
+ [rffi.INT, rffi.INT, rffi.LONG, rffi.INT],
rffi.INT, compilation_info=eci)
vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
compilation_info=eci)
@@ -97,130 +93,85 @@
return original_execute_frame(frame, w_inputvalue, operr)
-
-class __extend__(PyCode):
- __metaclass__ = extendabletype
-
- def _vmprof_setup_maybe(self):
- self._vmprof_virtual_ip = _vmprof.get_next_virtual_IP()
- self._vmprof_registered = 0
-
-# avoid rtyper warnings
-PyCode._vmprof_virtual_ip = 0
-PyCode._vmprof_registered = 0
-
-
-
[email protected]_lowlevel('main', [llmemory.GCREF],
+ 'pypy_vmprof_get_virtual_ip', True)
def get_virtual_ip(gc_frame):
frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
if jit._get_virtualizable_token(frame):
return rffi.cast(rffi.VOIDP, 0)
virtual_ip = do_get_virtual_ip(frame)
return rffi.cast(rffi.VOIDP, virtual_ip)
-get_virtual_ip.c_name = 'pypy_vmprof_get_virtual_ip'
-get_virtual_ip._dont_inline_ = True
-
-def strncpy(src, tgt, tgt_ofs, count):
- if len(src) < count:
- count = len(src)
- i = 0
- while i < count:
- tgt[i + tgt_ofs] = src[i]
- i += 1
- return i
-
-def int2str(num, s, ofs):
- if num == 0:
- s[ofs] = '0'
- return 1
- count = 0
- c = num
- while c != 0:
- count += 1
- c /= 10
- pos = ofs + count - 1
- c = num
- while c != 0:
- s[pos] = chr(ord('0') + c % 10)
- c /= 10
- pos -= 1
- return count
def do_get_virtual_ip(frame):
- virtual_ip = frame.pycode._vmprof_virtual_ip
- if not frame.pycode._vmprof_registered:
- # we need to register this code object
- name = frame.pycode.co_name
- filename = frame.pycode.co_filename
- firstlineno = frame.pycode.co_firstlineno
- start = rffi.cast(rffi.VOIDP, virtual_ip)
- end = start # ignored for now
- #
- # manually fill the C buffer; we cannot use str2charp because we
- # cannot call malloc from a signal handler
- strbuf = _vmprof.strbuf
- ofs = strncpy("py:", _vmprof.strbuf, 0, len("py:"))
- ofs += strncpy(filename, _vmprof.strbuf, ofs, 128)
- _vmprof.strbuf[ofs] = ':'
- ofs += 1
- ofs += int2str(firstlineno, _vmprof.strbuf, ofs)
- _vmprof.strbuf[ofs] = ':'
- ofs += 1
- ofs += strncpy(name, _vmprof.strbuf, ofs, 1024 - 1 - ofs)
- _vmprof.strbuf[ofs] = '\x00'
- vmprof_register_virtual_function(strbuf, start, end)
- frame.pycode._vmprof_registered = 1
- #
- return virtual_ip
+ return frame.pycode._unique_id
class VMProf(object):
def __init__(self):
- self.virtual_ip = 0x7000000000000000
self.is_enabled = False
self.ever_enabled = False
- self.strbuf = lltype.malloc(rffi.CCHARP.TO, 1024, flavor='raw',
immortal=True, zero=True)
+ self.mapping_so_far = [] # stored mapping in between runs
+ self.fileno = -1
- def get_next_virtual_IP(self):
- self.virtual_ip += 1
- return self.virtual_ip
-
- @jit.dont_look_inside
- def _annotate_get_virtual_ip(self):
- if FALSE_BUT_NON_CONSTANT:
- # make sure it's annotated
- gcref = rffi.cast(llmemory.GCREF, self.virtual_ip) # just a random
non-constant value
- get_virtual_ip(gcref)
-
- def enable(self, space, fileno, symno, period):
- self._annotate_get_virtual_ip()
+ def enable(self, space, fileno, period):
if self.is_enabled:
raise oefmt(space.w_ValueError, "_vmprof already enabled")
+ self.fileno = fileno
self.is_enabled = True
+ self.write_header(fileno, period)
if not self.ever_enabled:
- pypy_vmprof_init()
+ if we_are_translated():
+ pypy_vmprof_init()
self.ever_enabled = True
- res = vmprof_enable(fileno, symno, period)
+ for weakcode in space.all_code_objs.get_all_handles():
+ code = weakcode()
+ if code:
+ self.register_code(space, code)
+ space.set_code_callback(self.register_code)
+ if we_are_translated():
+ # does not work untranslated
+ res = vmprof_enable(fileno, -1, period, 0)
+ else:
+ res = 0
if res == -1:
raise wrap_oserror(space, OSError(rposix.get_errno(),
"_vmprof.enable"))
+ def write_header(self, fileno, period):
+ if period == -1:
+ period_usec = 1000000 / 100 # 100hz
+ else:
+ period_usec = period
+ os.write(fileno, struct.pack("lllll", 0, 3, 0, period_usec, 0))
+
+ def register_code(self, space, code):
+ if self.fileno == -1:
+ raise OperationError(space.w_RuntimeError,
+ space.wrap("vmprof not running"))
+ name = code._get_full_name()
+ s = '\x02' + struct.pack("ll", code._unique_id, len(name)) + name
+ os.write(self.fileno, s)
+
def disable(self, space):
if not self.is_enabled:
raise oefmt(space.w_ValueError, "_vmprof not enabled")
self.is_enabled = False
- res = vmprof_disable()
+ self.fileno = -1
+ if we_are_translated():
+ # does not work untranslated
+ res = vmprof_disable()
+ else:
+ res = 0
+ space.set_code_callback(None)
if res == -1:
raise wrap_oserror(space, OSError(rposix.get_errno(),
"_vmprof.disable"))
-_vmprof = VMProf()
-
-@unwrap_spec(fileno=int, symno=int, period=int)
-def enable(space, fileno, symno, period=-1):
- _vmprof.enable(space, fileno, symno, period)
+@unwrap_spec(fileno=int, period=int)
+def enable(space, fileno, period=-1):
+ space.getbuiltinmodule('_vmprof').vmprof.enable(space, fileno, period)
def disable(space):
- _vmprof.disable(space)
+ space.getbuiltinmodule('_vmprof').vmprof.disable(space)
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -35,7 +35,7 @@
#define MAX_STACK_DEPTH 64
static FILE* profile_file;
-static FILE* symbol_file;
+static FILE* symbol_file = NULL;
void* vmprof_mainloop_func;
static ptrdiff_t mainloop_sp_offset;
static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
@@ -45,10 +45,19 @@
* functions to write a profile file compatible with gperftools
* *************************************************************
*/
+
+#define MARKER_STACKTRACE '\x01'
+#define MARKER_VIRTUAL_IP '\x02'
+#define MARKER_TRAILER '\x03'
+
static void prof_word(FILE* f, long x) {
fwrite(&x, sizeof(x), 1, f);
}
+static void prof_char(FILE *f, char x) {
+ fwrite(&x, sizeof(x), 1, f);
+}
+
static void prof_header(FILE* f, long period_usec) {
prof_word(f, 0);
prof_word(f, 3);
@@ -59,18 +68,13 @@
static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count)
{
int i;
+ prof_char(f, MARKER_STACKTRACE);
prof_word(f, count);
prof_word(f, depth);
for(i=0; i<depth; i++)
prof_word(f, (long)stack[i]);
}
-static void prof_binary_trailer(FILE* f) {
- prof_word(f, 0);
- prof_word(f, 1);
- prof_word(f, 0);
-}
-
/* ******************************************************
* libunwind workaround for process JIT frames correctly
@@ -198,21 +202,26 @@
* *************************************************************
*/
-static int open_profile(int fd, int sym_fd, long period_usec) {
+static int open_profile(int fd, int sym_fd, long period_usec, int
write_header) {
if ((fd = dup(fd)) == -1) {
return -1;
}
- if ((sym_fd = dup(sym_fd)) == -1) {
- return -1;
- }
+ if (sym_fd != -1) {
+ if ((sym_fd = dup(sym_fd)) == -1) {
+ return -1;
+ }
+ }
profile_file = fdopen(fd, "wb");
if (!profile_file) {
return -1;
}
- prof_header(profile_file, period_usec);
- symbol_file = fdopen(sym_fd, "w");
- if (!symbol_file) {
- return -1;
+ if (write_header)
+ prof_header(profile_file, period_usec);
+ if (sym_fd != -1) {
+ symbol_file = fdopen(sym_fd, "w");
+ if (!symbol_file) {
+ return -1;
+ }
}
return 0;
}
@@ -222,7 +231,7 @@
FILE* src;
char buf[BUFSIZ];
size_t size;
- prof_binary_trailer(profile_file);
+ prof_char(profile_file, MARKER_TRAILER);
// copy /proc/PID/maps to the end of the profile file
sprintf(buf, "/proc/%d/maps", getpid());
@@ -232,7 +241,10 @@
}
fclose(src);
fclose(profile_file);
- fclose(symbol_file);
+ if (symbol_file) {
+ fclose(symbol_file);
+ symbol_file = NULL;
+ }
return 0;
}
@@ -291,10 +303,10 @@
mainloop_get_virtual_ip = get_virtual_ip;
}
-int vmprof_enable(int fd, int sym_fd, long period_usec) {
+int vmprof_enable(int fd, int sym_fd, long period_usec, int write_header) {
if (period_usec == -1)
period_usec = 1000000 / 100; /* 100hz */
- if (open_profile(fd, sym_fd, period_usec) == -1) {
+ if (open_profile(fd, sym_fd, period_usec, write_header) == -1) {
return -1;
}
if (install_sigprof_handler() == -1) {
diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h
--- a/pypy/module/_vmprof/src/vmprof.h
+++ b/pypy/module/_vmprof/src/vmprof.h
@@ -12,7 +12,7 @@
void vmprof_register_virtual_function(const char* name, void* start, void*
end);
-int vmprof_enable(int fd, int sym_fd, long period_usec);
+int vmprof_enable(int fd, int sym_fd, long period_usec, int write_header);
int vmprof_disable(void);
#endif
diff --git a/pypy/module/_vmprof/test/test__vmprof.py
b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -1,55 +1,55 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module._vmprof import interp_vmprof
-from pypy.module._vmprof.interp_vmprof import do_get_virtual_ip, _vmprof
-class FakePyFrame(object):
+import tempfile
+from pypy.tool.pytest.objspace import gettestobjspace
- def __init__(self, pycode):
- self.pycode = pycode
+class AppTestVMProf(object):
+ def setup_class(cls):
+ cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
+ cls.tmpfile = tempfile.NamedTemporaryFile()
+ cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno())
+ cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name)
+ cls.tmpfile2 = tempfile.NamedTemporaryFile()
+ cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno())
+ cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name)
-class FakePyCode(object):
+ def test_import_vmprof(self):
+ import struct, sys
- _vmprof_setup_maybe = interp_vmprof.PyCode._vmprof_setup_maybe.im_func
+ WORD = struct.calcsize('l')
+
+ def count(s):
+ i = 0
+ count = 0
+ i += 5 * WORD # header
+ while i < len(s):
+ assert s[i] == '\x02'
+ i += 1
+ _, size = struct.unpack("ll", s[i:i + 2 * WORD])
+ count += 1
+ i += 2 * WORD + size
+ return count
+
+ import _vmprof
+ _vmprof.enable(self.tmpfileno)
+ _vmprof.disable()
+ s = open(self.tmpfilename).read()
+ no_of_codes = count(s)
+ assert no_of_codes > 10
+ d = {}
- def __init__(self, co_name):
- self.co_name = co_name
- self.co_filename = 'filename'
- self.co_firstlineno = 13
- self._vmprof_setup_maybe()
+ exec """def foo():
+ pass
+ """ in d
-def test_get_virtual_ip(monkeypatch):
- functions = []
- def register_virtual_function(name, start, end):
- name = rffi.charp2str(name)
- start = rffi.cast(lltype.Signed, start)
- end = rffi.cast(lltype.Signed, end)
- functions.append((name, start, end))
- monkeypatch.setattr(interp_vmprof, 'vmprof_register_virtual_function',
register_virtual_function)
- #
- mycode = FakePyCode('foo')
- assert mycode._vmprof_virtual_ip < 0
- myframe = FakePyFrame(mycode)
+ _vmprof.enable(self.tmpfileno2)
- _vmprof.counter = 42
- ip = do_get_virtual_ip(myframe)
- assert ip == mycode._vmprof_virtual_ip
- assert functions == [('py:filename:13:foo', ip, ip)]
+ exec """def foo2():
+ pass
+ """ in d
- # the second time, we don't register it again
- functions = []
- ip = do_get_virtual_ip(myframe)
- assert ip == mycode._vmprof_virtual_ip
- assert functions == []
-
- # now, let's try with a long name
- mycode = FakePyCode('abcde' + 'f' * 20000)
- myframe = FakePyFrame(mycode)
- functions = []
- ip2 = do_get_virtual_ip(myframe)
- assert ip2 == mycode._vmprof_virtual_ip
- assert ip2 < ip # because it was generated later
- assert len(functions) == 1
- name, start, end = functions[0]
- assert len(name) < 1025
- assert name == 'py:filename:13:abcde' + 'f' * (1024 - 20 - 1)
-
+ _vmprof.disable()
+ s = open(self.tmpfilename2).read()
+ no_of_codes2 = count(s)
+ assert "py:foo:" in s
+ assert "py:foo2:" in s
+ assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests
diff --git a/rpython/jit/backend/llsupport/assembler.py
b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -278,6 +278,9 @@
# YYY very minor leak -- we need the counters to stay alive
# forever, just because we want to report them at the end
# of the process
+
+ # XXX the numbers here are ALMOST unique, but not quite, use a counter
+ # or something
struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
track_allocation=False)
struct.i = 0
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
