Author: Anton Gulenko <anton.gule...@googlemail.com>
Branch: storage
Changeset: r715:0c8b9379f9d5
Date: 2014-03-28 14:01 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/0c8b9379f9d5/
Log: Moved storage-statistics to own module, integrated with storage code
and main target. Added test. Added safety-assert to prevent storage-
access during bootstrapping phase (specialized storage like
ClassShadows cannot be used before the objects have been filled in).
Made store_w_superclass and store_w_methoddict of ClassShadow
slightly more consistent.
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -15,7 +15,7 @@
that create W_PointersObjects of correct size with attached shadows.
"""
import sys, weakref
-from spyvm import constants, error, version
+from spyvm import constants, error, version, storage_statistics
from spyvm.version import elidable_for_version
from rpython.rlib import rrandom, objectmodel, jit, signature
@@ -24,7 +24,6 @@
from rpython.tool.pairtype import extendabletype
from rpython.rlib.objectmodel import instantiate, compute_hash,
import_from_mixin, we_are_translated
from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.listsort import TimSort
from rsdl import RSDL, RSDL_helper
class W_Object(object):
@@ -468,6 +467,7 @@
Float)."""
_attrs_ = ['w_class']
repr_classname = "W_AbstractObjectWithClassReference"
+ w_class = None
def __init__(self, space, w_class):
if w_class is not None: # it's None only for testing and space
generation
@@ -490,10 +490,12 @@
def guess_classname(self):
if self.has_class():
- class_shadow = self.class_shadow(self.w_class.space())
- # Three question marks, because it would be highly irregular to
have
- # an initialized ClassShadow without an initialized name field.
- return class_shadow.name or "???"
+ if self.w_class.has_shadow():
+ class_shadow = self.class_shadow(self.w_class.space())
+ return class_shadow.name
+ else:
+ # We cannot access the class during the initialization
sequence.
+ return "?? (class not initialized)"
else:
return "? (no class)"
@@ -517,50 +519,12 @@
assert w_class is not None
return w_class.as_class_get_shadow(space)
-class StatsSorter(TimSort):
- def lt(self, a, b):
- if a[0] == b[0]:
- if a[1] == b[1]:
- return a[2] < b[2]
- else:
- return a[1] < b[1]
- else:
- return a[0] < b[0]
-class StrategyStatistics(object):
- # Key: (operation_name, old_strategy, new_strategy)
- # Value: [sizes]
- stats = {}
- do_log = False
- do_stats = False
- do_stats_sizes = False
-
- def stat_operation(self, operation_name, old_strategy, new_strategy, size):
- key = (operation_name, old_strategy, new_strategy)
- if not key in self.stats:
- self.stats[key] = []
- self.stats[key].append(size)
- def log_operation(self, op, new_strategy_tag, old_strategy_tag, classname,
size):
- print "%s (%s, was %s) of %s size %d" % (op, new_strategy_tag,
old_strategy_tag, classname, size)
- def sorted_keys(self):
- keys = [ x for x in self.stats ]
- StatsSorter(keys).sort()
- return keys
- def print_stats(self):
- for key in self.sorted_keys():
- sizes = self.stats[key]
- sum = 0
- for s in sizes:
- sum += s
- print "%s: %d times, avg size: %d" % (key, len(sizes),
sum/len(sizes))
- if self.do_stats_sizes:
- print " All sizes: %s" % sizes
-strategy_stats = StrategyStatistics()
-
class W_AbstractPointersObject(W_AbstractObjectWithClassReference):
"""Common object."""
_attrs_ = ['shadow']
shadow = None
repr_classname = "W_AbstractPointersObject"
+ log_storage = storage_statistics.log
@jit.unroll_safe
def __init__(self, space, w_class, size):
@@ -570,6 +534,7 @@
def initialize_storage(self, space, size):
self.store_shadow(self.empty_storage(space, size))
+ self.log_storage("Initialized")
def fillin(self, space, g_self):
W_AbstractObjectWithClassReference.fillin(self, space, g_self)
@@ -579,17 +544,28 @@
pointers = g_self.get_pointers()
self.store_shadow(self.storage_for_list(space, pointers))
self.store_all(space, pointers)
+ self.log_storage("Filledin", log_classname=False)
def empty_storage(self, space, size):
raise NotImplementedError()
def storage_for_list(self, space, vars):
raise NotImplementedError()
+ def assert_shadow(self):
+ # Failing the following assert most likely indicates a bug. The shadow
can only be absent during
+ # the bootstrapping sequence. It will be initialized in the fillin()
method. Before that, it should
+ # not be switched to a specialized shadow, and the space is also not
yet available here! Otherwise,
+ # the specialized shadow will attempt to read information from an
uninitialized object.
+ shadow = self.shadow
+ assert shadow, "The shadow has not been initialized yet!"
+ return shadow
+
def switch_shadow(self, new_shadow):
- if self.shadow is not None:
- new_shadow.copy_from(self.shadow)
+ old_shadow = self.assert_shadow()
+ new_shadow.copy_from(old_shadow)
self.store_shadow(new_shadow)
new_shadow.attach_shadow()
+ self.log_storage("Switched", old_shadow)
def store_with_new_storage(self, new_storage, n0, w_val):
space = self.space()
@@ -597,8 +573,7 @@
self.store(space, n0, w_val)
def space(self):
- assert self.shadow, "Cannot access space without a shadow!"
- return self.shadow.space
+ return self.assert_shadow().space
def __str__(self):
if self.has_shadow() and self.shadow.provides_getname:
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -263,8 +263,8 @@
_attrs_ = ["name", "_instance_size", "instance_varsized", "instance_kind",
"_s_methoddict", "_s_superclass", "subclass_s"]
- name = '??'
- _s_superclass = None
+ name = '??? (incomplete class info)'
+ _s_superclass = _s_methoddict = None
provides_getname = True
repr_classname = "ClassShadow"
@@ -277,10 +277,7 @@
if n0 == constants.CLASS_SUPERCLASS_INDEX:
self.store_w_superclass(w_val)
elif n0 == constants.CLASS_METHODDICT_INDEX:
- assert isinstance(w_val, model.W_PointersObject)
- if not w_val.is_same_object(self.space.w_nil):
- self._s_methoddict = w_val.as_methoddict_get_shadow(self.space)
- self._s_methoddict.s_class = self
+ self.store_w_methoddict(w_val)
elif n0 == constants.CLASS_FORMAT_INDEX:
# read and painfully decode the format
assert isinstance(w_val, model.W_SmallInteger)
@@ -340,18 +337,33 @@
self.changed()
def store_w_superclass(self, w_class):
+ superclass = self._s_superclass
if w_class is None or w_class.is_same_object(self.space.w_nil):
+ if superclass: superclass.detach_s_class(self)
self._s_superclass = None
else:
assert isinstance(w_class, model.W_PointersObject)
- s_scls = w_class.as_class_get_shadow(self.space)
- if self._s_superclass is s_scls:
+ s_new_superclass = w_class.as_class_get_shadow(self.space)
+ if superclass is s_new_superclass:
return
- if self._s_superclass is not None:
- self._s_superclass.detach_s_class(self)
- self._s_superclass = s_scls
- self._s_superclass.attach_s_class(self)
+ if superclass: superclass.detach_s_class(self)
+ self._s_superclass = s_new_superclass
+ s_new_superclass.attach_s_class(self)
+ def store_w_methoddict(self, w_methoddict):
+ methoddict = self._s_methoddict
+ if w_methoddict is None or
w_methoddict.is_same_object(self.space.w_nil):
+ if methoddict: methoddict.s_class = None
+ self._s_methoddict = None
+ else:
+ assert isinstance(w_methoddict, model.W_PointersObject)
+ s_new_methoddict =
w_methoddict.as_methoddict_get_shadow(self.space)
+ if methoddict is s_new_methoddict:
+ return
+ if methoddict: methoddict.s_class = None
+ self._s_methoddict = s_new_methoddict
+ self._s_methoddict.s_class = self
+
def attach_s_class(self, s_other):
self.subclass_s[s_other] = None
@@ -406,7 +418,7 @@
return self._s_superclass
def getname(self):
- return self.name or '?'
+ return self.name
# _______________________________________________________________
# Methods for querying the format word, taken from the blue book:
diff --git a/spyvm/storage_statistics.py b/spyvm/storage_statistics.py
new file mode 100644
--- /dev/null
+++ b/spyvm/storage_statistics.py
@@ -0,0 +1,92 @@
+
+from rpython.rlib.listsort import TimSort
+
+class StatsSorter(TimSort):
+ """Sort a tuple of 3 strings"""
+ def lt(self, a, b):
+ if a[0] == b[0]:
+ if a[1] == b[1]:
+ return a[2] < b[2]
+ else:
+ return a[1] < b[1]
+ else:
+ return a[0] < b[0]
+
+class StorageStatistics(object):
+ # Key: (operation_name, old_storage, new_storage)
+ # Value: [sizes]
+ stats = {}
+
+ do_log = False
+ do_stats = False
+ do_stats_sizes = False
+
+ def log(self, w_obj, operation, old_storage_object, log_classname):
+ if self.do_log or self.do_stats:
+ new_storage = w_obj.shadow.repr_classname
+ if old_storage_object:
+ old_storage = old_storage_object.repr_classname
+ else:
+ old_storage = None
+ size = w_obj.size()
+
+ key = self.make_key(operation, old_storage, new_storage)
+ if _stats.do_stats:
+ self.stat_operation(key, size)
+ if self.do_log:
+ if log_classname:
+ classname = w_obj.guess_classname()
+ else:
+ classname = None
+ self.log_operation(key, size, classname)
+
+ def make_key(self, operation, old_storage, new_storage):
+ return (operation, old_storage, new_storage)
+
+ def stat_operation(self, key, size):
+ if not key in self.stats:
+ self.stats[key] = []
+ self.stats[key].append(size)
+
+ def log_operation(self, key, size, classname):
+ print self.log_operation_string(key, size, classname)
+
+ def key_string(self, key):
+ if key[1]:
+ return "%s (%s -> %s)" % (key[0], key[1], key[2])
+ else:
+ return "%s (%s)" % (key[0], key[2])
+
+ def log_operation_string(self, key, size, classname):
+ if classname:
+ return "%s of %s size %d" % (self.key_string(key), classname, size)
+ else:
+ return "%s size %d" % (self.key_string(key), size)
+
+ def sorted_keys(self):
+ keys = [ x for x in self.stats ]
+ StatsSorter(keys).sort()
+ return keys
+
+ def print_stats(self):
+ for key in self.sorted_keys():
+ sizes = self.stats[key]
+ sum = 0
+ for s in sizes: sum += s
+ print "%s: %d times, avg size: %f" % (self.key_string(key),
len(sizes), sum/len(sizes))
+ if self.do_stats_sizes:
+ print " All sizes: %s" % sizes
+
+_stats = StorageStatistics()
+
+def activate_statistics(log=False, statistics=False, statstics_sizes=False):
+ _stats.do_log = _stats.do_log or log
+ _stats.do_stats = _stats.do_stats or statistics
+ _stats.do_stats_sizes = _stats.do_stats_sizes or statstics_sizes
+
+def print_statistics():
+ if _stats.do_stats:
+ _stats.print_stats()
+
+def log(w_obj, operation, old_storage=None, log_classname=True):
+ _stats.log(w_obj, operation, old_storage, log_classname)
diff --git a/spyvm/test/test_miniimage.py b/spyvm/test/test_miniimage.py
--- a/spyvm/test/test_miniimage.py
+++ b/spyvm/test/test_miniimage.py
@@ -369,7 +369,7 @@
def test_primitive_perform_with_args():
from spyvm.test.test_primitives import _prim
w_o = space.wrap_list([1, 2, 3])
- w_methoddict =
w_o.class_shadow(space)._s_superclass._s_superclass.w_methoddict()
+ w_methoddict =
w_o.class_shadow(space).s_superclass().s_superclass().w_methoddict()
w_methoddict.as_methoddict_get_shadow(space).sync_method_cache()
selectors_w = w_methoddict.shadow.methoddict.keys()
w_sel = None
diff --git a/spyvm/test/test_strategies.py b/spyvm/test/test_strategies.py
--- a/spyvm/test/test_strategies.py
+++ b/spyvm/test/test_strategies.py
@@ -1,5 +1,5 @@
import py
-from spyvm import wrapper, model, interpreter, shadow
+from spyvm import wrapper, model, interpreter, shadow, storage_statistics
from spyvm.error import WrapperException, FatalError
from .util import read_image, copy_to_module, cleanup_module
@@ -176,20 +176,27 @@
assert isinstance(a.shadow, shadow.ListStorageShadow)
check_arr(a, [1.2, 2, w_nil, w_nil, w_nil])
-def test_statistics():
- stats = model.StrategyStatistics()
- stats.stat_operation("B", "old", "new", 3)
- stats.stat_operation("B", "old", "new", 4)
- stats.stat_operation("B", "old2", "new2", 20)
- stats.stat_operation("B", "old", "new", 5)
- stats.stat_operation("A", "old", "new", 1)
- stats.stat_operation("A", "old", "new", 2)
- stats.stat_operation("C", "old", "new", 10)
- stats.stat_operation("C", "old", "new", 11)
+def test_statistics_stats():
+ stats = storage_statistics.StorageStatistics()
+ stats.stat_operation(stats.make_key("B", "old", "new"), 3)
+ stats.stat_operation(stats.make_key("B", "old", "new"), 4)
+ stats.stat_operation(stats.make_key("B", "old2", "new2"), 20)
+ stats.stat_operation(stats.make_key("B", "old", "new"), 5)
+ stats.stat_operation(stats.make_key("A", "old", "new"), 1)
+ stats.stat_operation(stats.make_key("A", "old", "new"), 2)
+ stats.stat_operation(stats.make_key("C", "old", "new"), 10)
+ stats.stat_operation(stats.make_key("C", "old", "new"), 11)
keys = stats.sorted_keys()
assert keys == [ ("A", "old", "new"), ("B", "old", "new"), ("B", "old2",
"new2"), ("C", "old", "new") ]
assert stats.stats[keys[0]] == [1, 2]
assert stats.stats[keys[1]] == [3, 4, 5]
assert stats.stats[keys[2]] == [20]
assert stats.stats[keys[3]] == [10, 11]
+
+def test_statistics_log():
+ stats = storage_statistics.StorageStatistics()
+ s = stats.log_operation_string(stats.make_key("Operation", "old_storage",
"new_storage"), 22, "classname")
+ assert s == "Operation (old_storage -> new_storage) of classname size 22"
+ s = stats.log_operation_string(stats.make_key("InitialOperation", None,
"some_new_storage"), 40, "a_classname")
+ assert s == "InitialOperation (some_new_storage) of a_classname size 40"
\ No newline at end of file
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -6,7 +6,7 @@
from rpython.rlib import jit, rpath
from spyvm import model, interpreter, squeakimage, objspace, wrapper,\
- error, shadow
+ error, shadow, storage_statistics
from spyvm.tool.analyseimage import create_image
from spyvm.interpreter_proxy import VirtualMachine
@@ -185,12 +185,11 @@
as_benchmark = True
idx += 1
elif arg == "--strategy-log":
- model.strategy_stats.do_log = True
+ storage_statistics.activate_statistics(log=True)
elif arg == "--strategy-stats":
- model.strategy_stats.do_stats = True
+ storage_statistics.activate_statistics(statistics=True)
elif arg == "--strategy-stats-with-sizes":
- model.strategy_stats.do_stats = True
- model.strategy_stats.do_stats_sizes = True
+ storage_statistics.activate_statistics(statistics=True,
statstics_sizes=True)
elif path is None:
path = argv[idx]
else:
@@ -224,8 +223,7 @@
else:
_run_image(interp)
result = 0
- if model.strategy_stats.do_stats:
- model.strategy_stats.print_stats()
+ storage_statistics.print_statistics()
return result
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
