Author: Richard Plangger <planri...@gmail.com> Branch: memop-simplify Changeset: r80939:80594aadbcd0 Date: 2015-11-25 13:46 +0100 http://bitbucket.org/pypy/pypy/changeset/80594aadbcd0/
Log: partly renamed getarrayitem_gc to gc_load, reenabled other ops that will be removed (that was a too big "big bang" approach) diff too long, truncating to 2000 out of 2536 lines diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -72,6 +72,7 @@ self._cdefsources = [] self._included_ffis = [] self._windows_unicode = None + self._init_once_cache = {} if hasattr(backend, 'set_ffi'): backend.set_ffi(self) for name in backend.__dict__: @@ -598,6 +599,30 @@ return recompile(self, module_name, source, tmpdir=tmpdir, source_extension=source_extension, **kwds) + def init_once(self, func, tag): + # Read _init_once_cache[tag], which is either (False, lock) if + # we're calling the function now in some thread, or (True, result). + # Don't call setdefault() in most cases, to avoid allocating and + # immediately freeing a lock; but still use setdefaut() to avoid + # races. + try: + x = self._init_once_cache[tag] + except KeyError: + x = self._init_once_cache.setdefault(tag, (False, allocate_lock())) + # Common case: we got (True, result), so we return the result. + if x[0]: + return x[1] + # Else, it's a lock. Acquire it to serialize the following tests. + with x[1]: + # Read again from _init_once_cache the current status. + x = self._init_once_cache[tag] + if x[0]: + return x[1] + # Call the function and store the result back. + result = func() + self._init_once_cache[tag] = (True, result) + return result + def _load_backend_lib(backend, name, flags): if name is None: diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h --- a/lib_pypy/cffi/parse_c_type.h +++ b/lib_pypy/cffi/parse_c_type.h @@ -1,5 +1,6 @@ -/* See doc/misc/parse_c_type.rst in the source of CFFI for more information */ +/* This part is from file 'cffi/parse_c_type.h'. It is copied at the + beginning of C sources generated by CFFI's ffi.set_source(). */ typedef void *_cffi_opcode_t; diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -21,6 +21,8 @@ import math as _math import struct as _struct +_SENTINEL = object() + def _cmp(x, y): return 0 if x == y else 1 if x > y else -1 @@ -31,6 +33,8 @@ MAXYEAR = 9999 _MINYEARFMT = 1900 +_MAX_DELTA_DAYS = 999999999 + # Utility functions, adapted from Python's Demo/classes/Dates.py, which # also assumes the current Gregorian calendar indefinitely extended in # both directions. Difference: Dates.py calls January 1 of year 0 day @@ -95,6 +99,15 @@ # pasting together 25 4-year cycles. assert _DI100Y == 25 * _DI4Y - 1 +_US_PER_US = 1 +_US_PER_MS = 1000 +_US_PER_SECOND = 1000000 +_US_PER_MINUTE = 60000000 +_SECONDS_PER_DAY = 24 * 3600 +_US_PER_HOUR = 3600000000 +_US_PER_DAY = 86400000000 +_US_PER_WEEK = 604800000000 + def _ord2ymd(n): "ordinal -> (year, month, day), considering 01-Jan-0001 as day 1." @@ -271,15 +284,17 @@ def _check_int_field(value): if isinstance(value, int): - return value + return int(value) if not isinstance(value, float): try: value = value.__int__() except AttributeError: pass else: - if isinstance(value, (int, long)): - return value + if isinstance(value, int): + return int(value) + elif isinstance(value, long): + return int(long(value)) raise TypeError('__int__ method should return an integer') raise TypeError('an integer is required') raise TypeError('integer argument expected, got float') @@ -344,75 +359,79 @@ raise TypeError("can't compare '%s' to '%s'" % ( type(x).__name__, type(y).__name__)) -# This is a start at a struct tm workalike. Goals: -# -# + Works the same way across platforms. -# + Handles all the fields datetime needs handled, without 1970-2038 glitches. -# -# Note: I suspect it's best if this flavor of tm does *not* try to -# second-guess timezones or DST. Instead fold whatever adjustments you want -# into the minutes argument (and the constructor will normalize). +def _normalize_pair(hi, lo, factor): + if not 0 <= lo <= factor-1: + inc, lo = divmod(lo, factor) + hi += inc + return hi, lo -class _tmxxx: +def _normalize_datetime(y, m, d, hh, mm, ss, us, ignore_overflow=False): + # Normalize all the inputs, and store the normalized values. + ss, us = _normalize_pair(ss, us, 1000000) + mm, ss = _normalize_pair(mm, ss, 60) + hh, mm = _normalize_pair(hh, mm, 60) + d, hh = _normalize_pair(d, hh, 24) + y, m, d = _normalize_date(y, m, d, ignore_overflow) + return y, m, d, hh, mm, ss, us - ordinal = None +def _normalize_date(year, month, day, ignore_overflow=False): + # That was easy. Now it gets muddy: the proper range for day + # can't be determined without knowing the correct month and year, + # but if day is, e.g., plus or minus a million, the current month + # and year values make no sense (and may also be out of bounds + # themselves). + # Saying 12 months == 1 year should be non-controversial. + if not 1 <= month <= 12: + year, month = _normalize_pair(year, month-1, 12) + month += 1 + assert 1 <= month <= 12 - def __init__(self, year, month, day, hour=0, minute=0, second=0, - microsecond=0): - # Normalize all the inputs, and store the normalized values. - if not 0 <= microsecond <= 999999: - carry, microsecond = divmod(microsecond, 1000000) - second += carry - if not 0 <= second <= 59: - carry, second = divmod(second, 60) - minute += carry - if not 0 <= minute <= 59: - carry, minute = divmod(minute, 60) - hour += carry - if not 0 <= hour <= 23: - carry, hour = divmod(hour, 24) - day += carry + # Now only day can be out of bounds (year may also be out of bounds + # for a datetime object, but we don't care about that here). + # If day is out of bounds, what to do is arguable, but at least the + # method here is principled and explainable. + dim = _days_in_month(year, month) + if not 1 <= day <= dim: + # Move day-1 days from the first of the month. First try to + # get off cheap if we're only one day out of range (adjustments + # for timezone alone can't be worse than that). + if day == 0: # move back a day + month -= 1 + if month > 0: + day = _days_in_month(year, month) + else: + year, month, day = year-1, 12, 31 + elif day == dim + 1: # move forward a day + month += 1 + day = 1 + if month > 12: + month = 1 + year += 1 + else: + ordinal = _ymd2ord(year, month, 1) + (day - 1) + year, month, day = _ord2ymd(ordinal) - # That was easy. Now it gets muddy: the proper range for day - # can't be determined without knowing the correct month and year, - # but if day is, e.g., plus or minus a million, the current month - # and year values make no sense (and may also be out of bounds - # themselves). - # Saying 12 months == 1 year should be non-controversial. - if not 1 <= month <= 12: - carry, month = divmod(month-1, 12) - year += carry - month += 1 - assert 1 <= month <= 12 + if not ignore_overflow and not MINYEAR <= year <= MAXYEAR: + raise OverflowError("date value out of range") + return year, month, day - # Now only day can be out of bounds (year may also be out of bounds - # for a datetime object, but we don't care about that here). - # If day is out of bounds, what to do is arguable, but at least the - # method here is principled and explainable. - dim = _days_in_month(year, month) - if not 1 <= day <= dim: - # Move day-1 days from the first of the month. First try to - # get off cheap if we're only one day out of range (adjustments - # for timezone alone can't be worse than that). - if day == 0: # move back a day - month -= 1 - if month > 0: - day = _days_in_month(year, month) - else: - year, month, day = year-1, 12, 31 - elif day == dim + 1: # move forward a day - month += 1 - day = 1 - if month > 12: - month = 1 - year += 1 - else: - self.ordinal = _ymd2ord(year, month, 1) + (day - 1) - year, month, day = _ord2ymd(self.ordinal) - - self.year, self.month, self.day = year, month, day - self.hour, self.minute, self.second = hour, minute, second - self.microsecond = microsecond +def _accum(tag, sofar, num, factor, leftover): + if isinstance(num, (int, long)): + prod = num * factor + rsum = sofar + prod + return rsum, leftover + if isinstance(num, float): + fracpart, intpart = _math.modf(num) + prod = int(intpart) * factor + rsum = sofar + prod + if fracpart == 0.0: + return rsum, leftover + assert isinstance(factor, (int, long)) + fracpart, intpart = _math.modf(factor * fracpart) + rsum += int(intpart) + return rsum, leftover + fracpart + raise TypeError("unsupported type for timedelta %s component: %s" % + (tag, type(num))) class timedelta(object): """Represent the difference between two datetime objects. @@ -433,100 +452,42 @@ """ __slots__ = '_days', '_seconds', '_microseconds', '_hashcode' - def __new__(cls, days=0, seconds=0, microseconds=0, - milliseconds=0, minutes=0, hours=0, weeks=0): - # Doing this efficiently and accurately in C is going to be difficult - # and error-prone, due to ubiquitous overflow possibilities, and that - # C double doesn't have enough bits of precision to represent - # microseconds over 10K years faithfully. The code here tries to make - # explicit where go-fast assumptions can be relied on, in order to - # guide the C implementation; it's way more convoluted than speed- - # ignoring auto-overflow-to-long idiomatic Python could be. + def __new__(cls, days=_SENTINEL, seconds=_SENTINEL, microseconds=_SENTINEL, + milliseconds=_SENTINEL, minutes=_SENTINEL, hours=_SENTINEL, weeks=_SENTINEL): + x = 0 + leftover = 0.0 + if microseconds is not _SENTINEL: + x, leftover = _accum("microseconds", x, microseconds, _US_PER_US, leftover) + if milliseconds is not _SENTINEL: + x, leftover = _accum("milliseconds", x, milliseconds, _US_PER_MS, leftover) + if seconds is not _SENTINEL: + x, leftover = _accum("seconds", x, seconds, _US_PER_SECOND, leftover) + if minutes is not _SENTINEL: + x, leftover = _accum("minutes", x, minutes, _US_PER_MINUTE, leftover) + if hours is not _SENTINEL: + x, leftover = _accum("hours", x, hours, _US_PER_HOUR, leftover) + if days is not _SENTINEL: + x, leftover = _accum("days", x, days, _US_PER_DAY, leftover) + if weeks is not _SENTINEL: + x, leftover = _accum("weeks", x, weeks, _US_PER_WEEK, leftover) + if leftover != 0.0: + x += _round(leftover) + return cls._from_microseconds(x) - # XXX Check that all inputs are ints, longs or floats. + @classmethod + def _from_microseconds(cls, us): + s, us = divmod(us, _US_PER_SECOND) + d, s = divmod(s, _SECONDS_PER_DAY) + return cls._create(d, s, us, False) - # Final values, all integer. - # s and us fit in 32-bit signed ints; d isn't bounded. - d = s = us = 0 + @classmethod + def _create(cls, d, s, us, normalize): + if normalize: + s, us = _normalize_pair(s, us, 1000000) + d, s = _normalize_pair(d, s, 24*3600) - # Normalize everything to days, seconds, microseconds. - days += weeks*7 - seconds += minutes*60 + hours*3600 - microseconds += milliseconds*1000 - - # Get rid of all fractions, and normalize s and us. - # Take a deep breath <wink>. - if isinstance(days, float): - dayfrac, days = _math.modf(days) - daysecondsfrac, daysecondswhole = _math.modf(dayfrac * (24.*3600.)) - assert daysecondswhole == int(daysecondswhole) # can't overflow - s = int(daysecondswhole) - assert days == int(days) - d = int(days) - else: - daysecondsfrac = 0.0 - d = days - assert isinstance(daysecondsfrac, float) - assert abs(daysecondsfrac) <= 1.0 - assert isinstance(d, (int, long)) - assert abs(s) <= 24 * 3600 - # days isn't referenced again before redefinition - - if isinstance(seconds, float): - secondsfrac, seconds = _math.modf(seconds) - assert seconds == int(seconds) - seconds = int(seconds) - secondsfrac += daysecondsfrac - assert abs(secondsfrac) <= 2.0 - else: - secondsfrac = daysecondsfrac - # daysecondsfrac isn't referenced again - assert isinstance(secondsfrac, float) - assert abs(secondsfrac) <= 2.0 - - assert isinstance(seconds, (int, long)) - days, seconds = divmod(seconds, 24*3600) - d += days - s += int(seconds) # can't overflow - assert isinstance(s, int) - assert abs(s) <= 2 * 24 * 3600 - # seconds isn't referenced again before redefinition - - usdouble = secondsfrac * 1e6 - assert abs(usdouble) < 2.1e6 # exact value not critical - # secondsfrac isn't referenced again - - if isinstance(microseconds, float): - microseconds = _round(microseconds + usdouble) - seconds, microseconds = divmod(microseconds, 1000000) - days, seconds = divmod(seconds, 24*3600) - d += days - s += int(seconds) - microseconds = int(microseconds) - else: - microseconds = int(microseconds) - seconds, microseconds = divmod(microseconds, 1000000) - days, seconds = divmod(seconds, 24*3600) - d += days - s += int(seconds) - microseconds = _round(microseconds + usdouble) - assert isinstance(s, int) - assert isinstance(microseconds, int) - assert abs(s) <= 3 * 24 * 3600 - assert abs(microseconds) < 3.1e6 - - # Just a little bit of carrying possible for microseconds and seconds. - seconds, us = divmod(microseconds, 1000000) - s += seconds - days, s = divmod(s, 24*3600) - d += days - - assert isinstance(d, (int, long)) - assert isinstance(s, int) and 0 <= s < 24*3600 - assert isinstance(us, int) and 0 <= us < 1000000 - - if abs(d) > 999999999: - raise OverflowError("timedelta # of days is too large: %d" % d) + if not -_MAX_DELTA_DAYS <= d <= _MAX_DELTA_DAYS: + raise OverflowError("days=%d; must have magnitude <= %d" % (d, _MAX_DELTA_DAYS)) self = object.__new__(cls) self._days = d @@ -535,6 +496,10 @@ self._hashcode = -1 return self + def _to_microseconds(self): + return ((self._days * _SECONDS_PER_DAY + self._seconds) * _US_PER_SECOND + + self._microseconds) + def __repr__(self): module = "datetime." if self.__class__ is timedelta else "" if self._microseconds: @@ -562,8 +527,7 @@ def total_seconds(self): """Total seconds in the duration.""" - return ((self.days * 86400 + self.seconds) * 10**6 + - self.microseconds) / 10**6 + return self._to_microseconds() / 10**6 # Read-only field accessors @property @@ -585,36 +549,37 @@ if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self._days + other._days, - self._seconds + other._seconds, - self._microseconds + other._microseconds) + return timedelta._create(self._days + other._days, + self._seconds + other._seconds, + self._microseconds + other._microseconds, + True) return NotImplemented - __radd__ = __add__ - def __sub__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(self._days - other._days, - self._seconds - other._seconds, - self._microseconds - other._microseconds) - return NotImplemented - - def __rsub__(self, other): - if isinstance(other, timedelta): - return -self + other + return timedelta._create(self._days - other._days, + self._seconds - other._seconds, + self._microseconds - other._microseconds, + True) return NotImplemented def __neg__(self): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta - return timedelta(-self._days, - -self._seconds, - -self._microseconds) + return timedelta._create(-self._days, + -self._seconds, + -self._microseconds, + True) def __pos__(self): - return self + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta._create(self._days, + self._seconds, + self._microseconds, + False) def __abs__(self): if self._days < 0: @@ -623,25 +588,18 @@ return self def __mul__(self, other): - if isinstance(other, (int, long)): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(self._days * other, - self._seconds * other, - self._microseconds * other) - return NotImplemented + if not isinstance(other, (int, long)): + return NotImplemented + usec = self._to_microseconds() + return timedelta._from_microseconds(usec * other) __rmul__ = __mul__ - def _to_microseconds(self): - return ((self._days * (24*3600) + self._seconds) * 1000000 + - self._microseconds) - def __div__(self, other): if not isinstance(other, (int, long)): return NotImplemented usec = self._to_microseconds() - return timedelta(0, 0, usec // other) + return timedelta._from_microseconds(usec // other) __floordiv__ = __div__ @@ -705,9 +663,8 @@ def __reduce__(self): return (self.__class__, self._getstate()) -timedelta.min = timedelta(-999999999) -timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, - microseconds=999999) +timedelta.min = timedelta(-_MAX_DELTA_DAYS) +timedelta.max = timedelta(_MAX_DELTA_DAYS, 24*3600-1, 1000000-1) timedelta.resolution = timedelta(microseconds=1) class date(object): @@ -948,32 +905,29 @@ # Computations - def _checkOverflow(self, year): - if not MINYEAR <= year <= MAXYEAR: - raise OverflowError("date +/-: result year %d not in %d..%d" % - (year, MINYEAR, MAXYEAR)) + def _add_timedelta(self, other, factor): + y, m, d = _normalize_date( + self._year, + self._month, + self._day + other.days * factor) + return date(y, m, d) def __add__(self, other): "Add a date to a timedelta." if isinstance(other, timedelta): - t = _tmxxx(self._year, - self._month, - self._day + other.days) - self._checkOverflow(t.year) - result = date(t.year, t.month, t.day) - return result + return self._add_timedelta(other, 1) return NotImplemented __radd__ = __add__ def __sub__(self, other): """Subtract two dates, or a date and a timedelta.""" - if isinstance(other, timedelta): - return self + timedelta(-other.days) if isinstance(other, date): days1 = self.toordinal() days2 = other.toordinal() - return timedelta(days1 - days2) + return timedelta._create(days1 - days2, 0, 0, False) + if isinstance(other, timedelta): + return self._add_timedelta(other, -1) return NotImplemented def weekday(self): @@ -1340,7 +1294,7 @@ offset = self._tzinfo.utcoffset(None) offset = _check_utc_offset("utcoffset", offset) if offset is not None: - offset = timedelta(minutes=offset) + offset = timedelta._create(0, offset * 60, 0, True) return offset # Return an integer (or None) instead of a timedelta (or None). @@ -1378,7 +1332,7 @@ offset = self._tzinfo.dst(None) offset = _check_utc_offset("dst", offset) if offset is not None: - offset = timedelta(minutes=offset) + offset = timedelta._create(0, offset * 60, 0, True) return offset # Return an integer (or None) instead of a timedelta (or None). @@ -1505,18 +1459,24 @@ A timezone info object may be passed in as well. """ + _check_tzinfo_arg(tz) + converter = _time.localtime if tz is None else _time.gmtime + self = cls._from_timestamp(converter, timestamp, tz) + if tz is not None: + self = tz.fromutc(self) + return self - _check_tzinfo_arg(tz) + @classmethod + def utcfromtimestamp(cls, t): + "Construct a UTC datetime from a POSIX timestamp (like time.time())." + return cls._from_timestamp(_time.gmtime, t, None) - converter = _time.localtime if tz is None else _time.gmtime - - if isinstance(timestamp, int): - us = 0 - else: - t_full = timestamp - timestamp = int(_math.floor(timestamp)) - frac = t_full - timestamp - us = _round(frac * 1e6) + @classmethod + def _from_timestamp(cls, converter, timestamp, tzinfo): + t_full = timestamp + timestamp = int(_math.floor(timestamp)) + frac = t_full - timestamp + us = _round(frac * 1e6) # If timestamp is less than one microsecond smaller than a # full second, us can be rounded up to 1000000. In this case, @@ -1527,32 +1487,7 @@ us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them - result = cls(y, m, d, hh, mm, ss, us, tz) - if tz is not None: - result = tz.fromutc(result) - return result - - @classmethod - def utcfromtimestamp(cls, t): - "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if isinstance(t, int): - us = 0 - else: - t_full = t - t = int(_math.floor(t)) - frac = t_full - t - us = _round(frac * 1e6) - - # If timestamp is less than one microsecond smaller than a - # full second, us can be rounded up to 1000000. In this case, - # roll over to seconds, otherwise, ValueError is raised - # by the constructor. - if us == 1000000: - t += 1 - us = 0 - y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) - ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us) + return cls(y, m, d, hh, mm, ss, us, tzinfo) @classmethod def now(cls, tz=None): @@ -1594,9 +1529,9 @@ hh, mm, ss = self.hour, self.minute, self.second offset = self._utcoffset() if offset: # neither None nor 0 - tm = _tmxxx(y, m, d, hh, mm - offset) - y, m, d = tm.year, tm.month, tm.day - hh, mm = tm.hour, tm.minute + mm -= offset + y, m, d, hh, mm, ss, _ = _normalize_datetime( + y, m, d, hh, mm, ss, 0, ignore_overflow=True) return _build_struct_time(y, m, d, hh, mm, ss, 0) def date(self): @@ -1730,7 +1665,7 @@ offset = self._tzinfo.utcoffset(self) offset = _check_utc_offset("utcoffset", offset) if offset is not None: - offset = timedelta(minutes=offset) + offset = timedelta._create(0, offset * 60, 0, True) return offset # Return an integer (or None) instead of a timedelta (or None). @@ -1768,7 +1703,7 @@ offset = self._tzinfo.dst(self) offset = _check_utc_offset("dst", offset) if offset is not None: - offset = timedelta(minutes=offset) + offset = timedelta._create(0, offset * 60, 0, True) return offset # Return an integer (or None) instead of a timedelta (or None). @@ -1859,22 +1794,22 @@ return -1 return diff and 1 or 0 + def _add_timedelta(self, other, factor): + y, m, d, hh, mm, ss, us = _normalize_datetime( + self._year, + self._month, + self._day + other.days * factor, + self._hour, + self._minute, + self._second + other.seconds * factor, + self._microsecond + other.microseconds * factor) + return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) + def __add__(self, other): "Add a datetime and a timedelta." if not isinstance(other, timedelta): return NotImplemented - t = _tmxxx(self._year, - self._month, - self._day + other.days, - self._hour, - self._minute, - self._second + other.seconds, - self._microsecond + other.microseconds) - self._checkOverflow(t.year) - result = datetime(t.year, t.month, t.day, - t.hour, t.minute, t.second, - t.microsecond, tzinfo=self._tzinfo) - return result + return self._add_timedelta(other, 1) __radd__ = __add__ @@ -1882,16 +1817,15 @@ "Subtract two datetimes, or a datetime and a timedelta." if not isinstance(other, datetime): if isinstance(other, timedelta): - return self + -other + return self._add_timedelta(other, -1) return NotImplemented - days1 = self.toordinal() - days2 = other.toordinal() - secs1 = self._second + self._minute * 60 + self._hour * 3600 - secs2 = other._second + other._minute * 60 + other._hour * 3600 - base = timedelta(days1 - days2, - secs1 - secs2, - self._microsecond - other._microsecond) + delta_d = self.toordinal() - other.toordinal() + delta_s = (self._hour - other._hour) * 3600 + \ + (self._minute - other._minute) * 60 + \ + (self._second - other._second) + delta_us = self._microsecond - other._microsecond + base = timedelta._create(delta_d, delta_s, delta_us, True) if self._tzinfo is other._tzinfo: return base myoff = self._utcoffset() diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -15,4 +15,17 @@ Fix the cpyext tests on OSX by linking with -flat_namespace .. branch: anntype + Refactor and improve exception analysis in the annotator. + +.. branch: posita/2193-datetime-timedelta-integrals + +Fix issue #2193. ``isinstance(..., int)`` => ``isinstance(..., numbers.Integral)`` +to allow for alternate ``int``-like implementations (e.g., ``future.types.newint``) + +.. branch: faster-rstruct + +Improve the performace of struct.unpack, which now directly reads inside the +string buffer and directly casts the bytes to the appropriate type, when +allowed. Unpacking of floats and doubles is about 15 times faster now, while +for integer types it's up to ~50% faster for 64bit integers. diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -49,6 +49,8 @@ ACCEPT_CDATA = ACCEPT_CDATA w_gc_wref_remove = None + w_init_once_cache = None + jit_init_once_cache = None @jit.dont_look_inside def __init__(self, space, src_ctx): @@ -585,6 +587,59 @@ return w_result + def descr_init_once(self, w_func, w_tag): + """XXX document me""" + # + # first, a fast-path for the JIT which only works if the very + # same w_tag object is passed; then it turns into no code at all + try: + return self._init_once_elidable(w_tag) + except KeyError: + return self._init_once_slowpath(w_func, w_tag) + + @jit.elidable + def _init_once_elidable(self, w_tag): + jit_cache = self.jit_init_once_cache + if jit_cache is not None: + return jit_cache[w_tag] + else: + raise KeyError + + @jit.dont_look_inside + def _init_once_slowpath(self, w_func, w_tag): + space = self.space + w_cache = self.w_init_once_cache + if w_cache is None: + w_cache = self.space.newdict() + jit_cache = {} + self.w_init_once_cache = w_cache + self.jit_init_once_cache = jit_cache + # + # get the lock or result from cache[tag] + w_res = space.finditem(w_cache, w_tag) + if w_res is None: + w_res = W_InitOnceLock(space) + w_res = space.call_method(w_cache, 'setdefault', w_tag, w_res) + if not isinstance(w_res, W_InitOnceLock): + return w_res + with w_res.lock: + w_res = space.finditem(w_cache, w_tag) + if w_res is None or isinstance(w_res, W_InitOnceLock): + w_res = space.call_function(w_func) + self.jit_init_once_cache[w_tag] = w_res + space.setitem(w_cache, w_tag, w_res) + else: + # the real result was put in the dict while we were + # waiting for lock.__enter__() above + pass + return w_res + + +class W_InitOnceLock(W_Root): + def __init__(self, space): + self.lock = space.allocate_lock() + + @jit.dont_look_inside def make_plain_ffi_object(space, w_ffitype=None): if w_ffitype is None: @@ -641,6 +696,7 @@ from_handle = interp2app(W_FFIObject.descr_from_handle), gc = interp2app(W_FFIObject.descr_gc), getctype = interp2app(W_FFIObject.descr_getctype), + init_once = interp2app(W_FFIObject.descr_init_once), integer_const = interp2app(W_FFIObject.descr_integer_const), memmove = interp2app(W_FFIObject.descr_memmove), new = interp2app(W_FFIObject.descr_new), diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -447,3 +447,19 @@ assert int(ffi.cast("_Bool", ffi.cast(type, 42))) == 1 assert int(ffi.cast("bool", ffi.cast(type, 42))) == 1 assert int(ffi.cast("_Bool", ffi.cast(type, 0))) == 0 + + def test_init_once(self): + import _cffi_backend as _cffi1_backend + def do_init(): + seen.append(1) + return 42 + ffi = _cffi1_backend.FFI() + seen = [] + for i in range(3): + res = ffi.init_once(do_init, "tag1") + assert res == 42 + assert seen == [1] + for i in range(3): + res = ffi.init_once(do_init, "tag2") + assert res == 42 + assert seen == [1, 1] diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -8,6 +8,7 @@ from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import specialize, instantiate from rpython.rlib.nonconst import NonConstant +from rpython.rlib.rarithmetic import base_int from pypy.module.micronumpy import boxes, ufuncs from pypy.module.micronumpy.arrayops import where from pypy.module.micronumpy.ndarray import W_NDimArray @@ -178,7 +179,7 @@ return BoolObject(obj) elif isinstance(obj, int): return IntObject(obj) - elif isinstance(obj, long): + elif isinstance(obj, base_int): return LongObject(obj) elif isinstance(obj, W_Root): return obj @@ -196,31 +197,31 @@ return self.float(f) def le(self, w_obj1, w_obj2): - assert isinstance(w_obj1, boxes.W_GenericBox) - assert isinstance(w_obj2, boxes.W_GenericBox) + assert isinstance(w_obj1, boxes.W_GenericBox) + assert isinstance(w_obj2, boxes.W_GenericBox) return w_obj1.descr_le(self, w_obj2) def lt(self, w_obj1, w_obj2): - assert isinstance(w_obj1, boxes.W_GenericBox) - assert isinstance(w_obj2, boxes.W_GenericBox) + assert isinstance(w_obj1, boxes.W_GenericBox) + assert isinstance(w_obj2, boxes.W_GenericBox) return w_obj1.descr_lt(self, w_obj2) def ge(self, w_obj1, w_obj2): - assert isinstance(w_obj1, boxes.W_GenericBox) - assert isinstance(w_obj2, boxes.W_GenericBox) + assert isinstance(w_obj1, boxes.W_GenericBox) + assert isinstance(w_obj2, boxes.W_GenericBox) return w_obj1.descr_ge(self, w_obj2) def add(self, w_obj1, w_obj2): - assert isinstance(w_obj1, boxes.W_GenericBox) - assert isinstance(w_obj2, boxes.W_GenericBox) + assert isinstance(w_obj1, boxes.W_GenericBox) + assert isinstance(w_obj2, boxes.W_GenericBox) return w_obj1.descr_add(self, w_obj2) def sub(self, w_obj1, w_obj2): return self.wrap(1) def mul(self, w_obj1, w_obj2): - assert isinstance(w_obj1, boxes.W_GenericBox) - assert isinstance(w_obj2, boxes.W_GenericBox) + assert isinstance(w_obj1, boxes.W_GenericBox) + assert isinstance(w_obj2, boxes.W_GenericBox) return w_obj1.descr_mul(self, w_obj2) def pow(self, w_obj1, w_obj2, _): @@ -836,7 +837,7 @@ elif self.name == 'reshape': w_arg = self.args[1] assert isinstance(w_arg, ArrayConstant) - order = -1 + order = -1 w_res = arr.reshape(interp.space, w_arg.wrap(interp.space), order) else: assert False diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py --- a/pypy/module/pypyjit/test_pypy_c/test_struct.py +++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py @@ -19,7 +19,8 @@ import struct i = 1 while i < n: - x = struct.unpack("i", struct.pack("i", i))[0] # ID: struct + buf = struct.pack("i", i) # ID: pack + x = struct.unpack("i", buf)[0] # ID: unpack i += x / i return i @@ -29,7 +30,7 @@ loop, = log.loops_by_filename(self.filepath) # This could, of course stand some improvement, to remove all these # arithmatic ops, but we've removed all the core overhead. - assert loop.match_by_id("struct", """ + assert loop.match_by_id("pack", """ guard_not_invalidated(descr=...) # struct.pack %s @@ -40,17 +41,22 @@ i17 = int_and(i16, 255) i19 = int_rshift(i16, 8) i20 = int_and(i19, 255) + """ % extra) + # the newstr and the strsetitems are because the string is forced, + # which is in turn because the optimizer doesn't know how to handle a + # getarrayitem_gc_i on a virtual string. It could be improved, but it + # is also true that in real life cases struct.unpack is called on + # strings which come from the outside, so it's a minor issue. + assert loop.match_by_id("unpack", """ # struct.unpack - i22 = int_lshift(i14, 8) - i23 = int_or(i11, i22) - i25 = int_lshift(i17, 16) - i26 = int_or(i23, i25) - i28 = int_ge(i20, 128) - guard_false(i28, descr=...) - i30 = int_lshift(i20, 24) - i31 = int_or(i26, i30) - """ % extra) + p88 = newstr(4) + strsetitem(p88, 0, i11) + strsetitem(p88, 1, i14) + strsetitem(p88, 2, i17) + strsetitem(p88, 3, i20) + i91 = getarrayitem_gc_i(p88, 0, descr=<ArrayS 4>) + """) def test_struct_object(self): def main(n): @@ -58,7 +64,8 @@ s = struct.Struct("i") i = 1 while i < n: - x = s.unpack(s.pack(i))[0] # ID: struct + buf = s.pack(i) # ID: pack + x = s.unpack(buf)[0] # ID: unpack i += x / i return i @@ -66,7 +73,7 @@ assert log.result == main(1000) loop, = log.loops_by_filename(self.filepath) - assert loop.match_by_id('struct', """ + assert loop.match_by_id('pack', """ guard_not_invalidated(descr=...) # struct.pack %s @@ -77,14 +84,14 @@ i17 = int_and(i16, 255) i19 = int_rshift(i16, 8) i20 = int_and(i19, 255) + """ % extra) + assert loop.match_by_id('unpack', """ # struct.unpack - i22 = int_lshift(i14, 8) - i23 = int_or(i11, i22) - i25 = int_lshift(i17, 16) - i26 = int_or(i23, i25) - i28 = int_ge(i20, 128) - guard_false(i28, descr=...) - i30 = int_lshift(i20, 24) - i31 = int_or(i26, i30) - """ % extra) + p88 = newstr(4) + strsetitem(p88, 0, i11) + strsetitem(p88, 1, i14) + strsetitem(p88, 2, i17) + strsetitem(p88, 3, i20) + i91 = getarrayitem_gc_i(p88, 0, descr=<ArrayS 4>) + """) diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py --- a/pypy/module/struct/formatiterator.py +++ b/pypy/module/struct/formatiterator.py @@ -149,3 +149,13 @@ @specialize.argtype(1) def appendobj(self, value): self.result_w.append(self.space.wrap(value)) + + def get_pos(self): + return self.pos + + def get_buffer_as_string_maybe(self): + string, pos = self.buf.as_str_and_offset_maybe() + return string, pos+self.pos + + def skip(self, size): + self.read(size) # XXX, could avoid taking the slice diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py --- a/pypy/module/struct/test/test_struct.py +++ b/pypy/module/struct/test/test_struct.py @@ -462,3 +462,29 @@ assert self.struct.unpack_from("ii", b, 2) == (17, 42) b[:sz] = self.struct.pack("ii", 18, 43) assert self.struct.unpack_from("ii", b) == (18, 43) + + +class AppTestFastPath(object): + spaceconfig = dict(usemodules=['struct', '__pypy__']) + + def setup_class(cls): + from rpython.rlib.rstruct import standardfmttable + standardfmttable.ALLOW_SLOWPATH = False + # + cls.w_struct = cls.space.appexec([], """(): + import struct + return struct + """) + cls.w_bytebuffer = cls.space.appexec([], """(): + import __pypy__ + return __pypy__.bytebuffer + """) + + def teardown_class(cls): + from rpython.rlib.rstruct import standardfmttable + standardfmttable.ALLOW_SLOWPATH = True + + def test_unpack_from(self): + buf = self.struct.pack("iii", 0, 42, 43) + offset = self.struct.calcsize("i") + assert self.struct.unpack_from("ii", buf, offset) == (42, 43) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1810,3 +1810,35 @@ assert lib.EE1 == 0 assert lib.EE2 == 0 assert lib.EE3 == 1 + + def test_init_once(self): + def do_init(): + seen.append(1) + return 42 + ffi = FFI() + seen = [] + for i in range(3): + res = ffi.init_once(do_init, "tag1") + assert res == 42 + assert seen == [1] + for i in range(3): + res = ffi.init_once(do_init, "tag2") + assert res == 42 + assert seen == [1, 1] + + def test_init_once_multithread(self): + import thread, time + def do_init(): + seen.append('init!') + time.sleep(1) + seen.append('init done') + return 7 + ffi = FFI() + seen = [] + for i in range(6): + def f(): + res = ffi.init_once(do_init, "tag") + seen.append(res) + thread.start_new_thread(f, ()) + time.sleep(1.5) + assert seen == ['init!', 'init done'] + 6 * [7] diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py @@ -487,7 +487,7 @@ ffi = FFI(backend=self.Backend()) ffi.cdef("double __stdcall sin(double x);") # stdcall ignored m = ffi.dlopen(lib_m) - if (sys.platform == 'win32' and sys.maxint < 2**32 and + if (sys.platform == 'win32' and sys.maxsize < 2**32 and self.Backend is not CTypesBackend): assert "double(__stdcall *)(double)" in str(ffi.typeof(m.sin)) else: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py @@ -194,6 +194,11 @@ yp = ffi.new_handle([6, 4, 2]) assert ffi.from_handle(yp) == [6, 4, 2] +def test_handle_unique(): + ffi = _cffi1_backend.FFI() + assert ffi.new_handle(None) is not ffi.new_handle(None) + assert ffi.new_handle(None) != ffi.new_handle(None) + def test_ffi_cast(): ffi = _cffi1_backend.FFI() assert ffi.cast("int(*)(int)", 0) == ffi.NULL @@ -416,3 +421,37 @@ assert int(ffi.cast("_Bool", ffi.cast(type, 42))) == 1 assert int(ffi.cast("bool", ffi.cast(type, 42))) == 1 assert int(ffi.cast("_Bool", ffi.cast(type, 0))) == 0 + +def test_init_once(): + def do_init(): + seen.append(1) + return 42 + ffi = _cffi1_backend.FFI() + seen = [] + for i in range(3): + res = ffi.init_once(do_init, "tag1") + assert res == 42 + assert seen == [1] + for i in range(3): + res = ffi.init_once(do_init, "tag2") + assert res == 42 + assert seen == [1, 1] + +def test_init_once_multithread(): + import thread, time + def do_init(): + print 'init!' + seen.append('init!') + time.sleep(1) + seen.append('init done') + print 'init done' + return 7 + ffi = _cffi1_backend.FFI() + seen = [] + for i in range(6): + def f(): + res = ffi.init_once(do_init, "tag") + seen.append(res) + thread.start_new_thread(f, ()) + time.sleep(1.5) + assert seen == ['init!', 'init done'] + 6 * [7] diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -170,14 +170,23 @@ self.value = value def __int__(self): return self.value + class SubInt(int): pass + class SubLong(long): pass + dt10 = datetime.datetime(10, 10, 10, 10, 10, 10, 10) for xx in [10L, decimal.Decimal(10), decimal.Decimal('10.9'), Number(10), - Number(10L)]: - assert datetime.datetime(10, 10, 10, 10, 10, 10, 10) == \ - datetime.datetime(xx, xx, xx, xx, xx, xx, xx) + Number(10L), + SubInt(10), + SubLong(10), + Number(SubInt(10)), + Number(SubLong(10))]: + dtxx = datetime.datetime(xx, xx, xx, xx, xx, xx, xx) + assert dt10 == dtxx + assert type(dtxx.month) is int + assert type(dtxx.second) is int with py.test.raises(TypeError) as e: datetime.datetime(10, 10, '10') @@ -242,6 +251,70 @@ naive == aware assert str(e.value) == "can't compare offset-naive and offset-aware times" + def test_future_types_newint(self): + try: + from future.types.newint import newint + except ImportError: + py.test.skip('requires future') + + dt_from_ints = datetime.datetime(2015, 12, 31, 12, 34, 56) + dt_from_newints = datetime.datetime(newint(2015), newint(12), newint(31), newint(12), newint(34), newint(56)) + dt_from_mixed = datetime.datetime(2015, newint(12), 31, newint(12), 34, newint(56)) + assert dt_from_ints == dt_from_newints + assert dt_from_newints == dt_from_mixed + assert dt_from_mixed == dt_from_ints + + d_from_int = datetime.date.fromtimestamp(1431216000) + d_from_newint = datetime.date.fromtimestamp(newint(1431216000)) + assert d_from_int == d_from_newint + + dt_from_int = datetime.datetime.fromtimestamp(1431216000) + dt_from_newint = datetime.datetime.fromtimestamp(newint(1431216000)) + assert dt_from_int == dt_from_newint + + dtu_from_int = datetime.datetime.utcfromtimestamp(1431216000) + dtu_from_newint = datetime.datetime.utcfromtimestamp(newint(1431216000)) + assert dtu_from_int == dtu_from_newint + + td_from_int = datetime.timedelta(16565) + tds_from_int = datetime.timedelta(seconds=1431216000) + td_from_newint = datetime.timedelta(newint(16565)) + tds_from_newint = datetime.timedelta(seconds=newint(1431216000)) + assert td_from_int == tds_from_int + assert td_from_int == td_from_newint + assert td_from_int == tds_from_newint + assert tds_from_int == td_from_newint + assert tds_from_int == tds_from_newint + assert td_from_newint == tds_from_newint + + td_mul_int_int = td_from_int * 2 + td_mul_int_newint = td_from_int * newint(2) + td_mul_newint_int = td_from_newint * 2 + td_mul_newint_newint = td_from_newint * newint(2) + assert td_mul_int_int == td_mul_int_newint + assert td_mul_int_int == td_mul_newint_int + assert td_mul_int_int == td_mul_newint_newint + assert td_mul_int_newint == td_mul_newint_int + assert td_mul_int_newint == td_mul_newint_newint + assert td_mul_newint_int == td_mul_newint_newint + + td_div_int_int = td_from_int / 3600 + td_div_int_newint = td_from_int / newint(3600) + td_div_newint_int = td_from_newint / 3600 + td_div_newint_newint = td_from_newint / newint(3600) + assert td_div_int_int == td_div_int_newint + assert td_div_int_int == td_div_newint_int + assert td_div_int_int == td_div_newint_newint + assert td_div_int_newint == td_div_newint_int + assert td_div_int_newint == td_div_newint_newint + assert td_div_newint_int == td_div_newint_newint + + def test_return_types(self): + td = datetime.timedelta(5) + assert type(td.total_seconds()) is float + class sub(datetime.timedelta): pass + assert type(+sub()) is datetime.timedelta + class TestDatetimeHost(BaseTestDatetime): def setup_class(cls): diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -638,9 +638,18 @@ return array.getlength() def bh_getarrayitem_gc(self, a, index, descr): - a = support.cast_arg(lltype.Ptr(descr.A), a) + assert index >= 0 + if descr.A is descr.OUTERA: + a = support.cast_arg(lltype.Ptr(descr.A), a) + else: + # we use rffi.cast instead of support.cast_arg because the types + # might not be "compatible" enough from the lltype point of + # view. In particular, this happens when we use + # str_storage_getitem, in which an rpy_string is casted to + # rpy_string_as_Signed (or similar) + a = rffi.cast(lltype.Ptr(descr.OUTERA), a) + a = getattr(a, descr.OUTERA._arrayfld) array = a._obj - assert index >= 0 return support.cast_result(descr.A.OF, array.getitem(index)) bh_getarrayitem_gc_pure_i = bh_getarrayitem_gc diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -570,20 +570,20 @@ return self.read_int_at_mem(array, ofs, WORD, 1) @specialize.argtype(1) - def bh_getarrayitem_gc_i(self, gcref, itemindex, arraydescr): + def bh_gc_load_i(self, gcref, itemindex, arraydescr): ofs, size, sign = self.unpack_arraydescr_size(arraydescr) return self.read_int_at_mem(gcref, ofs + itemindex * size, size, sign) - - def bh_getarrayitem_gc_r(self, gcref, itemindex, arraydescr): - ofs = self.unpack_arraydescr(arraydescr) - return self.read_ref_at_mem(gcref, itemindex * WORD + ofs) - @specialize.argtype(1) - def bh_getarrayitem_gc_f(self, gcref, itemindex, arraydescr): - ofs = self.unpack_arraydescr(arraydescr) - fsize = rffi.sizeof(longlong.FLOATSTORAGE) - return self.read_float_at_mem(gcref, itemindex * fsize + ofs) + def bh_gc_load_r(self, gcref, itemindex, arraydescr): + ofs, size, sign = self.unpack_arraydescr_size(arraydescr) + return self.read_ref_at_mem(gcref, ofs + itemindex * size, size, + sign) + @specialize.argtype(1) + def bh_gc_load_f(self, gcref, itemindex, arraydescr): + ofs, size, sign = self.unpack_arraydescr_size(arraydescr) + return self.read_float_at_mem(gcref, ofs + itemindex * size, size, + sign) @specialize.argtype(1) def bh_setarrayitem_gc_i(self, gcref, itemindex, newvalue, arraydescr): diff --git a/rpython/jit/backend/x86/test/test_strstorage.py b/rpython/jit/backend/x86/test/test_strstorage.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/x86/test/test_strstorage.py @@ -0,0 +1,8 @@ +from rpython.jit.backend.x86.test.test_basic import Jit386Mixin +from rpython.jit.metainterp.test.test_strstorage import TestStrStorage as _TestStrStorage + + +class TestStrStorage(Jit386Mixin, _TestStrStorage): + # for the individual tests see + # ====> ../../../metainterp/test/test_strstorage.py + pass diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1003,15 +1003,14 @@ elif optype == lltype.Ptr(rbytearray.BYTEARRAY): bytearraydescr = self.cpu.arraydescrof(rbytearray.BYTEARRAY) v_index = op.args[2] - return SpaceOperation('getarrayitem_gc_i', + return SpaceOperation('gc_load_i', [op.args[0], v_index, bytearraydescr], op.result) - else: + elif op.result.concretetype is lltype.Void: + return + elif isinstance(op.args[0].concretetype.TO, lltype.GcArray): + # special-case 1: GcArray of Struct v_inst, v_index, c_field = op.args - if op.result.concretetype is lltype.Void: - return - # only GcArray of Struct supported - assert isinstance(v_inst.concretetype.TO, lltype.GcArray) STRUCT = v_inst.concretetype.TO.OF assert isinstance(STRUCT, lltype.Struct) descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO, @@ -1020,6 +1019,20 @@ kind = getkind(op.result.concretetype)[0] return SpaceOperation('getinteriorfield_gc_%s' % kind, args, op.result) + elif isinstance(op.args[0].concretetype.TO, lltype.GcStruct): + # special-case 2: GcStruct with Array field + v_inst, c_field, v_index = op.args + STRUCT = v_inst.concretetype.TO + ARRAY = getattr(STRUCT, c_field.value) + assert isinstance(ARRAY, lltype.Array) + arraydescr = self.cpu.arraydescrof(STRUCT) + kind = getkind(op.result.concretetype)[0] + assert kind in ('i', 'f') + return SpaceOperation('gc_load_%s' % kind, + [op.args[0], v_index, arraydescr], + op.result) + else: + assert False, 'not supported' def rewrite_op_setinteriorfield(self, op): assert len(op.args) == 4 @@ -1128,10 +1141,13 @@ def rewrite_op_force_cast(self, op): v_arg = op.args[0] v_result = op.result - assert not self._is_gc(v_arg) - if v_arg.concretetype == v_result.concretetype: return + elif self._is_gc(v_arg) and self._is_gc(v_result): + # cast from GC to GC is always fine + return + else: + assert not self._is_gc(v_arg) float_arg = v_arg.concretetype in [lltype.Float, lltype.SingleFloat] float_res = v_result.concretetype in [lltype.Float, lltype.SingleFloat] diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1112,15 +1112,15 @@ @arguments("cpu", "r", "i", "d", "d", returns="i") def bhimpl_getlistitem_gc_i(cpu, lst, index, itemsdescr, arraydescr): items = cpu.bh_getfield_gc_r(lst, itemsdescr) - return cpu.bh_getarrayitem_gc_i(items, index, arraydescr) + return cpu.bh_gc_load_i(items, index, arraydescr) @arguments("cpu", "r", "i", "d", "d", returns="r") def bhimpl_getlistitem_gc_r(cpu, lst, index, itemsdescr, arraydescr): items = cpu.bh_getfield_gc_r(lst, itemsdescr) - return cpu.bh_getarrayitem_gc_r(items, index, arraydescr) + return cpu.bh_gc_load_r(items, index, arraydescr) @arguments("cpu", "r", "i", "d", "d", returns="f") def bhimpl_getlistitem_gc_f(cpu, lst, index, itemsdescr, arraydescr): items = cpu.bh_getfield_gc_r(lst, itemsdescr) - return cpu.bh_getarrayitem_gc_f(items, index, arraydescr) + return cpu.bh_gc_load_f(items, index, arraydescr) @arguments("cpu", "r", "i", "i", "d", "d") def bhimpl_setlistitem_gc_i(cpu, lst, index, nval, itemsdescr, arraydescr): @@ -1246,18 +1246,28 @@ return cpu.bh_new_array_clear(length, arraydescr) @arguments("cpu", "r", "i", "d", returns="i") - def bhimpl_getarrayitem_gc_i(cpu, array, index, arraydescr): - return cpu.bh_getarrayitem_gc_i(array, index, arraydescr) + def bhimpl_gc_load_i(cpu, array, index, arraydescr): + return cpu.bh_gc_load_i(array, index, arraydescr) + @arguments("cpu", "r", "i", "d", returns="i") + def bhimpl_gc_load_r(cpu, array, index, arraydescr): + return cpu.bh_gc_load_r(array, index, arraydescr) + @arguments("cpu", "r", "i", "d", returns="i") + def bhimpl_gc_load_f(cpu, array, index, arraydescr): + return cpu.bh_gc_load_f(array, index, arraydescr) + + @arguments("cpu", "r", "i", "d", returns="i") + def bhimpl_gc_load_i(cpu, array, index, arraydescr): + return cpu.bh_gc_load_i(array, index, arraydescr) @arguments("cpu", "r", "i", "d", returns="r") - def bhimpl_getarrayitem_gc_r(cpu, array, index, arraydescr): - return cpu.bh_getarrayitem_gc_r(array, index, arraydescr) + def bhimpl_gc_load_r(cpu, array, index, arraydescr): + return cpu.bh_gc_load_r(array, index, arraydescr) @arguments("cpu", "r", "i", "d", returns="f") - def bhimpl_getarrayitem_gc_f(cpu, array, index, arraydescr): - return cpu.bh_getarrayitem_gc_f(array, index, arraydescr) + def bhimpl_gc_load_f(cpu, array, index, arraydescr): + return cpu.bh_gc_load_f(array, index, arraydescr) - bhimpl_getarrayitem_gc_i_pure = bhimpl_getarrayitem_gc_i - bhimpl_getarrayitem_gc_r_pure = bhimpl_getarrayitem_gc_r - bhimpl_getarrayitem_gc_f_pure = bhimpl_getarrayitem_gc_f + bhimpl_getarrayitem_gc_i_pure = bhimpl_gc_load_i + bhimpl_getarrayitem_gc_r_pure = bhimpl_gc_load_r + bhimpl_getarrayitem_gc_f_pure = bhimpl_gc_load_f @arguments("cpu", "i", "i", "d", returns="i") def bhimpl_getarrayitem_raw_i(cpu, array, index, arraydescr): @@ -1293,17 +1303,17 @@ def bhimpl_getarrayitem_vable_i(cpu, vable, index, fielddescr, arraydescr): fielddescr.get_vinfo().clear_vable_token(vable) array = cpu.bh_getfield_gc_r(vable, fielddescr) - return cpu.bh_getarrayitem_gc_i(array, index, arraydescr) + return cpu.bh_gc_load_i(array, index, arraydescr) @arguments("cpu", "r", "i", "d", "d", returns="r") def bhimpl_getarrayitem_vable_r(cpu, vable, index, fielddescr, arraydescr): fielddescr.get_vinfo().clear_vable_token(vable) array = cpu.bh_getfield_gc_r(vable, fielddescr) - return cpu.bh_getarrayitem_gc_r(array, index, arraydescr) + return cpu.bh_gc_load_r(array, index, arraydescr) @arguments("cpu", "r", "i", "d", "d", returns="f") def bhimpl_getarrayitem_vable_f(cpu, vable, index, fielddescr, arraydescr): fielddescr.get_vinfo().clear_vable_token(vable) array = cpu.bh_getfield_gc_r(vable, fielddescr) - return cpu.bh_getarrayitem_gc_f(array, index, arraydescr) + return cpu.bh_gc_load_f(array, index, arraydescr) @arguments("cpu", "r", "i", "i", "d", "d") def bhimpl_setarrayitem_vable_i(cpu, vable, index, newval, fdescr, adescr): diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -101,20 +101,20 @@ if condbox.getint(): do_call_n(cpu, metainterp, argboxes[1:], descr) -def do_getarrayitem_gc_i(cpu, _, arraybox, indexbox, arraydescr): +def do_gc_load_i(cpu, _, arraybox, indexbox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() - return cpu.bh_getarrayitem_gc_i(array, index, arraydescr) + return cpu.bh_gc_load_i(array, index, arraydescr) -def do_getarrayitem_gc_r(cpu, _, arraybox, indexbox, arraydescr): +def do_gc_load_r(cpu, _, arraybox, indexbox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() - return cpu.bh_getarrayitem_gc_r(array, index, arraydescr) + return cpu.bh_gc_load_r(array, index, arraydescr) -def do_getarrayitem_gc_f(cpu, _, arraybox, indexbox, arraydescr): +def do_gc_load_f(cpu, _, arraybox, indexbox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() - return cpu.bh_getarrayitem_gc_f(array, index, arraydescr) + return cpu.bh_gc_load_f(array, index, arraydescr) def do_getarrayitem_raw_i(cpu, _, arraybox, indexbox, arraydescr): array = arraybox.getint() diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -534,11 +534,17 @@ cf = self.field_cache(op.getdescr()) cf.do_setfield(self, op) - def optimize_GETARRAYITEM_GC_I(self, op): + def optimize_GC_LOAD_I(self, op): + # When using str_storage_getitem it might happen that op.getarg(0) is + # a virtual string, NOT an array. In that case, we cannot cache the + # getarrayitem as if it were an array, obviously. In theory we could + # improve by writing special code to interpter the buffer of the + # virtual string as if it were an array, but it looks complicate, + # fragile and not worth it. arrayinfo = self.ensure_ptr_info_arg0(op) indexb = self.getintbound(op.getarg(1)) cf = None - if indexb.is_constant(): + if indexb.is_constant() and not arrayinfo.is_vstring(): index = indexb.getint() arrayinfo.getlenbound(None).make_gt_const(index) # use the cache on (arraydescr, index), which is a constant @@ -555,13 +561,13 @@ self.make_nonnull(op.getarg(0)) self.emit_operation(op) # the remember the result of reading the array item - if cf is not None: + if cf is not None and not arrayinfo.is_vstring(): arrayinfo.setitem(op.getdescr(), indexb.getint(), self.get_box_replacement(op.getarg(0)), self.get_box_replacement(op), cf, self) - optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I - optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I + optimize_GC_LOAD_R = optimize_GC_LOAD_I + optimize_GC_LOAD_F = optimize_GC_LOAD_I def optimize_GETARRAYITEM_GC_PURE_I(self, op): arrayinfo = self.ensure_ptr_info_arg0(op) diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -24,6 +24,9 @@ def is_virtual(self): return False + def is_vstring(self): + return False + def is_precise(self): return False diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -464,9 +464,10 @@ intbound.make_le(IntUpperBound(descr.get_item_integer_max())) optimize_GETARRAYITEM_RAW_F = optimize_GETARRAYITEM_RAW_I - optimize_GETARRAYITEM_GC_I = optimize_GETARRAYITEM_RAW_I - optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_RAW_I - optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_RAW_I + + optimize_GC_LOAD_I = optimize_GETARRAYITEM_RAW_I + optimize_GC_LOAD_F = optimize_GETARRAYITEM_RAW_I + optimize_GC_LOAD_R = optimize_GETARRAYITEM_RAW_I def optimize_UNICODEGETITEM(self, op): self.emit_operation(op) diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -314,9 +314,16 @@ args, virtuals = target_virtual_state.make_inputargs_and_virtuals( args, self.optimizer) short_preamble = target_token.short_preamble - extra = self.inline_short_preamble(args + virtuals, args, - short_preamble, self.optimizer.patchguardop, - target_token, label_op) + try: + extra = self.inline_short_preamble(args + virtuals, args, + short_preamble, self.optimizer.patchguardop, + target_token, label_op) + except KeyError: + # SHOULD NOT OCCUR BUT DOES: WHY?? issue #2185 + self.optimizer.metainterp_sd.logger_ops.log_short_preamble([], + short_preamble, {}) + raise + self.send_extra_operation(jump_op.copy_and_change(rop.JUMP, args=args + extra, descr=target_token)) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -270,9 +270,11 @@ self.make_nonnull(op.getarg(0)) self.emit_operation(op) - def optimize_GETARRAYITEM_GC_I(self, op): + def optimize_GC_LOAD_I(self, op): + # When using str_storage_getitem we op.getarg(0) is a string, NOT an + # array, hence the check. In that case, it will be forced opinfo = self.getptrinfo(op.getarg(0)) - if opinfo and opinfo.is_virtual(): + if opinfo and opinfo.is_virtual() and not opinfo.is_vstring(): indexbox = self.get_constant_box(op.getarg(1)) if indexbox is not None: item = opinfo.getitem(op.getdescr(), indexbox.getint()) @@ -283,14 +285,15 @@ return self.make_nonnull(op.getarg(0)) self.emit_operation(op) - optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I - optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I + optimize_GC_LOAD_R = optimize_GC_LOAD_I + optimize_GC_LOAD_F = optimize_GC_LOAD_I + _optimize_GETARRAYITEM_GC_I = optimize_GC_LOAD_I # TODO # note: the following line does not mean that the two operations are # completely equivalent, because GETARRAYITEM_GC_PURE is_always_pure(). - optimize_GETARRAYITEM_GC_PURE_I = optimize_GETARRAYITEM_GC_I - optimize_GETARRAYITEM_GC_PURE_R = optimize_GETARRAYITEM_GC_I - optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_I + optimize_GETARRAYITEM_GC_PURE_I = _optimize_GETARRAYITEM_GC_I + optimize_GETARRAYITEM_GC_PURE_R = _optimize_GETARRAYITEM_GC_I + optimize_GETARRAYITEM_GC_PURE_F = _optimize_GETARRAYITEM_GC_I def optimize_SETARRAYITEM_GC(self, op): opinfo = self.getptrinfo(op.getarg(0)) diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -62,6 +62,9 @@ self.mode = mode self.length = length + def is_vstring(self): + return True + def getlenbound(self, mode): from rpython.jit.metainterp.optimizeopt import intutils diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -475,18 +475,18 @@ return resop @arguments("box", "box", "descr") - def opimpl_getarrayitem_gc_i(self, arraybox, indexbox, arraydescr): - return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_I, arraybox, + def opimpl_gc_load_i(self, arraybox, indexbox, arraydescr): + return self._do_getarrayitem_gc_any(rop.GC_LOAD_I, arraybox, indexbox, arraydescr) @arguments("box", "box", "descr") - def opimpl_getarrayitem_gc_r(self, arraybox, indexbox, arraydescr): - return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_R, arraybox, + def opimpl_gc_load_r(self, arraybox, indexbox, arraydescr): + return self._do_getarrayitem_gc_any(rop.GC_LOAD_R, arraybox, indexbox, arraydescr) @arguments("box", "box", "descr") - def opimpl_getarrayitem_gc_f(self, arraybox, indexbox, arraydescr): - return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_F, arraybox, + def opimpl_gc_load_f(self, arraybox, indexbox, arraydescr): + return self._do_getarrayitem_gc_any(rop.GC_LOAD_F, arraybox, indexbox, arraydescr) @arguments("box", "box", "descr") @@ -614,17 +614,17 @@ def opimpl_getlistitem_gc_i(self, listbox, indexbox, itemsdescr, arraydescr): arraybox = self.opimpl_getfield_gc_r(listbox, itemsdescr) - return self.opimpl_getarrayitem_gc_i(arraybox, indexbox, arraydescr) + return self.opimpl_gc_load_i(arraybox, indexbox, arraydescr) @arguments("box", "box", "descr", "descr") def opimpl_getlistitem_gc_r(self, listbox, indexbox, itemsdescr, arraydescr): arraybox = self.opimpl_getfield_gc_r(listbox, itemsdescr) - return self.opimpl_getarrayitem_gc_r(arraybox, indexbox, arraydescr) + return self.opimpl_gc_load_r(arraybox, indexbox, arraydescr) @arguments("box", "box", "descr", "descr") def opimpl_getlistitem_gc_f(self, listbox, indexbox, itemsdescr, arraydescr): arraybox = self.opimpl_getfield_gc_r(listbox, itemsdescr) - return self.opimpl_getarrayitem_gc_f(arraybox, indexbox, arraydescr) + return self.opimpl_gc_load_f(arraybox, indexbox, arraydescr) @arguments("box", "box", "box", "descr", "descr") def _opimpl_setlistitem_gc_any(self, listbox, indexbox, valuebox, @@ -956,11 +956,11 @@ if self._nonstandard_virtualizable(pc, box, fdescr): arraybox = self.opimpl_getfield_gc_r(box, fdescr) if adescr.is_array_of_pointers(): - return self.opimpl_getarrayitem_gc_r(arraybox, indexbox, adescr) + return self.opimpl_gc_load_r(arraybox, indexbox, adescr) elif adescr.is_array_of_floats(): - return self.opimpl_getarrayitem_gc_f(arraybox, indexbox, adescr) + return self.opimpl_gc_load_f(arraybox, indexbox, adescr) else: - return self.opimpl_getarrayitem_gc_i(arraybox, indexbox, adescr) + return self.opimpl_gc_load_i(arraybox, indexbox, adescr) self.metainterp.check_synchronized_virtualizable() index = self._get_arrayitem_vable_index(pc, fdescr, indexbox) return self.metainterp.virtualizable_boxes[index] diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -396,8 +396,8 @@ rop.GETFIELD_GC_PURE_R, rop.GETFIELD_GC_PURE_F) def is_getarrayitem(self): - return self.opnum in (rop.GETARRAYITEM_GC_I, rop.GETARRAYITEM_GC_F, - rop.GETARRAYITEM_GC_R, rop.GETARRAYITEM_GC_PURE_I, + return self.opnum in (rop.GC_LOAD_I, rop.GC_LOAD_F, + rop.GC_LOAD_R, rop.GETARRAYITEM_GC_PURE_I, rop.GETARRAYITEM_GC_PURE_F, rop.GETARRAYITEM_GC_PURE_R) @@ -1087,34 +1087,35 @@ 'INSTANCE_PTR_NE/2b/i', 'NURSERY_PTR_INCREMENT/2/r', # - # DEL 'ARRAYLEN_GC/1d/i', - # DEL 'STRLEN/1/i', + 'ARRAYLEN_GC/1d/i', + 'STRLEN/1/i', 'STRGETITEM/2/i', 'GETFIELD_GC_PURE/1d/rfi', - # DEL 'GETARRAYITEM_GC_PURE/2d/rfi', + 'GETARRAYITEM_GC_PURE/2d/rfi', #'GETFIELD_RAW_PURE/1d/rfi', these two operations not useful and #'GETARRAYITEM_RAW_PURE/2d/fi', dangerous when unrolling speculatively - # DEL 'UNICODELEN/1/i', - # DEL 'UNICODEGETITEM/2/i', + 'UNICODELEN/1/i', + 'UNICODEGETITEM/2/i', # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- + 'GC_LOAD/2d/irf', + # NEW 'GC_LOAD_INDEXED/2d/rfi', + '_RAW_LOAD_FIRST', - #'GETARRAYITEM_GC/2d/rfi', - #'VEC_GETARRAYITEM_GC/2d/fi', - #'GETARRAYITEM_RAW/2d/fi', - #'VEC_GETARRAYITEM_RAW/2d/fi', - 'RAW_LOAD/2d/rfi', - 'RAW_LOAD_INDEXED/2d/rfi', - 'VEC_RAW_LOAD/2d/rfi', + # DEL 'GETARRAYITEM_GC/2d/rfi', + 'VEC_GETARRAYITEM_GC/2d/fi', + 'GETARRAYITEM_RAW/2d/fi', + 'VEC_GETARRAYITEM_RAW/2d/fi', + 'RAW_LOAD/2d/fi', + # NEW 'RAW_LOAD_INDEXED/2d/rfi', + 'VEC_RAW_LOAD/2d/fi', '_RAW_LOAD_LAST', - 'GC_LOAD/2d/rfi', - 'GC_LOAD_INDEXED/2d/rfi', - # DEL 'GETINTERIORFIELD_GC/2d/rfi', - # DEL 'GETFIELD_GC/1d/rfi', - # DEL 'GETFIELD_RAW/1d/rfi', + 'GETINTERIORFIELD_GC/2d/rfi', + 'GETFIELD_GC/1d/rfi', + 'GETFIELD_RAW/1d/rfi', '_MALLOC_FIRST', 'NEW/0d/r', #-> GcStruct, gcptrs inside are zeroed (not the rest) 'NEW_WITH_VTABLE/0d/r',#-> GcStruct with vtable, gcptrs inside are zeroed @@ -1129,30 +1130,31 @@ # must be forced, however we need to execute it anyway '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- + # NEW 'GC_STORE/3d/n', + # NEW 'GC_STORE_INDEXED/3d/n', + 'INCREMENT_DEBUG_COUNTER/1/n', '_RAW_STORE_FIRST', - # DEL 'SETARRAYITEM_GC/3d/n', - # DEL 'VEC_SETARRAYITEM_GC/3d/n', - # DEL 'SETARRAYITEM_RAW/3d/n', - # DEL 'VEC_SETARRAYITEM_RAW/3d/n', + 'SETARRAYITEM_GC/3d/n', + 'VEC_SETARRAYITEM_GC/3d/n', + 'SETARRAYITEM_RAW/3d/n', + 'VEC_SETARRAYITEM_RAW/3d/n', 'RAW_STORE/3d/n', - 'RAW_STORE_INDEXED/3d/n', + #'RAW_STORE_INDEXED/3d/n', 'VEC_RAW_STORE/3d/n', '_RAW_STORE_LAST', - 'GC_STORE/3d/n', - 'GC_STORE_INDEXED/3d/n', - # DEL 'SETINTERIORFIELD_GC/3d/n', - # DEL 'SETINTERIORFIELD_RAW/3d/n', # right now, only used by tests - # DEL 'SETFIELD_GC/2d/n', + 'SETINTERIORFIELD_GC/3d/n', + 'SETINTERIORFIELD_RAW/3d/n', # right now, only used by tests + 'SETFIELD_GC/2d/n', 'ZERO_PTR_FIELD/2/n', # only emitted by the rewrite, clears a pointer field # at a given constant offset, no descr 'ZERO_ARRAY/3d/n', # only emitted by the rewrite, clears (part of) an array # [arraygcptr, firstindex, length], descr=ArrayDescr - # DEL 'SETFIELD_RAW/2d/n', - # DEL 'STRSETITEM/3/n', - # DEL 'UNICODESETITEM/3/n', + 'SETFIELD_RAW/2d/n', + 'STRSETITEM/3/n', + 'UNICODESETITEM/3/n', 'COND_CALL_GC_WB/1d/n', # [objptr] (for the write barrier) 'COND_CALL_GC_WB_ARRAY/2d/n', # [objptr, arrayindex] (write barr. for array) '_JIT_DEBUG_FIRST', @@ -1375,17 +1377,17 @@ rop.RAW_LOAD_I: rop.VEC_RAW_LOAD_I, rop.RAW_LOAD_F: rop.VEC_RAW_LOAD_F, rop.GC_LOAD_I: rop.VEC_RAW_LOAD_I, - rop.GC_LOAD_F: rop.VEC_RAW_LOAD_F, + #rop.GC_LOAD_F: rop.VEC_RAW_LOAD_F, #rop.GETARRAYITEM_RAW_I: rop.VEC_GETARRAYITEM_RAW_I, #rop.GETARRAYITEM_RAW_F: rop.VEC_GETARRAYITEM_RAW_F, - #rop.GETARRAYITEM_GC_I: rop.VEC_GETARRAYITEM_GC_I, - #rop.GETARRAYITEM_GC_F: rop.VEC_GETARRAYITEM_GC_F, + #rop.GC_LOAD_I: rop.VEC_GC_LOAD_I, + #rop.GC_LOAD_F: rop.VEC_GC_LOAD_F, # note that there is no _PURE operation for vector operations. # reason: currently we do not care if it is pure or not! - #rop.GETARRAYITEM_GC_PURE_I: rop.VEC_GETARRAYITEM_GC_I, - #rop.GETARRAYITEM_GC_PURE_F: rop.VEC_GETARRAYITEM_GC_F, + #rop.GETARRAYITEM_GC_PURE_I: rop.VEC_GC_LOAD_I, + #rop.GETARRAYITEM_GC_PURE_F: rop.VEC_GC_LOAD_F, rop.RAW_STORE: rop.VEC_RAW_STORE, - rop.GC_STORE: rop.VEC_RAW_STORE, # TODO + #rop.GC_STORE: rop.VEC_RAW_STORE, # TODO #rop.SETARRAYITEM_RAW: rop.VEC_SETARRAYITEM_RAW, #rop.SETARRAYITEM_GC: rop.VEC_SETARRAYITEM_GC, @@ -1550,10 +1552,10 @@ @staticmethod def getarrayitem_for_descr(descr): if descr.is_array_of_pointers(): - return rop.GETARRAYITEM_GC_R + return rop.GC_LOAD_R elif descr.is_array_of_floats(): - return rop.GETARRAYITEM_GC_F - return rop.GETARRAYITEM_GC_I + return rop.GC_LOAD_F + return rop.GC_LOAD_I @staticmethod def same_as_for_type(tp): diff --git a/rpython/jit/metainterp/test/test_strstorage.py b/rpython/jit/metainterp/test/test_strstorage.py new file mode 100644 --- /dev/null +++ b/rpython/jit/metainterp/test/test_strstorage.py @@ -0,0 +1,53 @@ +import py +import sys +import struct +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.strstorage import str_storage_getitem +from rpython.rlib.test.test_strstorage import BaseStrStorageTest +from rpython.jit.codewriter import longlong +from rpython.jit.metainterp.history import getkind +from rpython.jit.metainterp.test.support import LLJitMixin + +class TestStrStorage(BaseStrStorageTest, LLJitMixin): + + # for the individual tests see + # ====> ../../../rlib/test/test_strstorage.py + + def str_storage_getitem(self, TYPE, buf, offset): + def f(): + return str_storage_getitem(TYPE, buf, offset) + res = self.interp_operations(f, [], supports_singlefloats=True) + # + kind = getkind(TYPE)[0] # 'i' or 'f' + self.check_operations_history({'getarrayitem_gc_%s' % kind: 1, + 'finish': 1}) + # + if TYPE == lltype.SingleFloat: + # interp_operations returns the int version of r_singlefloat, but + # our tests expects to receive an r_singlefloat: let's convert it + # back! + return longlong.int2singlefloat(res) + return res + + def str_storage_supported(self, TYPE): + py.test.skip('this is not a JIT test') + + def test_force_virtual_str_storage(self): + byteorder = sys.byteorder + size = rffi.sizeof(lltype.Signed) + def f(val): + if byteorder == 'little': + x = chr(val) + '\x00'*(size-1) + else: + x = '\x00'*(size-1) + chr(val) + return str_storage_getitem(lltype.Signed, x, 0) + res = self.interp_operations(f, [42], supports_singlefloats=True) + assert res == 42 + self.check_operations_history({ + 'newstr': 1, # str forcing + 'strsetitem': 1, # str forcing + 'call_pure_r': 1, # str forcing (copystrcontent) + 'guard_no_exception': 1, # str forcing + 'getarrayitem_gc_i': 1, # str_storage_getitem + 'finish': 1 + }) diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py --- a/rpython/rlib/buffer.py +++ b/rpython/rlib/buffer.py @@ -22,6 +22,14 @@ # May be overridden. return self.getslice(0, self.getlength(), 1, self.getlength()) + def as_str_and_offset_maybe(self): + """ + If the buffer is backed by a string, return a pair (string, offset), where + offset is the offset inside the string where the buffer start. + Else, return (None, 0). + """ + return None, 0 + def getitem(self, index): "Returns the index'th character in the buffer." raise NotImplementedError # Must be overriden. No bounds checks. @@ -66,6 +74,9 @@ def as_str(self): return self.value + def as_str_and_offset_maybe(self): + return self.value, 0 + def getitem(self, index): return self.value[index] @@ -99,6 +110,12 @@ else: return 0 + def as_str_and_offset_maybe(self): + string, offset = self.buffer.as_str_and_offset_maybe() + if string is not None: + return string, offset+self.offset + return None, 0 + def getitem(self, index): return self.buffer.getitem(self.offset + index) diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py --- a/rpython/rlib/rstruct/nativefmttable.py +++ b/rpython/rlib/rstruct/nativefmttable.py @@ -8,15 +8,15 @@ from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_singlefloat, widen from rpython.rlib.rstruct import standardfmttable as std +from rpython.rlib.rstruct.standardfmttable import native_is_bigendian from rpython.rlib.rstruct.error import StructError from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.strstorage import str_storage_getitem from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.tool import rffi_platform from rpython.translator.tool.cbuild import ExternalCompilationInfo -native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1) - native_fmttable = { 'x': std.standard_fmttable['x'], 'c': std.standard_fmttable['c'], @@ -27,9 +27,6 @@ # ____________________________________________________________ _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit