Author: Matti Picus <matti.pi...@gmail.com> Branch: py3.5 Changeset: r93152:ce6402cbdf3c Date: 2017-11-23 22:08 +0200 http://bitbucket.org/pypy/pypy/changeset/ce6402cbdf3c/
Log: merge default into py3.5 diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py --- a/pypy/module/_io/interp_stringio.py +++ b/pypy/module/_io/interp_stringio.py @@ -184,9 +184,7 @@ start, end ) - if endpos >= 0: - endpos += start - else: + if endpos < 0: endpos = end assert endpos >= 0 self.pos = endpos diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -216,44 +216,41 @@ def _find_line_ending(self, line, start, end): size = end - start - if self.readtranslate: - - # Newlines are already translated, only search for \n - pos = line.find(u'\n', start, end) - if pos >= 0: - return pos - start + 1, 0 - else: - return -1, size - elif self.readuniversal: + if self.readuniversal: # Universal newline search. Find any of \r, \r\n, \n # The decoder ensures that \r\n are not split in two pieces - i = 0 + i = start while True: - # Fast path for non-control chars. The loop always ends - # since the Py_UNICODE storage is NUL-terminated. - while i < size and line[start + i] > '\r': + # Fast path for non-control chars. + while i < end and line[i] > '\r': i += 1 - if i >= size: + if i >= end: return -1, size - ch = line[start + i] + ch = line[i] i += 1 if ch == '\n': return i, 0 if ch == '\r': - if line[start + i] == '\n': + if line[i] == '\n': return i + 1, 0 else: return i, 0 + if self.readtranslate: + # Newlines are already translated, only search for \n + newline = u'\n' else: # Non-universal mode. - pos = line.find(self.readnl, start, end) - if pos >= 0: - return pos - start + len(self.readnl), 0 - else: - pos = line.find(self.readnl[0], start, end) - if pos >= 0: - return -1, pos - start - return -1, size + newline = self.readnl + end_scan = end - len(newline) + 1 + for i in range(start, end_scan): + ch = line[i] + if ch == newline[0]: + for j in range(1, len(newline)): + if line[i + j] != newline[j]: + break + else: + return i + len(newline), 0 + return -1, end_scan W_TextIOBase.typedef = TypeDef( @@ -549,8 +546,13 @@ # _____________________________________________________________ # read methods - def _set_decoded_chars(self, chars): - self.decoded_chars = chars + def _unset_decoded(self): + self.decoded_chars = None + self.decoded_chars_used = 0 + + def _set_decoded(self, space, w_decoded): + check_decoded(space, w_decoded) + self.decoded_chars = space.unicode_w(w_decoded) self.decoded_chars_used = 0 def _get_decoded_chars(self, size): @@ -574,6 +576,10 @@ self.decoded_chars_used += size return chars + def _has_data(self): + return (self.decoded_chars is not None and + self.decoded_chars_used < len(self.decoded_chars)) + def _read_chunk(self, space): """Read and decode the next chunk of data from the BufferedReader. The return value is True unless EOF was reached. The decoded string @@ -616,8 +622,7 @@ eof = input_buf.getlength() == 0 w_decoded = space.call_method(self.w_decoder, "decode", w_input, space.newbool(eof)) - check_decoded(space, w_decoded) - self._set_decoded_chars(space.unicode_w(w_decoded)) + self._set_decoded(space, w_decoded) if space.len_w(w_decoded) > 0: eof = False @@ -629,6 +634,19 @@ return not eof + def _ensure_data(self, space): + while not self._has_data(): + try: + if not self._read_chunk(space): + self._unset_decoded() + self.snapshot = None + return False + except OperationError as e: + if trap_eintr(space, e): + continue + raise + return True + def next_w(self, space): self._check_attached(space) self.telling = False @@ -662,23 +680,13 @@ builder = UnicodeBuilder(size) # Keep reading chunks until we have n characters to return - while True: + while remaining > 0: + if not self._ensure_data(space): + break data = self._get_decoded_chars(remaining) builder.append(data) remaining -= len(data) - if remaining <= 0: # Done - break - - try: - if not self._read_chunk(space): - # EOF - break - except OperationError as e: - if trap_eintr(space, e): - continue - raise - return space.newunicode(builder.build()) def readline_w(self, space, w_limit=None): @@ -687,28 +695,16 @@ self._writeflush(space) limit = convert_size(space, w_limit) - chunked = 0 line = None remaining = None - chunks = [] + builder = UnicodeBuilder() while True: # First, get some data if necessary - has_data = True - while not self.decoded_chars: - try: - if not self._read_chunk(space): - has_data = False - break - except OperationError as e: - if trap_eintr(space, e): - continue - raise + has_data = self._ensure_data(space) if not has_data: # end of file - self._set_decoded_chars(None) - self.snapshot = None start = endpos = offset_to_buffer = 0 break @@ -725,8 +721,8 @@ line_len = len(line) endpos, consumed = self._find_line_ending(line, start, line_len) + chunked = builder.getlength() if endpos >= 0: - endpos += start if limit >= 0 and endpos >= start + limit - chunked: endpos = start + limit - chunked assert endpos >= 0 @@ -744,15 +740,15 @@ # No line ending seen yet - put aside current data if endpos > start: s = line[start:endpos] - chunks.append(s) - chunked += len(s) + builder.append(s) + # There may be some remaining bytes we'll have to prepend to the # next chunk of data if endpos < line_len: remaining = line[endpos:] line = None # We have consumed the buffer - self._set_decoded_chars(None) + self._unset_decoded() if line: # Our line ends in the current buffer @@ -761,18 +757,12 @@ self.decoded_chars_used = decoded_chars_used if start > 0 or endpos < len(line): line = line[start:endpos] - if remaining: - chunks.append(remaining) - remaining = None - if chunks: - if line: - chunks.append(line) - line = u''.join(chunks) + builder.append(line) + elif remaining: + builder.append(remaining) - if line: - return space.newunicode(line) - else: - return space.newunicode(u'') + result = builder.build() + return space.newunicode(result) # _____________________________________________________________ # write methods @@ -913,7 +903,7 @@ self._unsupportedoperation( space, "can't do nonzero end-relative seeks") space.call_method(self, "flush") - self._set_decoded_chars(None) + self._unset_decoded() self.snapshot = None if self.w_decoder: space.call_method(self.w_decoder, "reset") @@ -943,7 +933,7 @@ # Seek back to the safe start point space.call_method(self.w_buffer, "seek", space.newint(cookie.start_pos)) - self._set_decoded_chars(None) + self._unset_decoded() self.snapshot = None # Restore the decoder to its state from the safe start point. @@ -964,8 +954,7 @@ w_decoded = space.call_method(self.w_decoder, "decode", w_chunk, space.newbool(bool(cookie.need_eof))) - check_decoded(space, w_decoded) - self._set_decoded_chars(space.unicode_w(w_decoded)) + self._set_decoded(space, w_decoded) # Skip chars_to_skip of the decoded characters if len(self.decoded_chars) < cookie.chars_to_skip: @@ -1034,7 +1023,7 @@ w_decoded = space.call_method(self.w_decoder, "decode", space.newbytes(input[i])) check_decoded(space, w_decoded) - chars_decoded += len(space.unicode_w(w_decoded)) + chars_decoded += space.len_w(w_decoded) cookie.bytes_to_feed += 1 diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ cffi>=1.4.0 -vmprof>=0.4.10 # required to parse log files in rvmprof tests + +# parse log files in rvmprof tests +vmprof>=0.4.10; 'x86' in platform.machine #skip arm, s390x # hypothesis is used for test generation on untranslated tests hypothesis diff --git a/rpython/rlib/rstacklet.py b/rpython/rlib/rstacklet.py --- a/rpython/rlib/rstacklet.py +++ b/rpython/rlib/rstacklet.py @@ -3,7 +3,6 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import fetch_translated_config from rpython.rtyper.lltypesystem import lltype, llmemory -from rpython.rlib import rvmprof from rpython.rlib.rvmprof import cintf DEBUG = False @@ -41,13 +40,11 @@ def switch(self, stacklet): if DEBUG: debug.remove(stacklet) - rvmprof.stop_sampling() x = cintf.save_rvmprof_stack() try: h = self._gcrootfinder.switch(stacklet) finally: cintf.restore_rvmprof_stack(x) - rvmprof.start_sampling() if DEBUG: debug.add(h) return h diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py --- a/rpython/rlib/rvmprof/__init__.py +++ b/rpython/rlib/rvmprof/__init__.py @@ -56,8 +56,10 @@ return None def stop_sampling(): - fd = _get_vmprof().cintf.vmprof_stop_sampling() + from rpython.rlib.rvmprof.cintf import vmprof_stop_sampling + fd = vmprof_stop_sampling() return rffi.cast(lltype.Signed, fd) def start_sampling(): - _get_vmprof().cintf.vmprof_start_sampling() + from rpython.rlib.rvmprof.cintf import vmprof_start_sampling + vmprof_start_sampling() diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -9,6 +9,7 @@ from rpython.rtyper.tool import rffi_platform as platform from rpython.rlib import rthread, jit from rpython.rlib.objectmodel import we_are_translated +from rpython.config.translationoption import get_translation_config class VMProfPlatformUnsupported(Exception): pass @@ -40,7 +41,7 @@ compile_extra += ['-DVMPROF_UNIX'] compile_extra += ['-DVMPROF_LINUX'] elif sys.platform == 'win32': - compile_extra = ['-DRPYTHON_VMPROF', '-DVMPROF_WINDOWS'] + compile_extra += ['-DVMPROF_WINDOWS'] separate_module_files = [SHARED.join('vmprof_win.c')] _libs = [] else: @@ -120,16 +121,32 @@ vmprof_get_profile_path = rffi.llexternal("vmprof_get_profile_path", [rffi.CCHARP, lltype.Signed], lltype.Signed, compilation_info=eci, _nowrapper=True) - vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [], - rffi.INT, compilation_info=eci, - _nowrapper=True) - vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [], - lltype.Void, compilation_info=eci, - _nowrapper=True) return CInterface(locals()) +# this is always present, but compiles to no-op if RPYTHON_VMPROF is not +# defined (i.e. if we don't actually use vmprof in the generated C) +auto_eci = ExternalCompilationInfo(post_include_bits=[""" +#ifndef RPYTHON_VMPROF +# define vmprof_stop_sampling() (-1) +# define vmprof_start_sampling() ((void)0) +#endif +"""]) + +if get_translation_config() is None: + # tests need the full eci here + _eci = global_eci +else: + _eci = auto_eci + +vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [], + rffi.INT, compilation_info=_eci, + _nowrapper=True) +vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [], + lltype.Void, compilation_info=_eci, + _nowrapper=True) + class CInterface(object): def __init__(self, namespace): @@ -218,6 +235,7 @@ # stacklet support def save_rvmprof_stack(): + vmprof_stop_sampling() return vmprof_tl_stack.get_or_make_raw() def empty_rvmprof_stack(): @@ -225,6 +243,7 @@ def restore_rvmprof_stack(x): vmprof_tl_stack.setraw(x) + vmprof_start_sampling() # # traceback support diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -12,6 +12,7 @@ #endif +#include "vmprof_common.h" #include "shared/vmprof_get_custom_offset.h" #ifdef VMPROF_UNIX @@ -30,7 +31,7 @@ } #endif -long vmprof_get_profile_path(const char * buffer, long size) +long vmprof_get_profile_path(char * buffer, long size) { return vmp_fd_to_path(vmp_profile_fileno(), buffer, size); } diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -36,8 +36,8 @@ RPY_EXTERN int vmprof_stack_append(void*, long); RPY_EXTERN long vmprof_stack_pop(void*); RPY_EXTERN void vmprof_stack_free(void*); -RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t); -RPY_EXTERN long vmprof_get_profile_path(const char *, long); +RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, void**, intptr_t); +RPY_EXTERN long vmprof_get_profile_path(char *, long); RPY_EXTERN int vmprof_stop_sampling(void); RPY_EXTERN void vmprof_start_sampling(void); diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -262,7 +262,7 @@ } int depth = 0; - PY_STACK_FRAME_T * top_most_frame = frame; + //PY_STACK_FRAME_T * top_most_frame = frame; while ((depth + _per_loop()) <= max_depth) { unw_get_proc_info(&cursor, &pip); @@ -400,7 +400,7 @@ if (fd == NULL) { return 0; } - char * saveptr; + char * saveptr = NULL; char * line = NULL; char * he = NULL; char * name; diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.c b/rpython/rlib/rvmprof/src/shared/vmprof_common.c --- a/rpython/rlib/rvmprof/src/shared/vmprof_common.c +++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.c @@ -4,6 +4,9 @@ #include <errno.h> #ifdef RPYTHON_VMPROF + +int get_stack_trace(PY_THREAD_STATE_T * current, void** result, int max_depth, intptr_t pc); + #ifdef RPYTHON_LL2CTYPES /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */ @@ -193,7 +196,7 @@ #endif intptr_t vmprof_get_traceback(void *stack, void *ucontext, - intptr_t *result_p, intptr_t result_length) + void **result_p, intptr_t result_length) { int n; int enabled; diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h @@ -96,7 +96,7 @@ #endif RPY_EXTERN intptr_t vmprof_get_traceback(void *stack, void *ucontext, - intptr_t *result_p, intptr_t result_length); + void **result_p, intptr_t result_length); #endif int vmprof_get_signal_type(void); diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py --- a/rpython/rlib/rvmprof/test/test_rvmprof.py +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -144,7 +144,8 @@ @pytest.fixture def init(self, tmpdir): - eci = ExternalCompilationInfo(compile_extra=['-g','-O0'], + eci = ExternalCompilationInfo(compile_extra=['-g','-O0', '-Werror'], + post_include_bits = ['int native_func(int);'], separate_module_sources=[""" RPY_EXTERN int native_func(int d) { int j = 0; _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit