Author: Matti Picus <[email protected]>
Branch: py3.6
Changeset: r97429:6cbd0f9d8052
Date: 2019-09-11 08:58 +0300
http://bitbucket.org/pypy/pypy/changeset/6cbd0f9d8052/
Log: Merge with default
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -34,6 +34,13 @@
#define NCURSES_OPAQUE 0
#endif
+
+/* ncurses 6 change behaviour and makes all pointers opaque,
+ lets define backward compatibility. It doesn't harm
+ previous versions */
+
+#define NCURSES_INTERNALS 1
+#define NCURSES_REENTRANT 0
#include <ncurses.h>
#include <panel.h>
#include <term.h>
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -36,8 +36,11 @@
for _ver in ['8.6', '8.5', '']:
incdirs = []
linklibs = ['tcl' + _ver, 'tk' + _ver]
- if os.path.isfile(''.join(['/usr/lib/lib', linklibs[1], '.so'])):
- found = True
+ for lib in ['/usr/lib/lib', '/usr/lib64/lib']:
+ if os.path.isfile(''.join([lib, linklibs[1], '.so'])):
+ found = True
+ break
+ if found:
break
if not found:
sys.stderr.write("*** TCL libraries not found! Falling back...\n")
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -70,3 +70,11 @@
.. branch: cryptograhpt-2.7
Update vendored cryptography used for _ssl to 2.7
+
+.. branch: compile_ncurses_tcl_tk_suse_latest
+
+Check for headers and runtime libraries in more locations to support other
linuxes
+
+.. branch: openssl-for-macos
+
+Update _ssl on macos to statically link to openssl-1.1.1c
\ No newline at end of file
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -702,7 +702,8 @@
if isinstance(ctx, rsre_utf8.Utf8MatchContext):
index_storage = ctx.w_unicode_obj._get_index_storage()
return rutf8.codepoint_index_at_byte_position(
- ctx.w_unicode_obj._utf8, index_storage, bytepos)
+ ctx.w_unicode_obj._utf8, index_storage, bytepos,
+ ctx.w_unicode_obj._len())
else:
return bytepos
diff --git a/pypy/objspace/std/unicodeobject.py
b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1047,6 +1047,15 @@
return end - start
return rutf8.codepoints_in_utf8(self._utf8, start, end)
+ def _byte_to_index(self, bytepos):
+ """ this returns index such that self._index_to_byte(index) == bytepos
+ NB: this is slow! roughly logarithmic with a big constant
+ """
+ if self.is_ascii():
+ return bytepos
+ return rutf8.codepoint_index_at_byte_position(
+ self._utf8, self._get_index_storage(), bytepos, self._len())
+
@always_inline
def _unwrap_and_search(self, space, w_sub, w_start, w_end, forward=True):
w_sub = self.convert_arg_to_w_unicode(space, w_sub)
@@ -1068,16 +1077,14 @@
res_index = self._utf8.find(w_sub._utf8, start_index, end_index)
if res_index < 0:
return None
- skip = self._codepoints_in_utf8(start_index, res_index)
- res = start + skip
+ res = self._byte_to_index(res_index)
assert res >= 0
return space.newint(res)
else:
res_index = self._utf8.rfind(w_sub._utf8, start_index, end_index)
if res_index < 0:
return None
- skip = self._codepoints_in_utf8(res_index, end_index)
- res = end - skip
+ res = self._byte_to_index(res_index)
assert res >= 0
return space.newint(res)
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -29,9 +29,9 @@
'lzma': ('https://tukaani.org/xz/xz-5.2.3.tar.gz',
'71928b357d0a09a12a4b4c5fafca8c31c19b0e7d3b8ebb19622e96f26dbf28cb',
[]),
- '_ssl':
('http://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.6.2.tar.gz',
- 'b029d2492b72a9ba5b5fcd9f3d602c9fd0baa087912f2aaecc28f52f567ec478',
- ['--without-openssldir']),
+ '_ssl': ('https://www.openssl.org/source/openssl-1.1.1c.tar.gz',
+ 'f6fb3079ad15076154eda9413fed42877d668e7069d9b87396d0804fdb3f4c90',
+ ['no-shared']),
'_gdbm': ('http://ftp.gnu.org/gnu/gdbm/gdbm-1.13.tar.gz',
'9d252cbd7d793f7b12bcceaddda98d257c14f4d1890d851c386c37207000a253',
['--without-readline']),
@@ -113,12 +113,9 @@
# configure & build it
status, stdout, stderr = run_subprocess(
- './configure',
+ './config',
[
'--prefix=/usr',
- '--disable-shared',
- '--enable-silent-rules',
- '--disable-dependency-tracking',
] + args,
cwd=sources,
)
@@ -132,16 +129,25 @@
'make',
[
'-s', '-j' + str(multiprocessing.cpu_count()),
+ ],
+ cwd=sources,
+ )
+ if status != 0:
+ return status, stdout, stderr
+
+ print('installing to', destdir, file=sys.stderr)
+ status, stdout, stderr = run_subprocess(
+ 'make',
+ [
'install', 'DESTDIR={}/'.format(destdir),
],
cwd=sources,
)
-
return status, stdout, stderr
def create_cffi_import_libraries(pypy_c, options, basedir, only=None,
- embed_dependencies=False):
+ embed_dependencies=False, rebuild=False):
from rpython.tool.runsubprocess import run_subprocess
shutil.rmtree(str(join(basedir,'lib_pypy','__pycache__')),
@@ -160,12 +166,13 @@
continue
if module is None or getattr(options, 'no_' + key, False):
continue
- # the key is the module name, has it already been built?
- status, stdout, stderr = run_subprocess(str(pypy_c), ['-c', 'import
%s' % key])
- if status == 0:
- print('*', ' %s already built' % key, file=sys.stderr)
- continue
-
+ if not rebuild:
+ # the key is the module name, has it already been built?
+ status, stdout, stderr = run_subprocess(str(pypy_c), ['-c',
'import %s' % key])
+ if status == 0:
+ print('*', ' %s already built' % key, file=sys.stderr)
+ continue
+
if module.endswith('.py'):
args = [module]
cwd = str(join(basedir,'lib_pypy'))
@@ -182,18 +189,7 @@
shutil.rmtree(destdir, ignore_errors=True)
os.makedirs(destdir)
- if key == '_ssl' and sys.platform == 'darwin':
- # this patch is loosely inspired by an Apple and adds
- # a fallback to the OS X roots when none are available
- patches = [
- os.path.join(curdir,
- '../../lib_pypy/_cffi_ssl/osx-roots.diff'),
- ]
- else:
- patches = []
-
- status, stdout, stderr = _build_dependency(key, destdir,
- patches=patches)
+ status, stdout, stderr = _build_dependency(key, destdir)
if status != 0:
failures.append((key, module))
@@ -207,10 +203,6 @@
'-I{}/usr/include {}'.format(destdir, env.get('CPPFLAGS', ''))
env['LDFLAGS'] = \
'-L{}/usr/lib {}'.format(destdir, env.get('LDFLAGS', ''))
-
- if key == '_ssl' and sys.platform == 'darwin':
- # needed for our roots patch
- env['LDFLAGS'] += ' -framework CoreFoundation -framework
Security'
elif sys.platform == 'win32':
env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '')
env['LIB'] = r'..\externals\lib;' + env.get('LIB', '')
@@ -247,6 +239,8 @@
parser.add_argument('--exefile', dest='exefile', default=sys.executable,
help='instead of executing sys.executable' \
' you can specify an alternative pypy vm here')
+ parser.add_argument('--rebuild', dest='rebuild', action='store_true',
+ help='Rebuild the module even if it already appears to have been
built.')
parser.add_argument('--only', dest='only', default=None,
help='Only build the modules delimited by a colon.
E.g. _ssl,sqlite')
parser.add_argument('--embed-dependencies', dest='embed_dependencies',
action='store_true',
@@ -268,7 +262,8 @@
else:
only = set(args.only.split(','))
failures = create_cffi_import_libraries(exename, options, basedir,
only=only,
-
embed_dependencies=args.embed_dependencies)
+
embed_dependencies=args.embed_dependencies,
+ rebuild=args.rebuild)
if len(failures) > 0:
print('*** failed to build the CFFI modules %r' % (
[f[1] for f in failures],), file=sys.stderr)
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -87,6 +87,7 @@
failures = create_cffi_import_libraries(
str(pypy_c), options, str(basedir),
embed_dependencies=options.embed_dependencies,
+ rebuild=True,
)
for key, module in failures:
diff --git a/rpython/jit/metainterp/test/test_ajit.py
b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -134,7 +134,7 @@
u = U(x, len(x))
st = u._get_index_storage()
return rutf8.codepoint_index_at_byte_position(
- u.u, st, 1)
+ u.u, st, 1, len(x))
self.interp_operations(m, [123232])
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -486,6 +486,7 @@
length += 1
return length
+
@jit.elidable
def surrogate_in_utf8(value):
"""Check if the UTF-8 byte string 'value' contains a surrogate.
@@ -583,7 +584,7 @@
return codepoint_at_pos(utf8, bytepos)
@jit.elidable
-def codepoint_index_at_byte_position(utf8, storage, bytepos):
+def codepoint_index_at_byte_position(utf8, storage, bytepos, num_codepoints):
""" Return the character index for which
codepoint_position_at_index(index) == bytepos.
This is a relatively slow operation in that it runs in a time
@@ -592,17 +593,38 @@
"""
if bytepos < 0:
return bytepos
+ # binary search on elements of storage
index_min = 0
index_max = len(storage) - 1
while index_min < index_max:
+ # this addition can't overflow because storage has a length that is
+ # 1/64 of the length of a string
index_middle = (index_min + index_max + 1) // 2
base_bytepos = storage[index_middle].baseindex
if bytepos < base_bytepos:
index_max = index_middle - 1
else:
index_min = index_middle
- bytepos1 = storage[index_min].baseindex
+
+ baseindex = storage[index_min].baseindex
+ if baseindex == bytepos:
+ return index_min << 6
+
+ # use ofs to get closer to the correct character index
result = index_min << 6
+ bytepos1 = baseindex
+ if index_min == len(storage) - 1:
+ maxindex = ((num_codepoints - 1) >> 2) & 0x0F
+ else:
+ maxindex = 16
+ for i in range(maxindex):
+ x = baseindex + ord(storage[index_min].ofs[i])
+ if x >= bytepos:
+ break
+ bytepos1 = x
+ result = (index_min << 6) + (i << 2) + 1
+
+ # this loop should runs at most four times
while bytepos1 < bytepos:
bytepos1 = next_codepoint_pos(utf8, bytepos1)
result += 1
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
--- a/rpython/rlib/test/test_rutf8.py
+++ b/rpython/rlib/test/test_rutf8.py
@@ -1,3 +1,4 @@
+#encoding: utf-8
import pytest
import sys
from hypothesis import given, strategies, settings, example
@@ -133,12 +134,24 @@
@given(strategies.text())
@example(u'x' * 64 * 5)
@example(u'x' * (64 * 5 - 1))
+@example(u'ä' + u'x«' * 1000 + u'–' + u'y' * 100)
def test_codepoint_index_at_byte_position(u):
- storage = rutf8.create_utf8_index_storage(u.encode('utf8'), len(u))
+ b = u.encode('utf8')
+ storage = rutf8.create_utf8_index_storage(b, len(u))
for i in range(len(u) + 1):
bytepos = len(u[:i].encode('utf8'))
assert rutf8.codepoint_index_at_byte_position(
- u.encode('utf8'), storage, bytepos) == i
+ b, storage, bytepos, len(u)) == i
+
+@given(strategies.text())
+def test_codepoint_position_at_index_inverse(u):
+ print u
+ b = u.encode('utf8')
+ storage = rutf8.create_utf8_index_storage(b, len(u))
+ for i in range(len(u) + 1):
+ bytepos = rutf8.codepoint_position_at_index(b, storage, i)
+ assert rutf8.codepoint_index_at_byte_position(
+ b, storage, bytepos, len(u)) == i
repr_func = rutf8.make_utf8_escape_function(prefix='u', pass_printable=False,
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
