Author: Alex Gaynor <alex.gay...@gmail.com> Branch: stdlib-2.7.11 Changeset: r83191:f3d2f640ffca Date: 2016-03-20 10:46 -0400 http://bitbucket.org/pypy/pypy/changeset/f3d2f640ffca/
Log: I have conquered the latest round of utf7 bugs! diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -28,8 +28,8 @@ raises( UnicodeDecodeError, unicode,'\\NSPACE}','unicode-escape') raises( UnicodeDecodeError, unicode,'\\NSPACE','unicode-escape') raises( UnicodeDecodeError, unicode,'\\N','unicode-escape') - assert unicode('\\N{SPACE}\\N{SPACE}','unicode-escape') == u" " - assert unicode('\\N{SPACE}a\\N{SPACE}','unicode-escape') == u" a " + assert unicode('\\N{SPACE}\\N{SPACE}','unicode-escape') == u" " + assert unicode('\\N{SPACE}a\\N{SPACE}','unicode-escape') == u" a " assert "\\N{foo}xx".decode("unicode-escape", "ignore") == u"xx" assert 1 <= len(u"\N{CJK UNIFIED IDEOGRAPH-20000}") <= 2 @@ -676,6 +676,9 @@ (b'a+//,+IKw-b', u'a\ufffd\u20acb'), (b'a+///,+IKw-b', u'a\uffff\ufffd\u20acb'), (b'a+////,+IKw-b', u'a\uffff\ufffd\u20acb'), + (b'a+2AE\xe1b', u'a\ufffdb'), + (b'a+2AEA-b', u'a\ufffdb'), + (b'a+2AH-b', u'a\ufffdb'), ] for raw, expected in tests: raises(UnicodeDecodeError, codecs.utf_7_decode, raw, 'strict', True) diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -812,10 +812,9 @@ startinpos = 0 while pos < size: ch = s[pos] - oc = ord(ch) if inShift: # in a base-64 section - if _utf7_IS_BASE64(oc): #consume a base-64 character + if _utf7_IS_BASE64(ord(ch)): #consume a base-64 character base64buffer = (base64buffer << 6) | _utf7_FROM_BASE64(ch) base64bits += 6 pos += 1 @@ -828,7 +827,7 @@ assert outCh <= 0xffff if surrogate: # expecting a second surrogate - if outCh >= 0xDC00 and outCh <= 0xDFFFF: + if outCh >= 0xDC00 and outCh <= 0xDFFF: if MAXUNICODE < 65536: result.append(unichr(surrogate)) result.append(unichr(outCh)) @@ -851,15 +850,11 @@ else: # now leaving a base-64 section inShift = False - pos += 1 - - if surrogate: - result.append(unichr(surrogate)) - surrogate = 0 if base64bits > 0: # left-over bits if base64bits >= 6: # We've seen at least one base-64 character + pos += 1 msg = "partial character in shift sequence" res, pos = errorhandler(errors, 'utf7', msg, s, pos-1, pos) @@ -868,20 +863,21 @@ else: # Some bits remain; they should be zero if base64buffer != 0: + pos += 1 msg = "non-zero padding bits in shift sequence" res, pos = errorhandler(errors, 'utf7', msg, s, pos-1, pos) result.append(res) continue + if surrogate and _utf7_DECODE_DIRECT(ord(ch)): + result.append(unichr(surrogate)) + surrogate = 0 + if ch == '-': # '-' is absorbed; other terminating characters are # preserved - base64bits = 0 - base64buffer = 0 - surrogate = 0 - else: - result.append(unichr(ord(ch))) + pos += 1 elif ch == '+': startinpos = pos @@ -891,12 +887,13 @@ result.append(u'+') else: # begin base64-encoded section inShift = 1 + surrogate = 0 shiftOutStartPos = result.getlength() base64bits = 0 base64buffer = 0 - elif _utf7_DECODE_DIRECT(oc): # character decodes at itself - result.append(unichr(oc)) + elif _utf7_DECODE_DIRECT(ord(ch)): # character decodes at itself + result.append(unichr(ord(ch))) pos += 1 else: startinpos = pos @@ -909,6 +906,7 @@ final_length = result.getlength() if inShift and final: # in shift sequence, no more to follow # if we're in an inconsistent state, that's an error + inShift = 0 if (surrogate or base64bits >= 6 or (base64bits > 0 and base64buffer != 0)): _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit