Diff
Modified: trunk/LayoutTests/ChangeLog (244826 => 244827)
--- trunk/LayoutTests/ChangeLog 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/LayoutTests/ChangeLog 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,3 +1,16 @@
+2019-05-01 Shawn Roberts <[email protected]>
+
+ Unreviewed, rolling out r244821.
+
+ Causing 4 Test262 failures on JSC Release and Debug
+
+ Reverted changeset:
+
+ "WebKit has too much of its own UTF-8 code and should rely
+ more on ICU's UTF-8 support"
+ https://bugs.webkit.org/show_bug.cgi?id=195535
+ https://trac.webkit.org/changeset/244821
+
2019-05-01 Youenn Fablet <[email protected]>
Reject/throw when calling AudioContext methods on a stopped AudioContext
Modified: trunk/LayoutTests/css3/escape-dom-api-expected.txt (244826 => 244827)
--- trunk/LayoutTests/css3/escape-dom-api-expected.txt 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/LayoutTests/css3/escape-dom-api-expected.txt 2019-05-01 17:13:58 UTC (rev 244827)
@@ -4,14 +4,14 @@
PASS CSS.escape.length is 1
-PASS CSS.escape('\0') is "�"
-PASS CSS.escape('a\0') is "a�"
-PASS CSS.escape('\0b') is "�b"
-PASS CSS.escape('a\0b') is "a�b"
-PASS CSS.escape('�') is "�"
-PASS CSS.escape('a�') is "a�"
-PASS CSS.escape('�b') is "�b"
-PASS CSS.escape('a�b') is "a�b"
+PASS CSS.escape('\0') is "�"
+PASS CSS.escape('a\0') is "a�"
+PASS CSS.escape('\0b') is "�b"
+PASS CSS.escape('a\0b') is "a�b"
+PASS CSS.escape('�') is "�"
+PASS CSS.escape('a�') is "a�"
+PASS CSS.escape('�b') is "�b"
+PASS CSS.escape('a�b') is "a�b"
PASS CSS.escape() threw exception TypeError: Not enough arguments.
PASS CSS.escape(undefined) is "undefined"
PASS CSS.escape(true) is "true"
@@ -53,16 +53,16 @@
PASS CSS.escape('-a') is "-a"
PASS CSS.escape('--') is "--"
PASS CSS.escape('--a') is "--a"
-PASS CSS.escape('-_©') is "-_©"
-PASS CSS.escape('
') is "\\7f
"
-PASS CSS.escape(' ¡¢') is " ¡¢"
+PASS CSS.escape('Â-_©') is "Â-_©"
+PASS CSS.escape('ÂÂÂÂÂÂ
ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ') is "\\7f ÂÂÂÂÂÂ
ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ"
+PASS CSS.escape(' ¡¢') is " ¡¢"
PASS CSS.escape('a0123456789b') is "a0123456789b"
PASS CSS.escape('abcdefghijklmnopqrstuvwxyz') is "abcdefghijklmnopqrstuvwxyz"
PASS CSS.escape('ABCDEFGHIJKLMNOPQRSTUVWXYZ') is "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
PASS CSS.escape(' !xy') is "\\ \\!xy"
-PASS CSS.escape('𝌆') is "𝌆"
-PASS CSS.escape('�') is "\udf06"
-PASS CSS.escape('�') is "\ud834"
+PASS CSS.escape('ð') is "ð"
+PASS CSS.escape('í¼') is "\udf06"
+PASS CSS.escape('í ´') is "\ud834"
PASS successfullyParsed is true
TEST COMPLETE
Modified: trunk/LayoutTests/fast/text/dangling-surrogates-expected.txt (244826 => 244827)
--- trunk/LayoutTests/fast/text/dangling-surrogates-expected.txt 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/LayoutTests/fast/text/dangling-surrogates-expected.txt 2019-05-01 17:13:58 UTC (rev 244827)
@@ -3,8 +3,8 @@
On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
-PASS danglingFirst is "�"
-PASS danglingSecond is "�"
+PASS danglingFirst is "í "
+PASS danglingSecond is "í°"
PASS successfullyParsed is true
TEST COMPLETE
Modified: trunk/LayoutTests/imported/w3c/ChangeLog (244826 => 244827)
--- trunk/LayoutTests/imported/w3c/ChangeLog 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/LayoutTests/imported/w3c/ChangeLog 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,3 +1,16 @@
+2019-05-01 Shawn Roberts <[email protected]>
+
+ Unreviewed, rolling out r244821.
+
+ Causing
+
+ Reverted changeset:
+
+ "WebKit has too much of its own UTF-8 code and should rely
+ more on ICU's UTF-8 support"
+ https://bugs.webkit.org/show_bug.cgi?id=195535
+ https://trac.webkit.org/changeset/244821
+
2019-05-01 Youenn Fablet <[email protected]>
Kept alive loaders should use the redirected request in case of redirections
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/encoding/textdecoder-utf16-surrogates-expected.txt (244826 => 244827)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/encoding/textdecoder-utf16-surrogates-expected.txt 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/encoding/textdecoder-utf16-surrogates-expected.txt 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,21 +1,21 @@
-FAIL utf-16le - lone surrogate lead assert_equals: expected "\ufffd" but got "�"
+FAIL utf-16le - lone surrogate lead assert_equals: expected "\ufffd" but got "í "
FAIL utf-16le - lone surrogate lead (fatal flag set) assert_throws: function "function () {
new TextDecoder(t.encoding, {fatal: true}).decode(new Uint8Array(t.input))
}" did not throw
-FAIL utf-16le - lone surrogate trail assert_equals: expected "\ufffd" but got "�"
+FAIL utf-16le - lone surrogate trail assert_equals: expected "\ufffd" but got "í°"
FAIL utf-16le - lone surrogate trail (fatal flag set) assert_throws: function "function () {
new TextDecoder(t.encoding, {fatal: true}).decode(new Uint8Array(t.input))
}" did not throw
-FAIL utf-16le - unmatched surrogate lead assert_equals: expected "\ufffd\0" but got "�\0"
+FAIL utf-16le - unmatched surrogate lead assert_equals: expected "\ufffd\0" but got "í \0"
FAIL utf-16le - unmatched surrogate lead (fatal flag set) assert_throws: function "function () {
new TextDecoder(t.encoding, {fatal: true}).decode(new Uint8Array(t.input))
}" did not throw
-FAIL utf-16le - unmatched surrogate trail assert_equals: expected "\ufffd\0" but got "�\0"
+FAIL utf-16le - unmatched surrogate trail assert_equals: expected "\ufffd\0" but got "í°\0"
FAIL utf-16le - unmatched surrogate trail (fatal flag set) assert_throws: function "function () {
new TextDecoder(t.encoding, {fatal: true}).decode(new Uint8Array(t.input))
}" did not throw
-FAIL utf-16le - swapped surrogate pair assert_equals: expected "\ufffd\ufffd" but got "��"
+FAIL utf-16le - swapped surrogate pair assert_equals: expected "\ufffd\ufffd" but got "í°í "
FAIL utf-16le - swapped surrogate pair (fatal flag set) assert_throws: function "function () {
new TextDecoder(t.encoding, {fatal: true}).decode(new Uint8Array(t.input))
}" did not throw
Modified: trunk/LayoutTests/js/dom/webidl-type-mapping-expected.txt (244826 => 244827)
--- trunk/LayoutTests/js/dom/webidl-type-mapping-expected.txt 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/LayoutTests/js/dom/webidl-type-mapping-expected.txt 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1009,48 +1009,48 @@
converter.testUSVString = '!@#123ABCabc\x00\x80\xFF\r\n\t'
converter.testString = '!@#123ABCabc\x00\x80\xFF\r\n\t'
-PASS converter.testUSVString is "!@#123ABCabc\u0000ÿ\r\n\t"
-PASS converter.testString is "!@#123ABCabc\u0000ÿ\r\n\t"
+PASS converter.testUSVString is "!@#123ABCabc\u0000Âÿ\r\n\t"
+PASS converter.testString is "!@#123ABCabc\u0000Âÿ\r\n\t"
converter.testUSVString = '\u0100'
converter.testString = '\u0100'
-PASS converter.testUSVString is "Ā"
-PASS converter.testString is "Ā"
+PASS converter.testUSVString is "Ä"
+PASS converter.testString is "Ä"
PASS converter.testUSVString = {toString: function() { throw Error(); }} threw exception Error.
PASS converter.testString = {toString: function() { throw Error(); }} threw exception Error.
-PASS converter.testUSVString is "Ā"
-PASS converter.testString is "Ā"
+PASS converter.testUSVString is "Ä"
+PASS converter.testString is "Ä"
converter.testUSVString = "\ud800"
converter.testString = "\ud800"
-PASS converter.testUSVString is "�"
+PASS converter.testUSVString is "�"
PASS converter.testString is "\ud800"
converter.testUSVString = "\udc00"
converter.testString = "\udc00"
-PASS converter.testUSVString is "�"
+PASS converter.testUSVString is "�"
PASS converter.testString is "\udc00"
converter.testUSVString = "\ud800\u0000"
converter.testString = "\ud800\u0000"
-PASS converter.testUSVString is "�\u0000"
+PASS converter.testUSVString is "�\u0000"
PASS converter.testString is "\ud800\u0000"
converter.testUSVString = "\udc00\u0000"
converter.testString = "\udc00\u0000"
-PASS converter.testUSVString is "�\u0000"
+PASS converter.testUSVString is "�\u0000"
PASS converter.testString is "\udc00\u0000"
converter.testUSVString = "\udc00\ud800"
converter.testString = "\udc00\ud800"
-PASS converter.testUSVString is "��"
+PASS converter.testUSVString is "��"
PASS converter.testString is "\udc00\ud800"
-converter.testUSVString = "𝄞"
-converter.testString = "𝄞"
-PASS converter.testUSVString is "𝄞"
-PASS converter.testString is "𝄞"
+converter.testUSVString = "ð"
+converter.testString = "ð"
+PASS converter.testUSVString is "ð"
+PASS converter.testString is "ð"
converter.testByteString = '!@#123ABCabc\x00\x80\xFF\r\n\t'
-PASS converter.testByteString is "!@#123ABCabc\u0000ÿ\r\n\t"
+PASS converter.testByteString is "!@#123ABCabc\u0000Âÿ\r\n\t"
converter.testByteString = '\u00FF'
-PASS converter.testByteString is "ÿ"
+PASS converter.testByteString is "ÿ"
PASS converter.testByteString = '\u0100' threw exception TypeError: Type error.
-PASS converter.testByteString is "ÿ"
+PASS converter.testByteString is "ÿ"
PASS converter.testByteString = {toString: function() { throw Error(); }} threw exception Error.
-PASS converter.testByteString is "ÿ"
+PASS converter.testByteString is "ÿ"
converter.testUSVString = true
converter.testString = true
converter.testByteString = true
@@ -1180,37 +1180,37 @@
PASS 'key2' in converter.testNodeRecord() is true
PASS converter.testNodeRecord()['key2'] is document.documentElement
PASS converter.setTestNodeRecord({ key: 'hello' }) threw exception TypeError: Type error.
-converter.setTestLongRecord({'�': 1 })
-PASS converter.testLongRecord()['�'] is 1
-converter.setTestNodeRecord({'�': document })
-PASS converter.testNodeRecord()['�'] is document
-converter.setTestLongRecord({'�': 1 })
-PASS converter.testLongRecord()['�'] is 1
-converter.setTestNodeRecord({'�': document })
-PASS converter.testNodeRecord()['�'] is document
-converter.setTestLongRecord({'�': 1 })
-PASS converter.testLongRecord()['�\0'] is 1
-converter.setTestNodeRecord({'�': document })
-PASS converter.testNodeRecord()['�\0'] is document
-converter.setTestLongRecord({'�': 1 })
-PASS converter.testLongRecord()['�\0'] is 1
-converter.setTestNodeRecord({'�': document })
-PASS converter.testNodeRecord()['�\0'] is document
-converter.setTestLongRecord({'��': 1 })
-PASS converter.testLongRecord()['��'] is 1
-converter.setTestNodeRecord({'��': document })
-PASS converter.testNodeRecord()['��'] is document
-converter.setTestLongRecord({'𝄞': 1 })
-PASS converter.testLongRecord()['𝄞'] is 1
-converter.setTestNodeRecord({'𝄞': document })
-PASS converter.testNodeRecord()['𝄞'] is document
+converter.setTestLongRecord({'í ': 1 })
+PASS converter.testLongRecord()['í '] is 1
+converter.setTestNodeRecord({'í ': document })
+PASS converter.testNodeRecord()['�'] is document
+converter.setTestLongRecord({'í°': 1 })
+PASS converter.testLongRecord()['í°'] is 1
+converter.setTestNodeRecord({'í°': document })
+PASS converter.testNodeRecord()['�'] is document
+converter.setTestLongRecord({'í ': 1 })
+PASS converter.testLongRecord()['í \0'] is 1
+converter.setTestNodeRecord({'í ': document })
+PASS converter.testNodeRecord()['�\0'] is document
+converter.setTestLongRecord({'í°': 1 })
+PASS converter.testLongRecord()['í°\0'] is 1
+converter.setTestNodeRecord({'í°': document })
+PASS converter.testNodeRecord()['�\0'] is document
+converter.setTestLongRecord({'í°í ': 1 })
+PASS converter.testLongRecord()['í°í '] is 1
+converter.setTestNodeRecord({'í°í ': document })
+PASS converter.testNodeRecord()['��'] is document
+converter.setTestLongRecord({'ð': 1 })
+PASS converter.testLongRecord()['ð'] is 1
+converter.setTestNodeRecord({'ð': document })
+PASS converter.testNodeRecord()['ð'] is document
converter.setTestSequenceRecord({ key: ['value', 'other value'] })
PASS converter.testSequenceRecord().hasOwnProperty('key') is true
PASS 'key' in converter.testSequenceRecord() is true
PASS converter.testSequenceRecord()['key'] is ['value', 'other value']
-PASS converter.setTestSequenceRecord({ 'Ā': ['value'] }) threw exception TypeError: Type error.
-converter.setTestSequenceRecord({ 'ÿ': ['value'] })
-PASS converter.testSequenceRecord()['ÿ'] is ['value']
+PASS converter.setTestSequenceRecord({ 'Ä': ['value'] }) threw exception TypeError: Type error.
+converter.setTestSequenceRecord({ 'ÿ': ['value'] })
+PASS converter.testSequenceRecord()['ÿ'] is ['value']
PASS converter.testImpureNaNUnrestrictedDouble is NaN
PASS converter.testImpureNaN2UnrestrictedDouble is NaN
PASS converter.testQuietNaNUnrestrictedDouble is NaN
Modified: trunk/LayoutTests/js/invalid-utf8-in-syntax-error-expected.txt (244826 => 244827)
--- trunk/LayoutTests/js/invalid-utf8-in-syntax-error-expected.txt 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/LayoutTests/js/invalid-utf8-in-syntax-error-expected.txt 2019-05-01 17:13:58 UTC (rev 244827)
@@ -3,7 +3,7 @@
On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
-PASS ({f("�")}) threw exception SyntaxError: Unexpected string literal "�". Expected a parameter pattern or a ')' in parameter list..
+PASS ({f("\x{DEAD}")}) threw exception SyntaxError: Unexpected string literal "íº". Expected a parameter pattern or a ')' in parameter list..
PASS successfullyParsed is true
TEST COMPLETE
Deleted: trunk/LayoutTests/js/invalid-utf8-in-syntax-error.html (244826 => 244827)
--- trunk/LayoutTests/js/invalid-utf8-in-syntax-error.html 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/LayoutTests/js/invalid-utf8-in-syntax-error.html 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,10 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<meta charset="utf-8">
-<script src=""
-</head>
-<body>
-<script src=""
-</body>
-</html>
Modified: trunk/Source/_javascript_Core/API/JSClassRef.cpp (244826 => 244827)
--- trunk/Source/_javascript_Core/API/JSClassRef.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/_javascript_Core/API/JSClassRef.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -35,8 +35,10 @@
#include "ObjectPrototype.h"
#include "JSCInlines.h"
#include <wtf/text/StringHash.h>
+#include <wtf/unicode/UTF8Conversion.h>
using namespace JSC;
+using namespace WTF::Unicode;
const JSClassDefinition kJSClassDefinitionEmpty = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
Modified: trunk/Source/_javascript_Core/API/JSStringRef.cpp (244826 => 244827)
--- trunk/Source/_javascript_Core/API/JSStringRef.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/_javascript_Core/API/JSStringRef.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -49,7 +49,7 @@
UChar* p = buffer.data();
bool sourceIsAllASCII;
const LChar* stringStart = reinterpret_cast<const LChar*>(string);
- if (convertUTF8ToUTF16(string, string + length, &p, p + length, &sourceIsAllASCII)) {
+ if (conversionOK == convertUTF8ToUTF16(&string, string + length, &p, p + length, &sourceIsAllASCII)) {
if (sourceIsAllASCII)
return &OpaqueJSString::create(stringStart, length).leakRef();
return &OpaqueJSString::create(buffer.data(), p - buffer.data()).leakRef();
@@ -102,18 +102,20 @@
return 0;
char* destination = buffer;
- bool failed = false;
+ ConversionResult result;
if (string->is8Bit()) {
const LChar* source = string->characters8();
- convertLatin1ToUTF8(&source, source + string->length(), &destination, destination + bufferSize - 1);
+ result = convertLatin1ToUTF8(&source, source + string->length(), &destination, destination + bufferSize - 1);
} else {
const UChar* source = string->characters16();
- ConversionResult result = convertUTF16ToUTF8(&source, source + string->length(), &destination, destination + bufferSize - 1);
- failed = result != ConversionOK && result != TargetExhausted;
+ result = convertUTF16ToUTF8(&source, source + string->length(), &destination, destination + bufferSize - 1, true);
}
*destination++ = '\0';
- return failed ? 0 : destination - buffer;
+ if (result != conversionOK && result != targetExhausted)
+ return 0;
+
+ return destination - buffer;
}
bool JSStringIsEqual(JSStringRef a, JSStringRef b)
Modified: trunk/Source/_javascript_Core/ChangeLog (244826 => 244827)
--- trunk/Source/_javascript_Core/ChangeLog 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/_javascript_Core/ChangeLog 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,3 +1,16 @@
+2019-05-01 Shawn Roberts <[email protected]>
+
+ Unreviewed, rolling out r244821.
+
+ Causing
+
+ Reverted changeset:
+
+ "WebKit has too much of its own UTF-8 code and should rely
+ more on ICU's UTF-8 support"
+ https://bugs.webkit.org/show_bug.cgi?id=195535
+ https://trac.webkit.org/changeset/244821
+
2019-04-29 Darin Adler <[email protected]>
WebKit has too much of its own UTF-8 code and should rely more on ICU's UTF-8 support
Modified: trunk/Source/_javascript_Core/runtime/JSGlobalObjectFunctions.cpp (244826 => 244827)
--- trunk/Source/_javascript_Core/runtime/JSGlobalObjectFunctions.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/_javascript_Core/runtime/JSGlobalObjectFunctions.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -58,9 +58,12 @@
#include <wtf/MathExtras.h>
#include <wtf/dtoa.h>
#include <wtf/text/StringBuilder.h>
+#include <wtf/unicode/UTF8Conversion.h>
namespace JSC {
+using namespace WTF::Unicode;
+
const ASCIILiteral ObjectProtoCalledOnNullOrUndefinedError { "Object.prototype.__proto__ called on null or undefined"_s };
template<unsigned charactersCount>
@@ -181,10 +184,10 @@
int charLen = 0;
if (k <= length - 3 && isASCIIHexDigit(p[1]) && isASCIIHexDigit(p[2])) {
const char b0 = Lexer<CharType>::convertHex(p[1], p[2]);
- const int sequenceLen = 1 + U8_COUNT_TRAIL_BYTES(b0);
- if (k <= length - sequenceLen * 3) {
+ const int sequenceLen = UTF8SequenceLength(b0);
+ if (sequenceLen && k <= length - sequenceLen * 3) {
charLen = sequenceLen * 3;
- uint8_t sequence[U8_MAX_LENGTH];
+ char sequence[5];
sequence[0] = b0;
for (int i = 1; i < sequenceLen; ++i) {
const CharType* q = p + i * 3;
@@ -196,20 +199,16 @@
}
}
if (charLen != 0) {
- UChar32 character;
- int32_t offset = 0;
- U8_NEXT(sequence, offset, sequenceLen, character);
- if (character < 0)
+ sequence[sequenceLen] = 0;
+ const int character = decodeUTF8Sequence(sequence);
+ if (character < 0 || character >= 0x110000)
charLen = 0;
- else if (!U_IS_BMP(character)) {
+ else if (character >= 0x10000) {
// Convert to surrogate pair.
- ASSERT(U_IS_SUPPLEMENTARY(character));
- builder.append(U16_LEAD(character));
- u = U16_TRAIL(character);
- } else {
- ASSERT(!U_IS_SURROGATE(character));
+ builder.append(static_cast<UChar>(0xD800 | ((character - 0x10000) >> 10)));
+ u = static_cast<UChar>(0xDC00 | ((character - 0x10000) & 0x3FF));
+ } else
u = static_cast<UChar>(character);
- }
}
}
}
Modified: trunk/Source/_javascript_Core/wasm/WasmParser.h (244826 => 244827)
--- trunk/Source/_javascript_Core/wasm/WasmParser.h 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/_javascript_Core/wasm/WasmParser.h 2019-05-01 17:13:58 UTC (rev 244827)
@@ -162,7 +162,7 @@
UChar* bufferCurrent = bufferStart;
const char* stringCurrent = reinterpret_cast<const char*>(stringStart);
- if (!WTF::Unicode::convertUTF8ToUTF16(stringCurrent, reinterpret_cast<const char *>(stringStart + stringLength), &bufferCurrent, bufferCurrent + buffer.size()))
+ if (WTF::Unicode::convertUTF8ToUTF16(&stringCurrent, reinterpret_cast<const char *>(stringStart + stringLength), &bufferCurrent, bufferCurrent + buffer.size()) != WTF::Unicode::conversionOK)
return false;
}
Modified: trunk/Source/WTF/ChangeLog (244826 => 244827)
--- trunk/Source/WTF/ChangeLog 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WTF/ChangeLog 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,5 +1,18 @@
2019-05-01 Shawn Roberts <[email protected]>
+ Unreviewed, rolling out r244821.
+
+ Causing
+
+ Reverted changeset:
+
+ "WebKit has too much of its own UTF-8 code and should rely
+ more on ICU's UTF-8 support"
+ https://bugs.webkit.org/show_bug.cgi?id=195535
+ https://trac.webkit.org/changeset/244821
+
+2019-05-01 Shawn Roberts <[email protected]>
+
Unreviewed, rolling out r244822.
Causing 4 Test262 failures on JSC Release and Debug
Modified: trunk/Source/WTF/wtf/text/AtomicString.cpp (244826 => 244827)
--- trunk/Source/WTF/wtf/text/AtomicString.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WTF/wtf/text/AtomicString.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -113,24 +113,19 @@
return numberToString(number, buffer);
}
-AtomicString AtomicString::fromUTF8Internal(const char* start, const char* end)
+AtomicString AtomicString::fromUTF8Internal(const char* charactersStart, const char* charactersEnd)
{
- ASSERT(start);
-
- // Caller needs to handle empty string.
- ASSERT(!end || end > start);
- ASSERT(end || start[0]);
-
- return AtomicStringImpl::addUTF8(start, end ? end : start + std::strlen(start));
+ auto impl = AtomicStringImpl::addUTF8(charactersStart, charactersEnd);
+ if (!impl)
+ return nullAtom();
+ return impl.get();
}
#ifndef NDEBUG
-
void AtomicString::show() const
{
m_string.show();
}
-
#endif
WTF_EXPORT_PRIVATE LazyNeverDestroyed<AtomicString> nullAtomData;
Modified: trunk/Source/WTF/wtf/text/AtomicStringImpl.cpp (244826 => 244827)
--- trunk/Source/WTF/wtf/text/AtomicStringImpl.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WTF/wtf/text/AtomicStringImpl.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -219,7 +219,7 @@
bool isAllASCII;
const char* source = buffer.characters;
- if (!convertUTF8ToUTF16(source, source + buffer.length, &target, target + buffer.utf16Length, &isAllASCII))
+ if (convertUTF8ToUTF16(&source, source + buffer.length, &target, target + buffer.utf16Length, &isAllASCII) != conversionOK)
ASSERT_NOT_REACHED();
if (isAllASCII)
Modified: trunk/Source/WTF/wtf/text/AtomicStringImpl.h (244826 => 244827)
--- trunk/Source/WTF/wtf/text/AtomicStringImpl.h 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WTF/wtf/text/AtomicStringImpl.h 2019-05-01 17:13:58 UTC (rev 244827)
@@ -56,8 +56,7 @@
WTF_EXPORT_PRIVATE static Ref<AtomicStringImpl> addLiteral(const char* characters, unsigned length);
// Returns null if the input data contains an invalid UTF-8 sequence.
- static RefPtr<AtomicStringImpl> addUTF8(const char* start, const char* end);
-
+ WTF_EXPORT_PRIVATE static RefPtr<AtomicStringImpl> addUTF8(const char* start, const char* end);
#if USE(CF)
WTF_EXPORT_PRIVATE static RefPtr<AtomicStringImpl> add(CFStringRef);
#endif
Modified: trunk/Source/WTF/wtf/text/StringImpl.cpp (244826 => 244827)
--- trunk/Source/WTF/wtf/text/StringImpl.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WTF/wtf/text/StringImpl.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1756,11 +1756,11 @@
char* bufferEnd = buffer + bufferSize;
while (characters < charactersEnd) {
// Use strict conversion to detect unpaired surrogates.
- ConversionResult result = convertUTF16ToUTF8(&characters, charactersEnd, &buffer, bufferEnd);
- ASSERT(result != TargetExhausted);
+ ConversionResult result = convertUTF16ToUTF8(&characters, charactersEnd, &buffer, bufferEnd, true);
+ ASSERT(result != targetExhausted);
// Conversion fails when there is an unpaired surrogate.
// Put replacement character (U+FFFD) instead of the unpaired surrogate.
- if (result != ConversionOK) {
+ if (result != conversionOK) {
ASSERT((0xD800 <= *characters && *characters <= 0xDFFF));
// There should be room left, since one UChar hasn't been converted.
ASSERT((buffer + 3) <= bufferEnd);
@@ -1772,16 +1772,16 @@
bool strict = mode == StrictConversion;
const UChar* originalCharacters = characters;
ConversionResult result = convertUTF16ToUTF8(&characters, characters + length, &buffer, buffer + bufferSize, strict);
- ASSERT(result != TargetExhausted); // (length * 3) should be sufficient for any conversion
+ ASSERT(result != targetExhausted); // (length * 3) should be sufficient for any conversion
// Only produced from strict conversion.
- if (result == SourceIllegal) {
+ if (result == sourceIllegal) {
ASSERT(strict);
return UTF8ConversionError::IllegalSource;
}
// Check for an unconverted high surrogate.
- if (result == SourceExhausted) {
+ if (result == sourceExhausted) {
if (strict)
return UTF8ConversionError::SourceExhausted;
// This should be one unpaired high surrogate. Treat it the same
@@ -1809,8 +1809,8 @@
Vector<char, 1024> bufferVector(length * 3);
char* buffer = bufferVector.data();
const LChar* source = characters;
- bool charactersFit = convertLatin1ToUTF8(&source, source + length, &buffer, buffer + bufferVector.size());
- ASSERT_UNUSED(charactersFit, charactersFit); // (length * 3) should be sufficient for any conversion
+ ConversionResult result = convertLatin1ToUTF8(&source, source + length, &buffer, buffer + bufferVector.size());
+ ASSERT_UNUSED(result, result != targetExhausted); // (length * 3) should be sufficient for any conversion
return CString(bufferVector.data(), buffer - bufferVector.data());
}
@@ -1854,8 +1854,9 @@
if (is8Bit()) {
const LChar* characters = this->characters8() + offset;
- bool charactersFit = convertLatin1ToUTF8(&characters, characters + length, &buffer, buffer + bufferVector.size());
- ASSERT_UNUSED(charactersFit, charactersFit); // (length * 3) should be sufficient for any conversion
+
+ ConversionResult result = convertLatin1ToUTF8(&characters, characters + length, &buffer, buffer + bufferVector.size());
+ ASSERT_UNUSED(result, result != targetExhausted); // (length * 3) should be sufficient for any conversion
} else {
UTF8ConversionError error = utf8Impl(this->characters16() + offset, length, buffer, bufferVector.size(), mode);
if (error != UTF8ConversionError::None)
Modified: trunk/Source/WTF/wtf/text/StringView.cpp (244826 => 244827)
--- trunk/Source/WTF/wtf/text/StringView.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WTF/wtf/text/StringView.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -35,9 +35,12 @@
#include <wtf/NeverDestroyed.h>
#include <wtf/Optional.h>
#include <wtf/text/TextBreakIterator.h>
+#include <wtf/unicode/UTF8Conversion.h>
namespace WTF {
+using namespace Unicode;
+
bool StringView::containsIgnoringASCIICase(const StringView& matchString) const
{
return findIgnoringASCIICase(matchString) != notFound;
Modified: trunk/Source/WTF/wtf/text/WTFString.cpp (244826 => 244827)
--- trunk/Source/WTF/wtf/text/WTFString.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WTF/wtf/text/WTFString.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -859,7 +859,7 @@
UChar* bufferCurrent = bufferStart;
const char* stringCurrent = reinterpret_cast<const char*>(stringStart);
- if (!convertUTF8ToUTF16(stringCurrent, reinterpret_cast<const char*>(stringStart + length), &bufferCurrent, bufferCurrent + buffer.size()))
+ if (convertUTF8ToUTF16(&stringCurrent, reinterpret_cast<const char *>(stringStart + length), &bufferCurrent, bufferCurrent + buffer.size()) != conversionOK)
return String();
unsigned utf16Length = bufferCurrent - bufferStart;
Modified: trunk/Source/WTF/wtf/unicode/UTF8Conversion.cpp (244826 => 244827)
--- trunk/Source/WTF/wtf/unicode/UTF8Conversion.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WTF/wtf/unicode/UTF8Conversion.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2007, 2010-2012, 2014, 2019 Apple Inc. All rights reserved.
+ * Copyright (C) 2007, 2014 Apple Inc. All rights reserved.
* Copyright (C) 2010 Patrick Gansterer <[email protected]>
*
* Redistribution and use in source and binary forms, with or without
@@ -34,107 +34,389 @@
namespace WTF {
namespace Unicode {
-bool convertLatin1ToUTF8(const LChar** sourceStart, const LChar* sourceEnd, char** targetStart, char* targetEnd)
+inline int inlineUTF8SequenceLengthNonASCII(char b0)
{
- const LChar* source;
+ if ((b0 & 0xC0) != 0xC0)
+ return 0;
+ if ((b0 & 0xE0) == 0xC0)
+ return 2;
+ if ((b0 & 0xF0) == 0xE0)
+ return 3;
+ if ((b0 & 0xF8) == 0xF0)
+ return 4;
+ return 0;
+}
+
+inline int inlineUTF8SequenceLength(char b0)
+{
+ return isASCII(b0) ? 1 : inlineUTF8SequenceLengthNonASCII(b0);
+}
+
+int UTF8SequenceLength(char b0)
+{
+ return isASCII(b0) ? 1 : inlineUTF8SequenceLengthNonASCII(b0);
+}
+
+int decodeUTF8Sequence(const char* sequence)
+{
+ // Handle 0-byte sequences (never valid).
+ const unsigned char b0 = sequence[0];
+ const int length = inlineUTF8SequenceLength(b0);
+ if (length == 0)
+ return -1;
+
+ // Handle 1-byte sequences (plain ASCII).
+ const unsigned char b1 = sequence[1];
+ if (length == 1) {
+ if (b1)
+ return -1;
+ return b0;
+ }
+
+ // Handle 2-byte sequences.
+ if ((b1 & 0xC0) != 0x80)
+ return -1;
+ const unsigned char b2 = sequence[2];
+ if (length == 2) {
+ if (b2)
+ return -1;
+ const int c = ((b0 & 0x1F) << 6) | (b1 & 0x3F);
+ if (c < 0x80)
+ return -1;
+ return c;
+ }
+
+ // Handle 3-byte sequences.
+ if ((b2 & 0xC0) != 0x80)
+ return -1;
+ const unsigned char b3 = sequence[3];
+ if (length == 3) {
+ if (b3)
+ return -1;
+ const int c = ((b0 & 0xF) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F);
+ if (c < 0x800)
+ return -1;
+ // UTF-16 surrogates should never appear in UTF-8 data.
+ if (c >= 0xD800 && c <= 0xDFFF)
+ return -1;
+ return c;
+ }
+
+ // Handle 4-byte sequences.
+ if ((b3 & 0xC0) != 0x80)
+ return -1;
+ const unsigned char b4 = sequence[4];
+ if (length == 4) {
+ if (b4)
+ return -1;
+ const int c = ((b0 & 0x7) << 18) | ((b1 & 0x3F) << 12) | ((b2 & 0x3F) << 6) | (b3 & 0x3F);
+ if (c < 0x10000 || c > 0x10FFFF)
+ return -1;
+ return c;
+ }
+
+ return -1;
+}
+
+// Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
+// into the first byte, depending on how many bytes follow. There are
+// as many entries in this table as there are UTF-8 sequence types.
+// (I.e., one byte sequence, two byte... etc.). Remember that sequencs
+// for *legal* UTF-8 will be 4 or fewer bytes total.
+static const unsigned char firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
+
+ConversionResult convertLatin1ToUTF8(
+ const LChar** sourceStart, const LChar* sourceEnd,
+ char** targetStart, char* targetEnd)
+{
+ ConversionResult result = conversionOK;
+ const LChar* source = *sourceStart;
char* target = *targetStart;
- unsigned i = 0;
- for (source = *sourceStart; source < sourceEnd; ++source) {
- UBool sawError = false;
- // Work around bug in either Windows compiler or old version of ICU, where passing a uint8_t to
- // U8_APPEND warns, by convering from uint8_t to a wider type.
- UChar32 character = *source;
- U8_APPEND(reinterpret_cast<uint8_t*>(target), i, targetEnd - *targetStart, character, sawError);
- if (sawError)
- return false;
+ while (source < sourceEnd) {
+ UChar32 ch;
+ unsigned short bytesToWrite = 0;
+ const UChar32 byteMask = 0xBF;
+ const UChar32 byteMark = 0x80;
+ const LChar* oldSource = source; // In case we have to back up because of target overflow.
+ ch = static_cast<unsigned short>(*source++);
+
+ // Figure out how many bytes the result will require
+ if (ch < (UChar32)0x80)
+ bytesToWrite = 1;
+ else
+ bytesToWrite = 2;
+
+ target += bytesToWrite;
+ if (target > targetEnd) {
+ source = oldSource; // Back up source pointer!
+ target -= bytesToWrite;
+ result = targetExhausted;
+ break;
+ }
+ switch (bytesToWrite) { // note: everything falls through.
+ case 2:
+ *--target = (char)((ch | byteMark) & byteMask);
+ ch >>= 6;
+ FALLTHROUGH;
+ case 1:
+ *--target = (char)(ch | firstByteMark[bytesToWrite]);
+ }
+ target += bytesToWrite;
}
*sourceStart = source;
- *targetStart = target + i;
- return true;
+ *targetStart = target;
+ return result;
}
-ConversionResult convertUTF16ToUTF8(const UChar** sourceStart, const UChar* sourceEnd, char** targetStart, char* targetEnd, bool strict)
+ConversionResult convertUTF16ToUTF8(
+ const UChar** sourceStart, const UChar* sourceEnd,
+ char** targetStart, char* targetEnd, bool strict)
{
- ConversionResult result = ConversionOK;
+ ConversionResult result = conversionOK;
const UChar* source = *sourceStart;
char* target = *targetStart;
- UBool sawError = false;
- unsigned i = 0;
while (source < sourceEnd) {
UChar32 ch;
- int j = 0;
- U16_NEXT(source, j, sourceEnd - source, ch);
- if (U_IS_SURROGATE(ch)) {
- if (source + j == sourceEnd && U_IS_SURROGATE_LEAD(ch)) {
- result = SourceExhausted;
+ unsigned short bytesToWrite = 0;
+ const UChar32 byteMask = 0xBF;
+ const UChar32 byteMark = 0x80;
+ const UChar* oldSource = source; // In case we have to back up because of target overflow.
+ ch = static_cast<unsigned short>(*source++);
+ // If we have a surrogate pair, convert to UChar32 first.
+ if (ch >= 0xD800 && ch <= 0xDBFF) {
+ // If the 16 bits following the high surrogate are in the source buffer...
+ if (source < sourceEnd) {
+ UChar32 ch2 = static_cast<unsigned short>(*source);
+ // If it's a low surrogate, convert to UChar32.
+ if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
+ ch = ((ch - 0xD800) << 10) + (ch2 - 0xDC00) + 0x0010000;
+ ++source;
+ } else if (strict) { // it's an unpaired high surrogate
+ --source; // return to the illegal value itself
+ result = sourceIllegal;
+ break;
+ }
+ } else { // We don't have the 16 bits following the high surrogate.
+ --source; // return to the high surrogate
+ result = sourceExhausted;
break;
}
- if (strict) {
- result = SourceIllegal;
+ } else if (strict) {
+ // UTF-16 surrogate values are illegal in UTF-32
+ if (ch >= 0xDC00 && ch <= 0xDFFF) {
+ --source; // return to the illegal value itself
+ result = sourceIllegal;
break;
}
+ }
+ // Figure out how many bytes the result will require
+ if (ch < (UChar32)0x80) {
+ bytesToWrite = 1;
+ } else if (ch < (UChar32)0x800) {
+ bytesToWrite = 2;
+ } else if (ch < (UChar32)0x10000) {
+ bytesToWrite = 3;
+ } else if (ch < (UChar32)0x110000) {
+ bytesToWrite = 4;
+ } else {
+ bytesToWrite = 3;
ch = replacementCharacter;
}
- U8_APPEND(reinterpret_cast<uint8_t*>(target), i, targetEnd - target, ch, sawError);
- if (sawError) {
- result = TargetExhausted;
+
+ target += bytesToWrite;
+ if (target > targetEnd) {
+ source = oldSource; // Back up source pointer!
+ target -= bytesToWrite;
+ result = targetExhausted;
break;
}
- source += j;
+ switch (bytesToWrite) { // note: everything falls through.
+ case 4: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6; FALLTHROUGH;
+ case 3: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6; FALLTHROUGH;
+ case 2: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6; FALLTHROUGH;
+ case 1: *--target = (char)(ch | firstByteMark[bytesToWrite]);
+ }
+ target += bytesToWrite;
}
*sourceStart = source;
- *targetStart = target + i;
+ *targetStart = target;
return result;
}
-bool convertUTF8ToUTF16(const char* source, const char* sourceEnd, UChar** targetStart, UChar* targetEnd, bool* sourceAllASCII)
+// This must be called with the length pre-determined by the first byte.
+// If presented with a length > 4, this returns false. The Unicode
+// definition of UTF-8 goes up to 4-byte sequences.
+static bool isLegalUTF8(const unsigned char* source, int length)
{
- RELEASE_ASSERT(sourceEnd - source <= std::numeric_limits<int>::max());
- UBool error = false;
+ unsigned char a;
+ const unsigned char* srcptr = source + length;
+ switch (length) {
+ default: return false;
+ // Everything else falls through when "true"...
+ case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; FALLTHROUGH;
+ case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; FALLTHROUGH;
+ case 2: if ((a = (*--srcptr)) > 0xBF) return false;
+
+ switch (*source) {
+ // no fall-through in this inner switch
+ case 0xE0: if (a < 0xA0) return false; break;
+ case 0xED: if (a > 0x9F) return false; break;
+ case 0xF0: if (a < 0x90) return false; break;
+ case 0xF4: if (a > 0x8F) return false; break;
+ default: if (a < 0x80) return false;
+ }
+ FALLTHROUGH;
+
+ case 1: if (*source >= 0x80 && *source < 0xC2) return false;
+ }
+ if (*source > 0xF4)
+ return false;
+ return true;
+}
+
+// Magic values subtracted from a buffer value during UTF8 conversion.
+// This table contains as many values as there might be trailing bytes
+// in a UTF-8 sequence.
+static const UChar32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 0x03C82080UL, static_cast<UChar32>(0xFA082080UL), static_cast<UChar32>(0x82082080UL) };
+
+static inline UChar32 readUTF8Sequence(const char*& sequence, unsigned length)
+{
+ UChar32 character = 0;
+
+ // The cases all fall through.
+ switch (length) {
+ case 6: character += static_cast<unsigned char>(*sequence++); character <<= 6; FALLTHROUGH;
+ case 5: character += static_cast<unsigned char>(*sequence++); character <<= 6; FALLTHROUGH;
+ case 4: character += static_cast<unsigned char>(*sequence++); character <<= 6; FALLTHROUGH;
+ case 3: character += static_cast<unsigned char>(*sequence++); character <<= 6; FALLTHROUGH;
+ case 2: character += static_cast<unsigned char>(*sequence++); character <<= 6; FALLTHROUGH;
+ case 1: character += static_cast<unsigned char>(*sequence++);
+ }
+
+ return character - offsetsFromUTF8[length - 1];
+}
+
+ConversionResult convertUTF8ToUTF16(
+ const char** sourceStart, const char* sourceEnd,
+ UChar** targetStart, UChar* targetEnd, bool* sourceAllASCII, bool strict)
+{
+ ConversionResult result = conversionOK;
+ const char* source = *sourceStart;
UChar* target = *targetStart;
- UChar32 orAllData = 0;
- unsigned targetOffset = 0;
- for (int sourceOffset = 0; sourceOffset < sourceEnd - source; ) {
- UChar32 character;
- U8_NEXT(reinterpret_cast<const uint8_t*>(source), sourceOffset, sourceEnd - source, character);
- if (character < 0)
- return false;
- U16_APPEND(target, targetOffset, targetEnd - target, character, error);
- if (error)
- return false;
- orAllData |= character;
+ UChar orAllData = 0;
+ while (source < sourceEnd) {
+ int utf8SequenceLength = inlineUTF8SequenceLength(*source);
+ if (sourceEnd - source < utf8SequenceLength) {
+ result = sourceExhausted;
+ break;
+ }
+ // Do this check whether lenient or strict
+ if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(source), utf8SequenceLength)) {
+ result = sourceIllegal;
+ break;
+ }
+
+ UChar32 character = readUTF8Sequence(source, utf8SequenceLength);
+
+ if (target >= targetEnd) {
+ source -= utf8SequenceLength; // Back up source pointer!
+ result = targetExhausted;
+ break;
+ }
+
+ if (U_IS_BMP(character)) {
+ // UTF-16 surrogate values are illegal in UTF-32
+ if (U_IS_SURROGATE(character)) {
+ if (strict) {
+ source -= utf8SequenceLength; // return to the illegal value itself
+ result = sourceIllegal;
+ break;
+ } else {
+ *target++ = replacementCharacter;
+ orAllData |= replacementCharacter;
+ }
+ } else {
+ *target++ = character; // normal case
+ orAllData |= character;
+ }
+ } else if (U_IS_SUPPLEMENTARY(character)) {
+ // target is a character in range 0xFFFF - 0x10FFFF
+ if (target + 1 >= targetEnd) {
+ source -= utf8SequenceLength; // Back up source pointer!
+ result = targetExhausted;
+ break;
+ }
+ *target++ = U16_LEAD(character);
+ *target++ = U16_TRAIL(character);
+ orAllData = 0xffff;
+ } else {
+ if (strict) {
+ source -= utf8SequenceLength; // return to the start
+ result = sourceIllegal;
+ break; // Bail out; shouldn't continue
+ } else {
+ *target++ = replacementCharacter;
+ orAllData |= replacementCharacter;
+ }
+ }
}
- *targetStart = target + targetOffset;
+ *sourceStart = source;
+ *targetStart = target;
+
if (sourceAllASCII)
- *sourceAllASCII = isASCII(orAllData);
- return true;
+ *sourceAllASCII = !(orAllData & ~0x7f);
+
+ return result;
}
unsigned calculateStringHashAndLengthFromUTF8MaskingTop8Bits(const char* data, const char* dataEnd, unsigned& dataLength, unsigned& utf16Length)
{
+ if (!data)
+ return 0;
+
StringHasher stringHasher;
+ dataLength = 0;
utf16Length = 0;
- int inputOffset = 0;
- int inputLength = dataEnd - data;
- while (inputOffset < inputLength) {
- UChar32 character;
- U8_NEXT(reinterpret_cast<const uint8_t*>(data), inputOffset, inputLength, character);
- if (character < 0)
+ while (data < dataEnd || (!dataEnd && *data)) {
+ if (isASCII(*data)) {
+ stringHasher.addCharacter(*data++);
+ dataLength++;
+ utf16Length++;
+ continue;
+ }
+
+ int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*data);
+ dataLength += utf8SequenceLength;
+
+ if (!dataEnd) {
+ for (int i = 1; i < utf8SequenceLength; ++i) {
+ if (!data[i])
+ return 0;
+ }
+ } else if (dataEnd - data < utf8SequenceLength)
return 0;
+ if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(data), utf8SequenceLength))
+ return 0;
+
+ UChar32 character = readUTF8Sequence(data, utf8SequenceLength);
+ ASSERT(!isASCII(character));
+
if (U_IS_BMP(character)) {
- ASSERT(!U_IS_SURROGATE(character));
- stringHasher.addCharacter(character);
+ // UTF-16 surrogate values are illegal in UTF-32
+ if (U_IS_SURROGATE(character))
+ return 0;
+ stringHasher.addCharacter(static_cast<UChar>(character)); // normal case
utf16Length++;
- } else {
- ASSERT(U_IS_SUPPLEMENTARY(character));
- stringHasher.addCharacters(U16_LEAD(character), U16_TRAIL(character));
+ } else if (U_IS_SUPPLEMENTARY(character)) {
+ stringHasher.addCharacters(static_cast<UChar>(U16_LEAD(character)),
+ static_cast<UChar>(U16_TRAIL(character)));
utf16Length += 2;
- }
+ } else
+ return 0;
}
- dataLength = inputOffset;
return stringHasher.hashWithTop8BitsMasked();
}
@@ -141,24 +423,36 @@
bool equalUTF16WithUTF8(const UChar* a, const char* b, const char* bEnd)
{
while (b < bEnd) {
- int offset = 0;
- UChar32 character;
- U8_NEXT(reinterpret_cast<const uint8_t*>(b), offset, bEnd - b, character);
- if (character < 0)
+ if (isASCII(*a) || isASCII(*b)) {
+ if (*a++ != *b++)
+ return false;
+ continue;
+ }
+
+ int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*b);
+
+ if (bEnd - b < utf8SequenceLength)
return false;
- b += offset;
+ if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(b), utf8SequenceLength))
+ return false;
+
+ UChar32 character = readUTF8Sequence(b, utf8SequenceLength);
+ ASSERT(!isASCII(character));
+
if (U_IS_BMP(character)) {
- ASSERT(!U_IS_SURROGATE(character));
+ // UTF-16 surrogate values are illegal in UTF-32
+ if (U_IS_SURROGATE(character))
+ return false;
if (*a++ != character)
return false;
- } else {
- ASSERT(U_IS_SUPPLEMENTARY(character));
+ } else if (U_IS_SUPPLEMENTARY(character)) {
if (*a++ != U16_LEAD(character))
return false;
if (*a++ != U16_TRAIL(character))
return false;
- }
+ } else
+ return false;
}
return true;
Modified: trunk/Source/WTF/wtf/unicode/UTF8Conversion.h (244826 => 244827)
--- trunk/Source/WTF/wtf/unicode/UTF8Conversion.h 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WTF/wtf/unicode/UTF8Conversion.h 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2007-2019 Apple Inc. All rights reserved.
+ * Copyright (C) 2007 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -31,28 +31,54 @@
namespace WTF {
namespace Unicode {
-enum ConversionResult {
- ConversionOK, // conversion successful
- SourceExhausted, // partial character in source, but hit end
- TargetExhausted, // insufficient room in target for conversion
- SourceIllegal // source sequence is illegal/malformed
-};
+ // Given a first byte, gives the length of the UTF-8 sequence it begins.
+ // Returns 0 for bytes that are not legal starts of UTF-8 sequences.
+ // Only allows sequences of up to 4 bytes, since that works for all Unicode characters (U-00000000 to U-0010FFFF).
+ WTF_EXPORT_PRIVATE int UTF8SequenceLength(char);
-// Conversion functions are strict, except for convertUTF16ToUTF8, which takes
-// "strict" argument. When strict, both illegal sequences and unpaired surrogates
-// will cause an error. When not, illegal sequences and unpaired surrogates are
-// converted to the replacement character, except for an unpaired lead surrogate
-// at the end of the source, which will instead cause a SourceExhausted error.
+ // Takes a null-terminated C-style string with a UTF-8 sequence in it and converts it to a character.
+ // Only allows Unicode characters (U-00000000 to U-0010FFFF).
+ // Returns -1 if the sequence is not valid (including presence of extra bytes).
+ WTF_EXPORT_PRIVATE int decodeUTF8Sequence(const char*);
-WTF_EXPORT_PRIVATE bool convertUTF8ToUTF16(const char* sourceStart, const char* sourceEnd, UChar** targetStart, UChar* targetEnd, bool* isSourceAllASCII = nullptr);
-WTF_EXPORT_PRIVATE bool convertLatin1ToUTF8(const LChar** sourceStart, const LChar* sourceEnd, char** targetStart, char* targetEnd);
-WTF_EXPORT_PRIVATE ConversionResult convertUTF16ToUTF8(const UChar** sourceStart, const UChar* sourceEnd, char** targetStart, char* targetEnd, bool strict = true);
+ typedef enum {
+ conversionOK, // conversion successful
+ sourceExhausted, // partial character in source, but hit end
+ targetExhausted, // insuff. room in target for conversion
+ sourceIllegal // source sequence is illegal/malformed
+ } ConversionResult;
-WTF_EXPORT_PRIVATE unsigned calculateStringHashAndLengthFromUTF8MaskingTop8Bits(const char* data, const char* dataEnd, unsigned& dataLength, unsigned& utf16Length);
+ // These conversion functions take a "strict" argument. When this
+ // flag is set to strict, both irregular sequences and isolated surrogates
+ // will cause an error. When the flag is set to lenient, both irregular
+ // sequences and isolated surrogates are converted.
+ //
+ // Whether the flag is strict or lenient, all illegal sequences will cause
+ // an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,
+ // or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code
+ // must check for illegal sequences.
+ //
+ // When the flag is set to lenient, characters over 0x10FFFF are converted
+ // to the replacement character; otherwise (when the flag is set to strict)
+ // they constitute an error.
-// Callers of these functions must check that the lengths are the same; accordingly we omit an end argument for UTF-16 and Latin-1.
-bool equalUTF16WithUTF8(const UChar* stringInUTF16, const char* stringInUTF8, const char* stringInUTF8End);
-bool equalLatin1WithUTF8(const LChar* stringInLatin1, const char* stringInUTF8, const char* stringInUTF8End);
+ WTF_EXPORT_PRIVATE ConversionResult convertUTF8ToUTF16(
+ const char** sourceStart, const char* sourceEnd,
+ UChar** targetStart, UChar* targetEnd, bool* isSourceAllASCII = 0, bool strict = true);
+ WTF_EXPORT_PRIVATE ConversionResult convertLatin1ToUTF8(
+ const LChar** sourceStart, const LChar* sourceEnd,
+ char** targetStart, char* targetEnd);
+
+ WTF_EXPORT_PRIVATE ConversionResult convertUTF16ToUTF8(
+ const UChar** sourceStart, const UChar* sourceEnd,
+ char** targetStart, char* targetEnd, bool strict = true);
+
+ WTF_EXPORT_PRIVATE unsigned calculateStringHashAndLengthFromUTF8MaskingTop8Bits(const char* data, const char* dataEnd, unsigned& dataLength, unsigned& utf16Length);
+
+ // The caller of these functions already knows that the lengths are the same, so we omit an end argument for UTF-16 and Latin-1.
+ bool equalUTF16WithUTF8(const UChar* stringInUTF16, const char* stringInUTF8, const char* stringInUTF8End);
+ bool equalLatin1WithUTF8(const LChar* stringInLatin1, const char* stringInUTF8, const char* stringInUTF8End);
+
} // namespace Unicode
} // namespace WTF
Modified: trunk/Source/WebCore/ChangeLog (244826 => 244827)
--- trunk/Source/WebCore/ChangeLog 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WebCore/ChangeLog 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,5 +1,18 @@
2019-05-01 Shawn Roberts <[email protected]>
+ Unreviewed, rolling out r244821.
+
+ Causing
+
+ Reverted changeset:
+
+ "WebKit has too much of its own UTF-8 code and should rely
+ more on ICU's UTF-8 support"
+ https://bugs.webkit.org/show_bug.cgi?id=195535
+ https://trac.webkit.org/changeset/244821
+
+2019-05-01 Shawn Roberts <[email protected]>
+
Unreviewed, rolling out r244822.
Causing
Modified: trunk/Source/WebCore/platform/SharedBuffer.cpp (244826 => 244827)
--- trunk/Source/WebCore/platform/SharedBuffer.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WebCore/platform/SharedBuffer.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -334,16 +334,17 @@
// Convert to runs of 8-bit characters.
char* p = buffer.data();
+ WTF::Unicode::ConversionResult result;
if (length) {
if (string.is8Bit()) {
const LChar* d = string.characters8();
- if (!WTF::Unicode::convertLatin1ToUTF8(&d, d + length, &p, p + buffer.size()))
- return nullptr;
+ result = WTF::Unicode::convertLatin1ToUTF8(&d, d + length, &p, p + buffer.size());
} else {
const UChar* d = string.characters16();
- if (WTF::Unicode::convertUTF16ToUTF8(&d, d + length, &p, p + buffer.size()) != WTF::Unicode::ConversionOK)
- return nullptr;
+ result = WTF::Unicode::convertUTF16ToUTF8(&d, d + length, &p, p + buffer.size(), true);
}
+ if (result != WTF::Unicode::conversionOK)
+ return nullptr;
}
buffer.shrink(p - buffer.data());
Modified: trunk/Source/WebCore/xml/XSLTProcessorLibxslt.cpp (244826 => 244827)
--- trunk/Source/WebCore/xml/XSLTProcessorLibxslt.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WebCore/xml/XSLTProcessorLibxslt.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -48,6 +48,8 @@
#include <libxslt/xslt.h>
#include <libxslt/xsltutils.h>
#include <wtf/Assertions.h>
+#include <wtf/text/StringBuffer.h>
+#include <wtf/unicode/UTF8Conversion.h>
#if OS(DARWIN) && !PLATFORM(GTK)
#include "SoftLinkLibxslt.h"
@@ -157,41 +159,27 @@
globalCachedResourceLoader = cachedResourceLoader;
}
-static int writeToStringBuilder(void* context, const char* buffer, int length)
+static int writeToStringBuilder(void* context, const char* buffer, int len)
{
StringBuilder& resultOutput = *static_cast<StringBuilder*>(context);
- // FIXME: Consider ways to make this more efficient by moving it into a
- // StringBuilder::appendUTF8 function, and then optimizing to not need a
- // Vector<UChar> and possibly optimize cases that can produce 8-bit Latin-1
- // strings, but that would need to be sophisticated about not processing
- // trailing incomplete sequences and communicating that to the caller.
+ if (!len)
+ return 0;
- Vector<UChar> outputBuffer(length);
+ StringBuffer<UChar> stringBuffer(len);
+ UChar* bufferUChar = stringBuffer.characters();
+ UChar* bufferUCharEnd = bufferUChar + len;
- UBool error = false;
- int inputOffset = 0;
- int outputOffset = 0;
- while (inputOffset < length) {
- UChar32 character;
- int nextInputOffset = inputOffset;
- U8_NEXT(reinterpret_cast<const uint8_t*>(buffer), nextInputOffset, length, character);
- if (character < 0) {
- if (nextInputOffset == length)
- break;
- ASSERT_NOT_REACHED();
- return -1;
- }
- inputOffset = nextInputOffset;
- U16_APPEND(outputBuffer.data(), outputOffset, length, character, error);
- if (error) {
- ASSERT_NOT_REACHED();
- return -1;
- }
+ const char* stringCurrent = buffer;
+ WTF::Unicode::ConversionResult result = WTF::Unicode::convertUTF8ToUTF16(&stringCurrent, buffer + len, &bufferUChar, bufferUCharEnd);
+ if (result != WTF::Unicode::conversionOK && result != WTF::Unicode::sourceExhausted) {
+ ASSERT_NOT_REACHED();
+ return -1;
}
- resultOutput.append(outputBuffer.data(), outputOffset);
- return inputOffset;
+ int utf16Length = bufferUChar - stringBuffer.characters();
+ resultOutput.append(stringBuffer.characters(), utf16Length);
+ return stringCurrent - buffer;
}
static bool saveResultToString(xmlDocPtr resultDoc, xsltStylesheetPtr sheet, String& resultString)
Modified: trunk/Source/WebCore/xml/parser/XMLDocumentParserLibxml2.cpp (244826 => 244827)
--- trunk/Source/WebCore/xml/parser/XMLDocumentParserLibxml2.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WebCore/xml/parser/XMLDocumentParserLibxml2.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1153,8 +1153,8 @@
static size_t convertUTF16EntityToUTF8(const UChar* utf16Entity, size_t numberOfCodeUnits, char* target, size_t targetSize)
{
const char* originalTarget = target;
- WTF::Unicode::ConversionResult conversionResult = WTF::Unicode::convertUTF16ToUTF8(&utf16Entity, utf16Entity + numberOfCodeUnits, &target, target + targetSize);
- if (conversionResult != WTF::Unicode::ConversionOK)
+ auto conversionResult = WTF::Unicode::convertUTF16ToUTF8(&utf16Entity, utf16Entity + numberOfCodeUnits, &target, target + targetSize);
+ if (conversionResult != WTF::Unicode::conversionOK)
return 0;
// Even though we must pass the length, libxml expects the entity string to be null terminated.
Modified: trunk/Source/WebKit/ChangeLog (244826 => 244827)
--- trunk/Source/WebKit/ChangeLog 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WebKit/ChangeLog 2019-05-01 17:13:58 UTC (rev 244827)
@@ -1,3 +1,16 @@
+2019-05-01 Shawn Roberts <[email protected]>
+
+ Unreviewed, rolling out r244821.
+
+ Causing
+
+ Reverted changeset:
+
+ "WebKit has too much of its own UTF-8 code and should rely
+ more on ICU's UTF-8 support"
+ https://bugs.webkit.org/show_bug.cgi?id=195535
+ https://trac.webkit.org/changeset/244821
+
2019-05-01 Youenn Fablet <[email protected]>
Kept alive loaders should use the redirected request in case of redirections
Modified: trunk/Source/WebKit/Shared/API/APIString.h (244826 => 244827)
--- trunk/Source/WebKit/Shared/API/APIString.h 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WebKit/Shared/API/APIString.h 2019-05-01 17:13:58 UTC (rev 244827)
@@ -23,10 +23,14 @@
* THE POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef APIString_h
+#define APIString_h
#include "APIObject.h"
+#include <wtf/Ref.h>
#include <wtf/text/StringView.h>
+#include <wtf/text/WTFString.h>
+#include <wtf/unicode/UTF8Conversion.h>
namespace API {
@@ -71,3 +75,5 @@
};
} // namespace WebKit
+
+#endif // APIString_h
Modified: trunk/Source/WebKit/Shared/API/c/WKString.cpp (244826 => 244827)
--- trunk/Source/WebKit/Shared/API/c/WKString.cpp 2019-05-01 17:12:19 UTC (rev 244826)
+++ trunk/Source/WebKit/Shared/API/c/WKString.cpp 2019-05-01 17:13:58 UTC (rev 244827)
@@ -30,7 +30,6 @@
#include "WKAPICast.h"
#include <_javascript_Core/InitializeThreading.h>
#include <_javascript_Core/OpaqueJSString.h>
-#include <wtf/unicode/UTF8Conversion.h>
WKTypeID WKStringGetTypeID()
{
@@ -79,18 +78,19 @@
auto stringView = WebKit::toImpl(stringRef)->stringView();
char* p = buffer;
+ WTF::Unicode::ConversionResult result;
if (stringView.is8Bit()) {
const LChar* characters = stringView.characters8();
- if (!WTF::Unicode::convertLatin1ToUTF8(&characters, characters + stringView.length(), &p, p + bufferSize - 1))
- return 0;
+ result = WTF::Unicode::convertLatin1ToUTF8(&characters, characters + stringView.length(), &p, p + bufferSize - 1);
} else {
const UChar* characters = stringView.characters16();
- WTF::Unicode::ConversionResult result = WTF::Unicode::convertUTF16ToUTF8(&characters, characters + stringView.length(), &p, p + bufferSize - 1, strict);
- if (result != WTF::Unicode::ConversionOK && result != WTF::Unicode::TargetExhausted)
- return 0;
+ result = WTF::Unicode::convertUTF16ToUTF8(&characters, characters + stringView.length(), &p, p + bufferSize - 1, strict);
}
+ if (result != WTF::Unicode::conversionOK && result != WTF::Unicode::targetExhausted)
+ return 0;
+
*p++ = '\0';
return p - buffer;
}
