chaokunyang commented on code in PR #1890:
URL: https://github.com/apache/fury/pull/1890#discussion_r1805960612
##########
java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java:
##########
@@ -625,4 +743,534 @@ public String readUTF8String(MemoryBuffer buffer) {
return new String(tmpArray, 0, numBytes, StandardCharsets.UTF_8);
}
}
+
+ public byte[] readBytesLatin1(MemoryBuffer buffer) {
+ int utf8AsciiBytes = buffer.readInt32();
+ buffer.checkReadableBytes(utf8AsciiBytes);
+ byte[] srcArray = buffer.getHeapMemory();
+ byte[] bytes = new byte[utf8AsciiBytes << 1];
+ if (srcArray != null) {
+ int srcIndex = buffer._unsafeHeapReaderIndex();
+ for (int i = 0, pos = Platform.IS_LITTLE_ENDIAN ? 0 : 1; i <
utf8AsciiBytes; ++i, pos += 2) {
+ bytes[pos] = srcArray[srcIndex++];
+ }
+ buffer._increaseReaderIndexUnsafe(utf8AsciiBytes);
+ } else {
+ // TODO: off-heap
+ }
+ return bytes;
+ }
+
+ public char[] readCharsLatin1(MemoryBuffer buffer) {
+ int utf8AsciiBytes = buffer.readInt32();
+ buffer.checkReadableBytes(utf8AsciiBytes);
+ byte[] srcArray = buffer.getHeapMemory();
+ char[] chars = new char[utf8AsciiBytes];
+ if (srcArray != null) {
+ int srcIndex = buffer._unsafeHeapReaderIndex();
+ for (int i = 0; i < utf8AsciiBytes; i++) {
+ chars[i] = (char) (srcArray[srcIndex++] & 0xff);
+ }
+ buffer._increaseReaderIndexUnsafe(utf8AsciiBytes);
+ } else {
+ // TODO: off-heap
+ }
+ return chars;
+ }
+
+ public byte[] readBytesUTF8(MemoryBuffer buffer) {
+ int utf16Bytes = buffer.readInt32();
+ int udf8Bytes = buffer.readInt32();
+ byte[] bytes = new byte[utf16Bytes];
+ buffer.checkReadableBytes(udf8Bytes);
+ byte[] srcArray = buffer.getHeapMemory();
+ if (srcArray != null) {
+ int srcIndex = buffer._unsafeHeapReaderIndex();
+ if (!fastDecodeUTF8(srcArray, srcIndex, udf8Bytes, bytes)) {
+ throw new RuntimeException("Decode failed");
+ }
+ buffer._increaseReaderIndexUnsafe(udf8Bytes);
+ } else {
+ // TODO: off-heap
+ }
+ return bytes;
+ }
+
+ public byte[] readBytesUTF16(MemoryBuffer buffer) {
+ int utf16Bytes = buffer.readInt32();
+ buffer.checkReadableBytes(utf16Bytes);
+ byte[] bytes;
+ byte[] heapMemory = buffer.getHeapMemory();
+ if (heapMemory != null) {
+ final int arrIndex = buffer._unsafeHeapReaderIndex();
+ buffer.increaseReaderIndex(utf16Bytes);
+ bytes = new byte[utf16Bytes];
+ System.arraycopy(heapMemory, arrIndex, bytes, 0, utf16Bytes);
+ } else {
+ bytes = buffer.readBytes(utf16Bytes);
+ }
+ return bytes;
+ }
+
+ public char[] readCharsUTF16(MemoryBuffer buffer) {
+ int utf16Bytes = buffer.readInt32();
+ char[] chars = new char[utf16Bytes >> 1];
+ if (Platform.IS_LITTLE_ENDIAN) {
+ // FIXME JDK11 utf16 string uses little-endian order.
+ buffer.readChars(chars, Platform.CHAR_ARRAY_OFFSET, utf16Bytes);
+ } else {
+ buffer.checkReadableBytes(utf16Bytes);
+ final byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int charIndex = 0;
+ for (int i = buffer._unsafeHeapReaderIndex(), end = i + utf16Bytes; i
< end; i += 2) {
+ char c =
+ (char)
+ ((targetArray[i] & 0xff << StringUTF16.HI_BYTE_SHIFT)
+ | ((targetArray[i + 1] & 0xff) <<
StringUTF16.LO_BYTE_SHIFT));
+ chars[charIndex++] = c;
+ }
+ buffer._increaseReaderIndexUnsafe(utf16Bytes);
+ } else {
+ final byte[] tmpArray = getByteArray(utf16Bytes);
+ buffer.readBytes(tmpArray, 0, utf16Bytes);
+ int charIndex = 0;
+ for (int i = 0; i < utf16Bytes; i += 2) {
+ char c =
+ (char)
+ ((tmpArray[i] & 0xff << StringUTF16.HI_BYTE_SHIFT)
+ | ((tmpArray[i + 1] & 0xff) <<
StringUTF16.LO_BYTE_SHIFT));
+ chars[charIndex++] = c;
+ }
+ }
+ }
+ return chars;
+ }
+
+ public char[] readCharsUTF8(MemoryBuffer buffer) {
+ int utf16Bytes = buffer.readInt32();
+ int udf8Bytes = buffer.readInt32();
+ char[] chars = new char[utf16Bytes >> 1];
+ buffer.checkReadableBytes(udf8Bytes);
+ byte[] srcArray = buffer.getHeapMemory();
+ if (srcArray != null) {
+ int srcIndex = buffer._unsafeHeapReaderIndex();
+ if (!fastDecodeUTF8(srcArray, srcIndex, udf8Bytes, chars)) {
+ throw new RuntimeException("Decode failed");
+ }
+ buffer._increaseReaderIndexUnsafe(udf8Bytes);
+ } else {
+ // TODO: off-heap
+ }
+ return chars;
+ }
+
+ public void writeBytesLatin1(MemoryBuffer buffer, byte[] bytes) {
+ int writerIndex = buffer.writerIndex();
+ int numBytes = bytes.length >> 1;
+ buffer.ensure(writerIndex + 5 + numBytes);
+ byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ buffer.putByte(arrIndex, LATIN1);
+ buffer.putInt32(arrIndex + 1, numBytes);
+ arrIndex += 5;
+ for (int i = Platform.IS_LITTLE_ENDIAN ? 0 : 1; i < numBytes; i += 2) {
+ targetArray[arrIndex++] = bytes[i];
+ }
+ writerIndex += 5;
+ } else {
+ // TODO: off-heap
+ }
+ writerIndex += numBytes;
+ buffer._unsafeWriterIndex(writerIndex);
+ }
+
+ public void writeCharsLatin1(MemoryBuffer buffer, char[] chars) {
+ int writerIndex = buffer.writerIndex();
+ int numBytes = chars.length;
+ buffer.ensure(writerIndex + 5 + numBytes);
+ byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ buffer.putByte(arrIndex, LATIN1);
+ buffer.putInt32(arrIndex + 1, numBytes);
+ arrIndex += 5;
+ for (int i = 0; i < numBytes; i++) {
+ targetArray[arrIndex + i] = (byte) chars[i];
+ }
+ writerIndex += 5;
+ } else {
+ // TODO: off-heap
+ }
+ writerIndex += numBytes;
+ buffer._unsafeWriterIndex(writerIndex);
+ }
+
+ public void writeBytesUTF16(MemoryBuffer buffer, byte[] bytes) {
+ int numBytes = bytes.length;
+ int writerIndex = buffer.writerIndex();
+ buffer.ensure(writerIndex + 5 + numBytes);
+ final byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ buffer.putByte(arrIndex, UTF16);
+ buffer.putInt32(arrIndex + 1, numBytes);
+ arrIndex += 5;
+ writerIndex += 5 + numBytes;
+ System.arraycopy(bytes, 0, targetArray, arrIndex, numBytes);
+ } else {
+ // TODO: off-heap
+ }
+ buffer._unsafeWriterIndex(writerIndex);
+ }
+
+ public void writeCharsUTF16(MemoryBuffer buffer, char[] chars) {
+ int numBytes = MathUtils.doubleExact(chars.length);
+ int writerIndex = buffer.writerIndex();
+ buffer.ensure(writerIndex + 5 + numBytes);
+ final byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ buffer.putByte(arrIndex, UTF16);
+ buffer.putInt32(arrIndex + 1, numBytes);
+ arrIndex += 5;
+ writerIndex += 5 + numBytes;
+ if (Platform.IS_LITTLE_ENDIAN) {
+ // FIXME JDK11 utf16 string uses little-endian order.
+ Platform.UNSAFE.copyMemory(
+ chars,
+ Platform.CHAR_ARRAY_OFFSET,
+ targetArray,
+ Platform.BYTE_ARRAY_OFFSET + arrIndex,
+ numBytes);
+ } else {
+ heapWriteCharsUTF16BE(chars, arrIndex, numBytes, targetArray);
+ }
+ } else {
+ // TODO: off-heap
+ }
+ buffer._unsafeWriterIndex(writerIndex);
+ }
+
+ public void writeCharsUTF8(MemoryBuffer buffer, char[] chars) {
+ int estimateMaxBytes = chars.length * 3;
+ int writerIndex = buffer.writerIndex();
+ buffer.ensure(writerIndex + 9 + estimateMaxBytes);
+ byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ int targetIndex = fastEncodeUTF8(chars, targetArray, arrIndex + 9);
+ int written = targetIndex - arrIndex - 9;
+ buffer.putByte(arrIndex, UTF8);
+ buffer.putInt32(arrIndex + 1, chars.length << 1);
+ buffer.putInt32(arrIndex + 5, written);
+ buffer._unsafeWriterIndex(targetIndex);
+ } else {
+ // TODO: off-heap
+ }
+ }
+
+ public void writeBytesUTF8(MemoryBuffer buffer, byte[] bytes) {
+ int estimateMaxBytes = bytes.length / 2 * 3;
+ int writerIndex = buffer.writerIndex();
+ buffer.ensure(writerIndex + 9 + estimateMaxBytes);
+ byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ int targetIndex = fastEncodeUTF8(bytes, targetArray, arrIndex + 9);
+ int written = targetIndex - arrIndex - 9;
+ buffer.putByte(arrIndex, UTF8);
+ buffer.putInt32(arrIndex + 1, bytes.length);
+ buffer.putInt32(arrIndex + 5, written);
+ buffer._unsafeWriterIndex(targetIndex);
+ }
+ }
+
+ private static boolean fastDecodeUTF8(byte[] src, int offset, int len,
byte[] dst) {
+ final int end = offset + len;
+ int dp = 0;
+
+ while (offset < end) {
+ if (offset + 8 <= end
+ && (Platform.getLong(src, Platform.BYTE_ARRAY_OFFSET + offset) &
0x8080808080808080L)
+ == 0) {
+ // ascii only
+ for (int i = 0, pos = Platform.IS_LITTLE_ENDIAN ? dp : dp + 1; i < 8;
++i, pos += 2) {
+ dst[pos] = src[offset++];
+ }
+ dp += 16;
+ } else {
+ int b0 = src[offset++];
+ if (b0 >= 0) {
+ // 1 byte, 7 bits: 0xxxxxxx
+ dst[dp] = (byte) b0;
+ dst[dp + 1] = 0;
+ dp += 2;
+ } else if ((b0 >> 5) == -2 && (b0 & 0x1e) != 0) {
+ // 2 bytes, 11 bits: 110xxxxx 10xxxxxx
+ if (offset >= end) {
+ return false;
+ }
+ int b1 = src[offset++];
+ if ((b1 & 0xc0) != 0x80) { // isNotContinuation(b2)
+ return false;
+ } else {
+ char c = (char) (((b0 << 6) ^ b1) ^ (((byte) 0xC0 << 6) ^ ((byte)
0x80)));
+ dst[dp] = (byte) c;
+ dst[dp + 1] = (byte) (c >> 8);
+ dp += 2;
+ }
+ } else if ((b0 >> 4) == -2) {
+ // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx
+ if (offset + 1 >= end) {
+ return false;
+ }
+ int b1 = src[offset];
+ int b2 = src[offset + 1];
+ offset += 2;
+ if ((b0 == (byte) 0xe0 && (b1 & 0xe0) == 0x80) //
+ || (b1 & 0xc0) != 0x80 //
+ || (b2 & 0xc0) != 0x80) { // isMalformed3(b0, b1, b2)
+ return false;
+ } else {
+ char c =
+ (char)
+ ((b0 << 12)
+ ^ (b1 << 6)
+ ^ (b2 ^ (((byte) 0xE0 << 12) ^ ((byte) 0x80 << 6) ^
((byte) 0x80))));
+ boolean isSurrogate = c >= '\uD800' && c < ('\uDFFF' + 1);
+ if (isSurrogate) {
+ return false;
+ } else {
+ dst[dp] = (byte) c;
+ dst[dp + 1] = (byte) (c >> 8);
+ dp += 2;
+ }
+ }
+ } else if ((b0 >> 3) == -2) {
+ // 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+ if (offset + 2 >= end) {
+ return false;
+ }
+ int b2 = src[offset];
+ int b3 = src[offset + 1];
+ int b4 = src[offset + 2];
+ offset += 3;
+ int uc =
+ ((b0 << 18)
+ ^ (b2 << 12)
+ ^ (b3 << 6)
+ ^ (b4
+ ^ (((byte) 0xF0 << 18)
+ ^ ((byte) 0x80 << 12)
+ ^ ((byte) 0x80 << 6)
+ ^ ((byte) 0x80))));
+ if (((b2 & 0xc0) != 0x80 || (b3 & 0xc0) != 0x80 || (b4 & 0xc0) !=
0x80) // isMalformed4
+ ||
+ // shortest form check
+ !(uc >= 0x010000 && uc < 0X10FFFF + 1) //
!Character.isSupplementaryCodePoint(uc)
+ ) {
+ return false;
+ } else {
+ char c = (char) ((uc >>> 10) + ('\uD800' - (0x010000 >>> 10)));
+ dst[dp] = (byte) c;
+ dst[dp + 1] = (byte) (c >> 8);
+ dp += 2;
+
+ c = (char) ((uc & 0x3ff) + '\uDC00');
+ dst[dp] = (byte) c;
+ dst[dp + 1] = (byte) (c >> 8);
+ dp += 2;
+ }
+ } else {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ private static boolean fastDecodeUTF8(byte[] src, int offset, int len,
char[] dst) {
+ int end = offset + len;
+ int dp = 0;
+ while (offset < end) {
+ if (offset + 8 <= end
+ && (Platform.getLong(src, Platform.BYTE_ARRAY_OFFSET + offset) &
0x8080808080808080L)
+ == 0) {
+ // ascii only
+ for (int i = 0; i < 8; ++i) {
Review Comment:
ditto for loop unroll
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
