chaokunyang commented on code in PR #1890:
URL: https://github.com/apache/fury/pull/1890#discussion_r1805960170
##########
java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java:
##########
@@ -625,4 +743,534 @@ public String readUTF8String(MemoryBuffer buffer) {
return new String(tmpArray, 0, numBytes, StandardCharsets.UTF_8);
}
}
+
+ public byte[] readBytesLatin1(MemoryBuffer buffer) {
+ int utf8AsciiBytes = buffer.readInt32();
+ buffer.checkReadableBytes(utf8AsciiBytes);
+ byte[] srcArray = buffer.getHeapMemory();
+ byte[] bytes = new byte[utf8AsciiBytes << 1];
+ if (srcArray != null) {
+ int srcIndex = buffer._unsafeHeapReaderIndex();
+ for (int i = 0, pos = Platform.IS_LITTLE_ENDIAN ? 0 : 1; i <
utf8AsciiBytes; ++i, pos += 2) {
+ bytes[pos] = srcArray[srcIndex++];
+ }
+ buffer._increaseReaderIndexUnsafe(utf8AsciiBytes);
+ } else {
+ // TODO: off-heap
+ }
+ return bytes;
+ }
+
+ public char[] readCharsLatin1(MemoryBuffer buffer) {
+ int utf8AsciiBytes = buffer.readInt32();
+ buffer.checkReadableBytes(utf8AsciiBytes);
+ byte[] srcArray = buffer.getHeapMemory();
+ char[] chars = new char[utf8AsciiBytes];
+ if (srcArray != null) {
+ int srcIndex = buffer._unsafeHeapReaderIndex();
+ for (int i = 0; i < utf8AsciiBytes; i++) {
+ chars[i] = (char) (srcArray[srcIndex++] & 0xff);
+ }
+ buffer._increaseReaderIndexUnsafe(utf8AsciiBytes);
+ } else {
+ // TODO: off-heap
+ }
+ return chars;
+ }
+
+ public byte[] readBytesUTF8(MemoryBuffer buffer) {
+ int utf16Bytes = buffer.readInt32();
+ int udf8Bytes = buffer.readInt32();
+ byte[] bytes = new byte[utf16Bytes];
+ buffer.checkReadableBytes(udf8Bytes);
+ byte[] srcArray = buffer.getHeapMemory();
+ if (srcArray != null) {
+ int srcIndex = buffer._unsafeHeapReaderIndex();
+ if (!fastDecodeUTF8(srcArray, srcIndex, udf8Bytes, bytes)) {
+ throw new RuntimeException("Decode failed");
+ }
+ buffer._increaseReaderIndexUnsafe(udf8Bytes);
+ } else {
+ // TODO: off-heap
+ }
+ return bytes;
+ }
+
+ public byte[] readBytesUTF16(MemoryBuffer buffer) {
+ int utf16Bytes = buffer.readInt32();
+ buffer.checkReadableBytes(utf16Bytes);
+ byte[] bytes;
+ byte[] heapMemory = buffer.getHeapMemory();
+ if (heapMemory != null) {
+ final int arrIndex = buffer._unsafeHeapReaderIndex();
+ buffer.increaseReaderIndex(utf16Bytes);
+ bytes = new byte[utf16Bytes];
+ System.arraycopy(heapMemory, arrIndex, bytes, 0, utf16Bytes);
+ } else {
+ bytes = buffer.readBytes(utf16Bytes);
+ }
+ return bytes;
+ }
+
+ public char[] readCharsUTF16(MemoryBuffer buffer) {
+ int utf16Bytes = buffer.readInt32();
+ char[] chars = new char[utf16Bytes >> 1];
+ if (Platform.IS_LITTLE_ENDIAN) {
+ // FIXME JDK11 utf16 string uses little-endian order.
+ buffer.readChars(chars, Platform.CHAR_ARRAY_OFFSET, utf16Bytes);
+ } else {
+ buffer.checkReadableBytes(utf16Bytes);
+ final byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int charIndex = 0;
+ for (int i = buffer._unsafeHeapReaderIndex(), end = i + utf16Bytes; i
< end; i += 2) {
+ char c =
+ (char)
+ ((targetArray[i] & 0xff << StringUTF16.HI_BYTE_SHIFT)
+ | ((targetArray[i + 1] & 0xff) <<
StringUTF16.LO_BYTE_SHIFT));
+ chars[charIndex++] = c;
+ }
+ buffer._increaseReaderIndexUnsafe(utf16Bytes);
+ } else {
+ final byte[] tmpArray = getByteArray(utf16Bytes);
+ buffer.readBytes(tmpArray, 0, utf16Bytes);
+ int charIndex = 0;
+ for (int i = 0; i < utf16Bytes; i += 2) {
+ char c =
+ (char)
+ ((tmpArray[i] & 0xff << StringUTF16.HI_BYTE_SHIFT)
+ | ((tmpArray[i + 1] & 0xff) <<
StringUTF16.LO_BYTE_SHIFT));
+ chars[charIndex++] = c;
+ }
+ }
+ }
+ return chars;
+ }
+
+ public char[] readCharsUTF8(MemoryBuffer buffer) {
+ int utf16Bytes = buffer.readInt32();
+ int udf8Bytes = buffer.readInt32();
+ char[] chars = new char[utf16Bytes >> 1];
+ buffer.checkReadableBytes(udf8Bytes);
+ byte[] srcArray = buffer.getHeapMemory();
+ if (srcArray != null) {
+ int srcIndex = buffer._unsafeHeapReaderIndex();
+ if (!fastDecodeUTF8(srcArray, srcIndex, udf8Bytes, chars)) {
+ throw new RuntimeException("Decode failed");
+ }
+ buffer._increaseReaderIndexUnsafe(udf8Bytes);
+ } else {
+ // TODO: off-heap
+ }
+ return chars;
+ }
+
+ public void writeBytesLatin1(MemoryBuffer buffer, byte[] bytes) {
+ int writerIndex = buffer.writerIndex();
+ int numBytes = bytes.length >> 1;
+ buffer.ensure(writerIndex + 5 + numBytes);
+ byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ buffer.putByte(arrIndex, LATIN1);
+ buffer.putInt32(arrIndex + 1, numBytes);
+ arrIndex += 5;
+ for (int i = Platform.IS_LITTLE_ENDIAN ? 0 : 1; i < numBytes; i += 2) {
+ targetArray[arrIndex++] = bytes[i];
+ }
+ writerIndex += 5;
+ } else {
+ // TODO: off-heap
+ }
+ writerIndex += numBytes;
+ buffer._unsafeWriterIndex(writerIndex);
+ }
+
+ public void writeCharsLatin1(MemoryBuffer buffer, char[] chars) {
+ int writerIndex = buffer.writerIndex();
+ int numBytes = chars.length;
+ buffer.ensure(writerIndex + 5 + numBytes);
+ byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ buffer.putByte(arrIndex, LATIN1);
+ buffer.putInt32(arrIndex + 1, numBytes);
+ arrIndex += 5;
+ for (int i = 0; i < numBytes; i++) {
+ targetArray[arrIndex + i] = (byte) chars[i];
+ }
+ writerIndex += 5;
+ } else {
+ // TODO: off-heap
+ }
+ writerIndex += numBytes;
+ buffer._unsafeWriterIndex(writerIndex);
+ }
+
+ public void writeBytesUTF16(MemoryBuffer buffer, byte[] bytes) {
+ int numBytes = bytes.length;
+ int writerIndex = buffer.writerIndex();
+ buffer.ensure(writerIndex + 5 + numBytes);
+ final byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ buffer.putByte(arrIndex, UTF16);
+ buffer.putInt32(arrIndex + 1, numBytes);
+ arrIndex += 5;
+ writerIndex += 5 + numBytes;
+ System.arraycopy(bytes, 0, targetArray, arrIndex, numBytes);
+ } else {
+ // TODO: off-heap
+ }
+ buffer._unsafeWriterIndex(writerIndex);
+ }
+
+ public void writeCharsUTF16(MemoryBuffer buffer, char[] chars) {
+ int numBytes = MathUtils.doubleExact(chars.length);
+ int writerIndex = buffer.writerIndex();
+ buffer.ensure(writerIndex + 5 + numBytes);
+ final byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ buffer.putByte(arrIndex, UTF16);
+ buffer.putInt32(arrIndex + 1, numBytes);
+ arrIndex += 5;
+ writerIndex += 5 + numBytes;
+ if (Platform.IS_LITTLE_ENDIAN) {
+ // FIXME JDK11 utf16 string uses little-endian order.
+ Platform.UNSAFE.copyMemory(
+ chars,
+ Platform.CHAR_ARRAY_OFFSET,
+ targetArray,
+ Platform.BYTE_ARRAY_OFFSET + arrIndex,
+ numBytes);
+ } else {
+ heapWriteCharsUTF16BE(chars, arrIndex, numBytes, targetArray);
+ }
+ } else {
+ // TODO: off-heap
+ }
+ buffer._unsafeWriterIndex(writerIndex);
+ }
+
+ public void writeCharsUTF8(MemoryBuffer buffer, char[] chars) {
+ int estimateMaxBytes = chars.length * 3;
+ int writerIndex = buffer.writerIndex();
+ buffer.ensure(writerIndex + 9 + estimateMaxBytes);
+ byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ int targetIndex = fastEncodeUTF8(chars, targetArray, arrIndex + 9);
+ int written = targetIndex - arrIndex - 9;
+ buffer.putByte(arrIndex, UTF8);
+ buffer.putInt32(arrIndex + 1, chars.length << 1);
+ buffer.putInt32(arrIndex + 5, written);
+ buffer._unsafeWriterIndex(targetIndex);
+ } else {
+ // TODO: off-heap
+ }
+ }
+
+ public void writeBytesUTF8(MemoryBuffer buffer, byte[] bytes) {
+ int estimateMaxBytes = bytes.length / 2 * 3;
+ int writerIndex = buffer.writerIndex();
+ buffer.ensure(writerIndex + 9 + estimateMaxBytes);
+ byte[] targetArray = buffer.getHeapMemory();
+ if (targetArray != null) {
+ int arrIndex = buffer._unsafeHeapWriterIndex();
+ int targetIndex = fastEncodeUTF8(bytes, targetArray, arrIndex + 9);
+ int written = targetIndex - arrIndex - 9;
+ buffer.putByte(arrIndex, UTF8);
+ buffer.putInt32(arrIndex + 1, bytes.length);
+ buffer.putInt32(arrIndex + 5, written);
+ buffer._unsafeWriterIndex(targetIndex);
+ }
+ }
+
+ private static boolean fastDecodeUTF8(byte[] src, int offset, int len,
byte[] dst) {
+ final int end = offset + len;
+ int dp = 0;
+
+ while (offset < end) {
+ if (offset + 8 <= end
+ && (Platform.getLong(src, Platform.BYTE_ARRAY_OFFSET + offset) &
0x8080808080808080L)
+ == 0) {
+ // ascii only
+ for (int i = 0, pos = Platform.IS_LITTLE_ENDIAN ? dp : dp + 1; i < 8;
++i, pos += 2) {
Review Comment:
Could we try to unroll this loop into 8 lines of code and test whether will
it be faster
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
