chaokunyang commented on code in PR #1890:
URL: https://github.com/apache/fury/pull/1890#discussion_r1808630319
##########
java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java:
##########
@@ -603,26 +722,121 @@ private static MethodHandle
getJavaStringZeroCopyCtrHandle() {
}
}
- public void writeUTF8String(MemoryBuffer buffer, String value) {
- byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
- buffer.writeVarUint32(bytes.length);
- buffer.writeBytes(bytes);
+ private static void heapWriteCharsUTF16BE(
+ char[] chars, int arrIndex, int numBytes, byte[] targetArray) {
+ // Write to heap memory then copy is 250% faster than unsafe write to
direct memory.
+ int charIndex = 0;
+ for (int i = arrIndex, end = i + numBytes; i < end; i += 2) {
+ char c = chars[charIndex++];
+ targetArray[i] = (byte) (c >> StringUTF16.HI_BYTE_SHIFT);
+ targetArray[i + 1] = (byte) (c >> StringUTF16.LO_BYTE_SHIFT);
+ }
}
- public String readUTF8String(MemoryBuffer buffer) {
- int numBytes = buffer.readVarUint32Small14();
- buffer.checkReadableBytes(numBytes);
- final byte[] targetArray = buffer.getHeapMemory();
- if (targetArray != null) {
- String str =
- new String(
- targetArray, buffer._unsafeHeapReaderIndex(), numBytes,
StandardCharsets.UTF_8);
- buffer.increaseReaderIndex(numBytes);
- return str;
+ private int offHeapWriteCharsUTF16(
+ MemoryBuffer buffer, char[] chars, int writerIndex, int numBytes) {
+ byte[] tmpArray = getByteArray(numBytes);
+ int charIndex = 0;
+ for (int i = 0; i < numBytes; i += 2) {
+ char c = chars[charIndex++];
+ tmpArray[i] = (byte) (c >> StringUTF16.HI_BYTE_SHIFT);
+ tmpArray[i + 1] = (byte) (c >> StringUTF16.LO_BYTE_SHIFT);
+ }
+ buffer.put(writerIndex, tmpArray, 0, numBytes);
+ writerIndex += numBytes;
+ return writerIndex;
+ }
+
+ private static byte bestCoder(char[] chars) {
+ int numChars = chars.length;
+ // sample 64 chars
+ int sampleNum = Math.min(64, numChars);
+ int vectorizedLen = sampleNum >> 2;
+ int vectorizedChars = vectorizedLen << 2;
+ int endOffset = Platform.CHAR_ARRAY_OFFSET + (vectorizedChars << 1);
+ int count = 0;
+ for (int offset = Platform.CHAR_ARRAY_OFFSET, charOffset = 0;
+ offset < endOffset;
+ offset += 8, charOffset += 4) {
+ long multiChars = Platform.getLong(chars, offset);
+ if ((multiChars & MULTI_CHARS_NON_LATIN_MASK) == 0) {
+ count += 4;
+ } else {
+ for (int i = 0; i < 4; ++i) {
+ if (chars[charOffset + i] < 0x80) {
+ count++;
+ }
+ }
+ }
+ }
+
+ for (int i = vectorizedChars; i < sampleNum; i++) {
+ if (chars[i] < 0x80) {
+ count++;
+ }
+ }
+
+ // ascii number > 50%, choose UTF-8
+ if (count >= sampleNum * 0.5) {
+ if (count == sampleNum && StringUtils.isLatin(chars)) {
Review Comment:
Could we use `StringUtils.isLatin(chars, sampleNum)` instead?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
