nizhikov commented on code in PR #13529: URL: https://github.com/apache/ignite/pull/13529#discussion_r3949435770
########## modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java: ########## @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmarks.jmh.binary; + +import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner; +import org.apache.ignite.internal.binary.StringWriter; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.binary.streams.BinaryStreams; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY; +import static org.openjdk.jmh.annotations.Mode.AverageTime; +import static org.openjdk.jmh.annotations.Scope.Thread; + +/** + * Compares zero-copy string serialization with the legacy serialization. + * @see org.apache.ignite.IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY + */ +@State(Thread) +@OutputTimeUnit(NANOSECONDS) +@BenchmarkMode(AverageTime) +@Warmup(iterations = 5, time = 5, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 10, timeUnit = SECONDS) +public class JmhBinaryStringWriteBenchmark { + /** */ + @Param({"true", "false"}) + private boolean zeroCopy; + + /** */ + @Param({"8", "64", "512", "4096"}) + private int len; + + /** */ + @Param({"ascii", "latin1", "cyrillic", "mixed"}) + private String content; + + /** */ + private BinaryOutputStream out; + + /** */ + private String str; + + /** */ + public static void main(String[] args) throws Exception { + OptionsBuilder builder = JmhIdeBenchmarkRunner.create() + .forks(1) + .benchmarks(JmhBinaryStringWriteBenchmark.class.getName()) + .profilers(GCProfiler.class) + .optionsBuilder(); + + new Runner(builder.build()).run(); + } + + /** */ + @Setup + public void setup() { + // Must be set before the first use of StringWriter in this JVM. + System.setProperty(IGNITE_BINARY_STRING_ZERO_COPY, String.valueOf(zeroCopy)); + + StringBuilder sb = new StringBuilder(len); + + for (int i = 0; sb.length() < len; i++) { + switch (content) { + case "ascii": + sb.append((char)('a' + i % 26)); + + break; + + case "latin1": + // Every 8th char is a Latin-1 char with the sign bit set. + sb.append(i % 8 == 7 ? (char)(0xC0 + i % 0x20) : (char)('a' + i % 26)); + + break; + + case "cyrillic": + sb.append((char)('\u0410' + i % 32)); + + break; + + case "mixed": + // ASCII, Latin-1, 2-byte, 3-byte chars and a surrogate pair. + switch (i % 5) { + case 0: + sb.append((char)('a' + i % 26)); + + break; + + case 1: + sb.append('\u00e9'); + + break; + + case 2: + sb.append('\u0416'); + + break; + + case 3: + sb.append('\u20ac'); + + break; + + default: + sb.append("\ud83d\ude00"); + } + + break; + + default: + throw new IllegalArgumentException("Unknown content type: " + content); + } + } + + str = sb.toString(); + + out = BinaryStreams.outputStream(4 * len + 64); + } + + /** */ + @TearDown + public void tearDown() { + out.close(); + } + + /** */ + @Benchmark + public void writeString(Blackhole bh) { + out.position(0); + + StringWriter.write(str, out); Review Comment: Correct. Sorry, last minute refactoring issue. ########## modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.binary.streams.BinaryStreams; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Tests that {@link StringWriter} output is byte-identical to serialization of the {@link String#getBytes()} result, + * which was used before zero-copy string serialization was introduced. + */ +public class StringWriterSelfTest extends GridCommonAbstractTest { + /** */ + public static final int ASCII_MAX = 0x80; + + /** */ + public static final int LATIN1_MAX = 0x100; + + /** */ + public static final int TWO_BYTES_MAX = 0x800; + + /** */ + public static final int THREE_BYTES_MAX = 0xD800; + + /** */ + public static final int FOUR_BYTES_MAX = 0xE000; + + /** */ + public static final int FOUR_BYTES_HIGH_BOUND = 0x10000; Review Comment: Done ########## modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.binary.streams.BinaryStreams; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Tests that {@link StringWriter} output is byte-identical to serialization of the {@link String#getBytes()} result, + * which was used before zero-copy string serialization was introduced. + */ +public class StringWriterSelfTest extends GridCommonAbstractTest { + /** */ + public static final int ASCII_MAX = 0x80; + + /** */ + public static final int LATIN1_MAX = 0x100; + + /** */ + public static final int TWO_BYTES_MAX = 0x800; + + /** */ + public static final int THREE_BYTES_MAX = 0xD800; + + /** */ + public static final int FOUR_BYTES_MAX = 0xE000; + + /** */ + public static final int FOUR_BYTES_HIGH_BOUND = 0x10000; + + /** Tests for all encoder paths: ASCII bulk copy, Latin-1, generic UTF-16 and malformed surrogates. */ + @Test + public void testCorpus() { + String[] cases = { + "", + "a", + "?", + "abcdefghijklmnopqrstuvwxyz0123456789", // Long ASCII: exercises the 8-byte stride scan and bulk copy. + "caf\u00e9", // Latin-1 with a negative byte. + "\u00ff\u0080\u00a0", // Latin-1, negative bytes only. + "\u041f\u0440\u0438\u0432\u0435\u0442", // Cyrillic: 2-byte UTF-8 sequences. + "\u0800\u1234\uffff", // 3-byte UTF-8 sequences. + "\ud83d\ude00", // Emoji: valid surrogate pair. + "a\ud83d\ude00b\u00e9\u0416\u0001", // Mixed content. + "\ud800", // Lone high surrogate. + "\udc00", // Lone low surrogate. + "a\ud800", // High surrogate at the end. + "\ud800a", // High surrogate followed by a regular char. + "\ud800\ud800", // Two high surrogates. + "\udc00\ud800", // Low surrogate before a high one. + "\u0000", // NUL char. + "nul\u0000nul" + }; + + for (String str : cases) + check(str); + } + + /** Randomized differential test against {@link String#getBytes()}. */ + @Test + public void testRandomStrings() { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + for (int iter = 0; iter < 100; iter++) { + StringBuilder sb = new StringBuilder(1 + rnd.nextInt(42)); + + for (int i = 0; i < sb.capacity(); i++) { + int bucket = rnd.nextInt(100); + + char c; + + if (bucket < 40) + // ASCII. + c = (char)rnd.nextInt(ASCII_MAX); + else if (bucket < 55) + // Latin-1. + c = (char)(ASCII_MAX + rnd.nextInt(LATIN1_MAX - ASCII_MAX)); + else if (bucket < 65) + // Other 2-byte chars. + c = (char)(LATIN1_MAX + rnd.nextInt(TWO_BYTES_MAX - LATIN1_MAX)); + else if (bucket < 75) + // 3-byte chars. + c = (char)(TWO_BYTES_MAX + rnd.nextInt(THREE_BYTES_MAX - TWO_BYTES_MAX)); + else if (bucket < 90) + // Surrogates, mostly malformed. + c = (char)(THREE_BYTES_MAX + rnd.nextInt(FOUR_BYTES_MAX - THREE_BYTES_MAX)); + else + // 3-byte chars above the surrogate range. + c = (char)(FOUR_BYTES_MAX + rnd.nextInt(FOUR_BYTES_HIGH_BOUND - FOUR_BYTES_MAX)); + + sb.append(c); + } + + assertFalse(sb.isEmpty()); + + check(sb.toString()); + } + } + + /** Tests strings whose UTF-8 form is larger than the stream's minimal capacity. */ + @Test + public void testLargeStrings() { + int len = 100_000; + + StringBuilder ascii = new StringBuilder(len); + StringBuilder latin1 = new StringBuilder(len); + StringBuilder cyrillic = new StringBuilder(len); + StringBuilder mixed = new StringBuilder(len); Review Comment: Done ########## modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java: ########## @@ -733,8 +735,12 @@ public void writeBooleanArray(boolean[] val) { */ public void writeString(String val) { if (val != null) { - if (curStrBackingArr == null) - curStrBackingArr = val.getBytes(); + if (curStrBackingArr == null) { Review Comment: Actually, `test_replace.test` Calcite test will fail. It runs with the `ScriptTestSuite`. Let's file a ticket and improve test coverage for this. ########## modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.binary.streams.BinaryStreams; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Tests that {@link StringWriter} output is byte-identical to serialization of the {@link String#getBytes()} result, + * which was used before zero-copy string serialization was introduced. + */ +public class StringWriterSelfTest extends GridCommonAbstractTest { + /** */ + public static final int ASCII_MAX = 0x80; + + /** */ + public static final int LATIN1_MAX = 0x100; + + /** */ + public static final int TWO_BYTES_MAX = 0x800; + + /** */ + public static final int THREE_BYTES_MAX = 0xD800; + + /** */ + public static final int FOUR_BYTES_MAX = 0xE000; + + /** */ + public static final int FOUR_BYTES_HIGH_BOUND = 0x10000; + + /** Tests for all encoder paths: ASCII bulk copy, Latin-1, generic UTF-16 and malformed surrogates. */ + @Test + public void testCorpus() { + String[] cases = { + "", + "a", + "?", + "abcdefghijklmnopqrstuvwxyz0123456789", // Long ASCII: exercises the 8-byte stride scan and bulk copy. + "caf\u00e9", // Latin-1 with a negative byte. + "\u00ff\u0080\u00a0", // Latin-1, negative bytes only. + "\u041f\u0440\u0438\u0432\u0435\u0442", // Cyrillic: 2-byte UTF-8 sequences. + "\u0800\u1234\uffff", // 3-byte UTF-8 sequences. + "\ud83d\ude00", // Emoji: valid surrogate pair. + "a\ud83d\ude00b\u00e9\u0416\u0001", // Mixed content. + "\ud800", // Lone high surrogate. + "\udc00", // Lone low surrogate. + "a\ud800", // High surrogate at the end. + "\ud800a", // High surrogate followed by a regular char. + "\ud800\ud800", // Two high surrogates. + "\udc00\ud800", // Low surrogate before a high one. + "\u0000", // NUL char. + "nul\u0000nul" + }; + + for (String str : cases) + check(str); + } + + /** Randomized differential test against {@link String#getBytes()}. */ + @Test + public void testRandomStrings() { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + for (int iter = 0; iter < 100; iter++) { + StringBuilder sb = new StringBuilder(1 + rnd.nextInt(42)); + + for (int i = 0; i < sb.capacity(); i++) { + int bucket = rnd.nextInt(100); + + char c; + + if (bucket < 40) + // ASCII. + c = (char)rnd.nextInt(ASCII_MAX); + else if (bucket < 55) + // Latin-1. + c = (char)(ASCII_MAX + rnd.nextInt(LATIN1_MAX - ASCII_MAX)); + else if (bucket < 65) + // Other 2-byte chars. + c = (char)(LATIN1_MAX + rnd.nextInt(TWO_BYTES_MAX - LATIN1_MAX)); + else if (bucket < 75) + // 3-byte chars. + c = (char)(TWO_BYTES_MAX + rnd.nextInt(THREE_BYTES_MAX - TWO_BYTES_MAX)); + else if (bucket < 90) + // Surrogates, mostly malformed. + c = (char)(THREE_BYTES_MAX + rnd.nextInt(FOUR_BYTES_MAX - THREE_BYTES_MAX)); + else + // 3-byte chars above the surrogate range. + c = (char)(FOUR_BYTES_MAX + rnd.nextInt(FOUR_BYTES_HIGH_BOUND - FOUR_BYTES_MAX)); + + sb.append(c); + } + + assertFalse(sb.isEmpty()); + + check(sb.toString()); + } + } + + /** Tests strings whose UTF-8 form is larger than the stream's minimal capacity. */ + @Test + public void testLargeStrings() { + int len = 100_000; + + StringBuilder ascii = new StringBuilder(len); + StringBuilder latin1 = new StringBuilder(len); + StringBuilder cyrillic = new StringBuilder(len); + StringBuilder mixed = new StringBuilder(len); Review Comment: Done ########## modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java: ########## @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import org.apache.ignite.IgniteCommonsSystemProperties; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.lang.IgniteBiTuple; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY; + +/** + * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8 without allocation of temporary byte arrays. + * + * @see IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY + */ +public final class StringWriter { + /** Latin-1 value of the {@code java.lang.String#coder} field. */ + private static final byte LATIN1 = 0; + + /** Offset of the {@code java.lang.String#value} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_VALUE_OFF; + + /** Offset of the {@code java.lang.String#coder} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_CODER_OFF; + + static { + IgniteBiTuple<Long, Long> result = fieldsOffsets(); + + STR_VALUE_OFF = result.get1(); + STR_CODER_OFF = result.get2(); + } + + /** + * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}, or {@code null} if unavailable. + * The intrinsic scans the array with SIMD instructions, far faster than any scalar loop. + */ + private static final MethodHandle HAS_NEGATIVES = hasNegatives(); + + /** */ + private StringWriter() { + // No-op. + } + + /** + * Writes a string to the output stream. + * + * @param val Value. + * @param out Output stream. + */ + public static void write(@NotNull String val, BinaryOutputStream out) { + // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + + int lenPos = out.position(); + + out.unsafePosition(out.position() + 4); + + int written; + + byte[] latin1 = latin1Value(val); + + if (latin1 != null) { + if (out.hasArray()) { + if (!hasNegatives(latin1)) { + out.unsafeEnsure(latin1.length); + // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. + System.arraycopy(latin1, 0, out.array(), out.position(), latin1.length); + + written = latin1.length; + } + else + written = encodeLatin1(latin1, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeLatin1(latin1, out); + } + else { + // Allocating memory for worst case - 3 bytes per char. + out.unsafeEnsure(Math.multiplyExact(3, val.length())); + + if (out.hasArray()) { + written = encodeChars(val, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeChars(val, out); + } + + out.unsafeWriteInt(lenPos, written); + } + + /** + * Writes a Latin-1 encoded string value to the stream. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + int utfLen = 0; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) { + out.unsafeWriteByte(b); + + utfLen++; + } + else { + int c = b & 0b1111_1111; + + out.unsafeWriteByte((byte)(0b1100_0000 | (c >> 6))); + out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); + + utfLen += 2; + } + } + + return utfLen; + } + + /** + * Encodes a Latin-1 string value to the buffer as UTF-8. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Count of written bytes. + */ + private static int encodeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + byte[] buf = out.array(); + + long off = out.position() + GridUnsafe.BYTE_ARR_OFF; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) + GridUnsafe.putByte(buf, off++, b); + else { + int c = b & 0xFF; + + GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); + } + } + + return (int)(off - GridUnsafe.BYTE_ARR_OFF - out.position()); + } + + /** + * Writes string chars UTF-8 encoded to the stream. Replicates {@code String#getBytes(UTF_8)} behavior exactly, + * including replacement of malformed surrogates with {@code '?'}. Stream capacity must be ensured by the caller. + * + * @param val Value. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeChars(String val, BinaryOutputStream out) { + int len = val.length(); + int utfLen = 0; + + for (int i = 0; i < len; i++) { + char c = val.charAt(i); + + if (c < 0x80) { + out.unsafeWriteByte((byte)c); + + utfLen++; + } + else if (c < 0x800) { + out.unsafeWriteByte((byte)(0b11_000000 | (c >> 6))); + out.unsafeWriteByte((byte)(0b10_000000 | (c & 0b00_111111))); + + utfLen += 2; + } + else if (!Character.isSurrogate(c)) { + out.unsafeWriteByte((byte)(0b1110_0000 | (c >> 12))); + out.unsafeWriteByte((byte)(0b1000_0000 | ((c >> 6) & 0b0011_1111))); + out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); + + utfLen += 3; + } + else { + char c2; + + if (Character.isHighSurrogate(c) && i + 1 < len && Character.isLowSurrogate(c2 = val.charAt(i + 1))) { + int cp = Character.toCodePoint(c, c2); + + out.unsafeWriteByte((byte)(0b1111_0000 | (cp >> 18))); + out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 12) & 0b0011_1111))); + out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 6) & 0b0011_1111))); + out.unsafeWriteByte((byte)(0b1000_0000 | (cp & 0b0011_1111))); + + utfLen += 4; + i++; + } + else { + out.unsafeWriteByte((byte)'?'); + + utfLen++; + } + } + } + + return utfLen; + } + + /** + * Encodes string chars to the buffer as UTF-8. Replicates {@code String#getBytes(UTF_8)} behavior exactly, + * including replacement of malformed surrogates with {@code '?'}. Buffer capacity must be ensured by the caller. + * + * @param val Value. + * @param out Output stream. + * @return Count of written bytes. + */ + private static int encodeChars(String val, BinaryOutputStream out) { Review Comment: Done ########## modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java: ########## @@ -38,6 +38,9 @@ public class IgniteCommonsSystemProperties { /** Default value of {@link IgniteCommonsSystemProperties#IGNITE_USE_BINARY_ARRAYS}. */ public static final boolean DFLT_IGNITE_USE_BINARY_ARRAYS = false; + /** Default value of {@link IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY}. */ + public static final boolean DFLT_ZERO_COPY = true; Review Comment: I think user must has a chance to disable this feature. Just in case measurement and checks miss some performance issue. ########## modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.binary.streams.BinaryStreams; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Tests that {@link StringWriter} output is byte-identical to serialization of the {@link String#getBytes()} result, + * which was used before zero-copy string serialization was introduced. + */ +public class StringWriterSelfTest extends GridCommonAbstractTest { + /** */ + public static final int ASCII_MAX = 0x80; + + /** */ + public static final int LATIN1_MAX = 0x100; + + /** */ + public static final int TWO_BYTES_MAX = 0x800; + + /** */ + public static final int THREE_BYTES_MAX = 0xD800; Review Comment: Replaced ########## modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java: ########## @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import org.apache.ignite.IgniteCommonsSystemProperties; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.lang.IgniteBiTuple; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY; + +/** + * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8 without allocation of temporary byte arrays. + * + * @see IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY + */ +public final class StringWriter { + /** Latin-1 value of the {@code java.lang.String#coder} field. */ + private static final byte LATIN1 = 0; + + /** Offset of the {@code java.lang.String#value} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_VALUE_OFF; + + /** Offset of the {@code java.lang.String#coder} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_CODER_OFF; + + static { + IgniteBiTuple<Long, Long> result = fieldsOffsets(); + + STR_VALUE_OFF = result.get1(); + STR_CODER_OFF = result.get2(); + } + + /** + * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}, or {@code null} if unavailable. + * The intrinsic scans the array with SIMD instructions, far faster than any scalar loop. + */ + private static final MethodHandle HAS_NEGATIVES = hasNegatives(); + + /** */ + private StringWriter() { + // No-op. + } + + /** + * Writes a string to the output stream. + * + * @param val Value. + * @param out Output stream. + */ + public static void write(@NotNull String val, BinaryOutputStream out) { + // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + + int lenPos = out.position(); + + out.unsafePosition(out.position() + 4); + + int written; + + byte[] latin1 = latin1Value(val); + + if (latin1 != null) { + if (out.hasArray()) { + if (!hasNegatives(latin1)) { + out.unsafeEnsure(latin1.length); + // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. + System.arraycopy(latin1, 0, out.array(), out.position(), latin1.length); + + written = latin1.length; + } + else + written = encodeLatin1(latin1, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeLatin1(latin1, out); + } + else { + // Allocating memory for worst case - 3 bytes per char. + out.unsafeEnsure(Math.multiplyExact(3, val.length())); + + if (out.hasArray()) { + written = encodeChars(val, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeChars(val, out); + } + + out.unsafeWriteInt(lenPos, written); + } + + /** + * Writes a Latin-1 encoded string value to the stream. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + int utfLen = 0; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) { + out.unsafeWriteByte(b); + + utfLen++; + } + else { + int c = b & 0b1111_1111; + + out.unsafeWriteByte((byte)(0b1100_0000 | (c >> 6))); + out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); + + utfLen += 2; + } + } + + return utfLen; + } + + /** + * Encodes a Latin-1 string value to the buffer as UTF-8. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Count of written bytes. + */ + private static int encodeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + byte[] buf = out.array(); + + long off = out.position() + GridUnsafe.BYTE_ARR_OFF; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) + GridUnsafe.putByte(buf, off++, b); + else { + int c = b & 0xFF; + + GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); + } + } + + return (int)(off - GridUnsafe.BYTE_ARR_OFF - out.position()); + } + + /** + * Writes string chars UTF-8 encoded to the stream. Replicates {@code String#getBytes(UTF_8)} behavior exactly, + * including replacement of malformed surrogates with {@code '?'}. Stream capacity must be ensured by the caller. + * + * @param val Value. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeChars(String val, BinaryOutputStream out) { + int len = val.length(); + int utfLen = 0; + + for (int i = 0; i < len; i++) { + char c = val.charAt(i); + + if (c < 0x80) { + out.unsafeWriteByte((byte)c); + + utfLen++; Review Comment: Done ########## modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java: ########## @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import org.apache.ignite.IgniteCommonsSystemProperties; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.lang.IgniteBiTuple; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY; + +/** + * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8 without allocation of temporary byte arrays. + * + * @see IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY + */ +public final class StringWriter { + /** Latin-1 value of the {@code java.lang.String#coder} field. */ + private static final byte LATIN1 = 0; + + /** Offset of the {@code java.lang.String#value} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_VALUE_OFF; + + /** Offset of the {@code java.lang.String#coder} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_CODER_OFF; + + static { + IgniteBiTuple<Long, Long> result = fieldsOffsets(); + + STR_VALUE_OFF = result.get1(); + STR_CODER_OFF = result.get2(); + } + + /** + * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}, or {@code null} if unavailable. + * The intrinsic scans the array with SIMD instructions, far faster than any scalar loop. + */ + private static final MethodHandle HAS_NEGATIVES = hasNegatives(); + + /** */ + private StringWriter() { + // No-op. + } + + /** + * Writes a string to the output stream. + * + * @param val Value. + * @param out Output stream. + */ + public static void write(@NotNull String val, BinaryOutputStream out) { + // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + + int lenPos = out.position(); + + out.unsafePosition(out.position() + 4); + + int written; + + byte[] latin1 = latin1Value(val); + + if (latin1 != null) { + if (out.hasArray()) { + if (!hasNegatives(latin1)) { + out.unsafeEnsure(latin1.length); + // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. + System.arraycopy(latin1, 0, out.array(), out.position(), latin1.length); + + written = latin1.length; + } + else + written = encodeLatin1(latin1, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeLatin1(latin1, out); + } + else { + // Allocating memory for worst case - 3 bytes per char. + out.unsafeEnsure(Math.multiplyExact(3, val.length())); + + if (out.hasArray()) { + written = encodeChars(val, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeChars(val, out); + } + + out.unsafeWriteInt(lenPos, written); + } + + /** + * Writes a Latin-1 encoded string value to the stream. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + int utfLen = 0; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) { + out.unsafeWriteByte(b); + + utfLen++; + } + else { + int c = b & 0b1111_1111; + + out.unsafeWriteByte((byte)(0b1100_0000 | (c >> 6))); + out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); + + utfLen += 2; + } + } + + return utfLen; + } + + /** + * Encodes a Latin-1 string value to the buffer as UTF-8. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Count of written bytes. + */ + private static int encodeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + byte[] buf = out.array(); + + long off = out.position() + GridUnsafe.BYTE_ARR_OFF; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) + GridUnsafe.putByte(buf, off++, b); Review Comment: Done ########## modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.binary.streams.BinaryStreams; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Tests that {@link StringWriter} output is byte-identical to serialization of the {@link String#getBytes()} result, + * which was used before zero-copy string serialization was introduced. + */ +public class StringWriterSelfTest extends GridCommonAbstractTest { + /** */ + public static final int ASCII_MAX = 0x80; + + /** */ + public static final int LATIN1_MAX = 0x100; + + /** */ + public static final int TWO_BYTES_MAX = 0x800; + + /** */ + public static final int THREE_BYTES_MAX = 0xD800; + + /** */ + public static final int FOUR_BYTES_MAX = 0xE000; Review Comment: Replaced with the `(Character.MAX_SURROGATE + 1)` ########## modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java: ########## @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import org.apache.ignite.IgniteCommonsSystemProperties; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.lang.IgniteBiTuple; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY; + +/** + * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8 without allocation of temporary byte arrays. + * + * @see IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY + */ +public final class StringWriter { + /** Latin-1 value of the {@code java.lang.String#coder} field. */ + private static final byte LATIN1 = 0; + + /** Offset of the {@code java.lang.String#value} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_VALUE_OFF; + + /** Offset of the {@code java.lang.String#coder} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_CODER_OFF; + + static { + IgniteBiTuple<Long, Long> result = fieldsOffsets(); + + STR_VALUE_OFF = result.get1(); + STR_CODER_OFF = result.get2(); + } + + /** + * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}, or {@code null} if unavailable. + * The intrinsic scans the array with SIMD instructions, far faster than any scalar loop. + */ + private static final MethodHandle HAS_NEGATIVES = hasNegatives(); + + /** */ + private StringWriter() { + // No-op. + } + + /** + * Writes a string to the output stream. + * + * @param val Value. + * @param out Output stream. + */ + public static void write(@NotNull String val, BinaryOutputStream out) { + // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + + int lenPos = out.position(); + + out.unsafePosition(out.position() + 4); + + int written; + + byte[] latin1 = latin1Value(val); + + if (latin1 != null) { + if (out.hasArray()) { + if (!hasNegatives(latin1)) { + out.unsafeEnsure(latin1.length); + // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. + System.arraycopy(latin1, 0, out.array(), out.position(), latin1.length); + + written = latin1.length; + } + else + written = encodeLatin1(latin1, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeLatin1(latin1, out); + } + else { + // Allocating memory for worst case - 3 bytes per char. + out.unsafeEnsure(Math.multiplyExact(3, val.length())); + + if (out.hasArray()) { + written = encodeChars(val, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeChars(val, out); + } + + out.unsafeWriteInt(lenPos, written); + } + + /** + * Writes a Latin-1 encoded string value to the stream. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + int utfLen = 0; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) { + out.unsafeWriteByte(b); + + utfLen++; + } + else { + int c = b & 0b1111_1111; + + out.unsafeWriteByte((byte)(0b1100_0000 | (c >> 6))); + out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); + + utfLen += 2; + } + } + + return utfLen; + } + + /** + * Encodes a Latin-1 string value to the buffer as UTF-8. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Count of written bytes. + */ + private static int encodeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + byte[] buf = out.array(); + + long off = out.position() + GridUnsafe.BYTE_ARR_OFF; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) + GridUnsafe.putByte(buf, off++, b); + else { + int c = b & 0xFF; + + GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); + } + } + + return (int)(off - GridUnsafe.BYTE_ARR_OFF - out.position()); + } + + /** + * Writes string chars UTF-8 encoded to the stream. Replicates {@code String#getBytes(UTF_8)} behavior exactly, + * including replacement of malformed surrogates with {@code '?'}. Stream capacity must be ensured by the caller. + * + * @param val Value. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeChars(String val, BinaryOutputStream out) { + int len = val.length(); + int utfLen = 0; + + for (int i = 0; i < len; i++) { + char c = val.charAt(i); + + if (c < 0x80) { + out.unsafeWriteByte((byte)c); + + utfLen++; + } + else if (c < 0x800) { + out.unsafeWriteByte((byte)(0b11_000000 | (c >> 6))); + out.unsafeWriteByte((byte)(0b10_000000 | (c & 0b00_111111))); + + utfLen += 2; + } + else if (!Character.isSurrogate(c)) { + out.unsafeWriteByte((byte)(0b1110_0000 | (c >> 12))); + out.unsafeWriteByte((byte)(0b1000_0000 | ((c >> 6) & 0b0011_1111))); + out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); + + utfLen += 3; + } + else { + char c2; + + if (Character.isHighSurrogate(c) && i + 1 < len && Character.isLowSurrogate(c2 = val.charAt(i + 1))) { + int cp = Character.toCodePoint(c, c2); + + out.unsafeWriteByte((byte)(0b1111_0000 | (cp >> 18))); + out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 12) & 0b0011_1111))); + out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 6) & 0b0011_1111))); + out.unsafeWriteByte((byte)(0b1000_0000 | (cp & 0b0011_1111))); + + utfLen += 4; + i++; + } + else { + out.unsafeWriteByte((byte)'?'); + + utfLen++; + } + } + } + + return utfLen; + } + + /** + * Encodes string chars to the buffer as UTF-8. Replicates {@code String#getBytes(UTF_8)} behavior exactly, + * including replacement of malformed surrogates with {@code '?'}. Buffer capacity must be ensured by the caller. + * + * @param val Value. + * @param out Output stream. + * @return Count of written bytes. + */ + private static int encodeChars(String val, BinaryOutputStream out) { + byte[] buf = out.array(); + int len = val.length(); + + // Unsafe writes skip the array bounds checks: capacity is ensured by the caller. + long off = GridUnsafe.BYTE_ARR_OFF + out.position(); + + for (int i = 0; i < len; i++) { + char c = val.charAt(i); + + if (c < 0x80) + GridUnsafe.putByte(buf, off++, (byte)c); + else if (c < 0x800) { + GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); + } + else if (!Character.isSurrogate(c)) { + GridUnsafe.putByte(buf, off++, (byte)(0b1110_0000 | (c >> 12))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((c >> 6) & 0b0011_1111))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); + } + else { + char c2; + + if (Character.isHighSurrogate(c) && i + 1 < len && Character.isLowSurrogate(c2 = val.charAt(i + 1))) { + int cp = Character.toCodePoint(c, c2); + + GridUnsafe.putByte(buf, off++, (byte)(0b1111_0000 | (cp >> 18))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((cp >> 12) & 0b0011_1111))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((cp >> 6) & 0b0011_1111))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (cp & 0b0011_1111))); + + i++; + } + else + GridUnsafe.putByte(buf, off++, (byte)'?'); + } + } + + return (int)(off - GridUnsafe.BYTE_ARR_OFF - out.position()); + } + + /** + * @param val String. + * @return Internal Latin-1 array of the string, + * or {@code null} if the string is UTF-16 encoded or the internal layout of {@link String} is unknown. + */ + @Nullable public static byte[] latin1Value(String val) { + if (STR_VALUE_OFF < 0 || GridUnsafe.getByteField(val, STR_CODER_OFF) != LATIN1) + return null; + + return (byte[])GridUnsafe.getObjectField(val, STR_VALUE_OFF); + } + + /** + * @param arr Array. + * @return {@code True} if the array contains a byte with the sign bit set. + */ + public static boolean hasNegatives(byte[] arr) { + if (HAS_NEGATIVES != null) { + try { + return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); + } + catch (Throwable ignored) { + // Fall through to the generic implementation. + } + } + + // 8-byte strides with an early exit. + int i = 0; + + for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) { + long hasNegatives = GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i) + & 0b10000000_10000000_10000000_10000000_10000000_10000000_10000000_10000000L; + + if (hasNegatives != 0) + return true; + } + + for (; i < arr.length; i++) { + if (arr[i] < 0) + return true; + } + + return false; + } + + /** */ + private static IgniteBiTuple<Long, Long> fieldsOffsets() { + if (ZERO_COPY) { + try { + Field valField = String.class.getDeclaredField("value"); + Field coderField = String.class.getDeclaredField("coder"); + + // On JDK 8 the value field is a char[], only the generic encoder can be used. Review Comment: Remove check. ########## modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java: ########## @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import org.apache.ignite.IgniteCommonsSystemProperties; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.lang.IgniteBiTuple; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY; + +/** + * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8 without allocation of temporary byte arrays. + * + * @see IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY + */ +public final class StringWriter { + /** Latin-1 value of the {@code java.lang.String#coder} field. */ + private static final byte LATIN1 = 0; + + /** Offset of the {@code java.lang.String#value} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_VALUE_OFF; + + /** Offset of the {@code java.lang.String#coder} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_CODER_OFF; + + static { + IgniteBiTuple<Long, Long> result = fieldsOffsets(); + + STR_VALUE_OFF = result.get1(); + STR_CODER_OFF = result.get2(); + } + + /** + * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}, or {@code null} if unavailable. + * The intrinsic scans the array with SIMD instructions, far faster than any scalar loop. + */ + private static final MethodHandle HAS_NEGATIVES = hasNegatives(); + + /** */ + private StringWriter() { + // No-op. + } + + /** + * Writes a string to the output stream. + * + * @param val Value. + * @param out Output stream. + */ + public static void write(@NotNull String val, BinaryOutputStream out) { + // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + + int lenPos = out.position(); + + out.unsafePosition(out.position() + 4); + + int written; + + byte[] latin1 = latin1Value(val); + + if (latin1 != null) { + if (out.hasArray()) { + if (!hasNegatives(latin1)) { + out.unsafeEnsure(latin1.length); + // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. + System.arraycopy(latin1, 0, out.array(), out.position(), latin1.length); + + written = latin1.length; + } + else + written = encodeLatin1(latin1, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeLatin1(latin1, out); + } + else { + // Allocating memory for worst case - 3 bytes per char. + out.unsafeEnsure(Math.multiplyExact(3, val.length())); + + if (out.hasArray()) { + written = encodeChars(val, out); + + out.unsafePosition(out.position() + written); + } + else + written = writeChars(val, out); + } + + out.unsafeWriteInt(lenPos, written); + } + + /** + * Writes a Latin-1 encoded string value to the stream. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + int utfLen = 0; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) { + out.unsafeWriteByte(b); + + utfLen++; + } + else { + int c = b & 0b1111_1111; + + out.unsafeWriteByte((byte)(0b1100_0000 | (c >> 6))); + out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); + + utfLen += 2; + } + } + + return utfLen; + } + + /** + * Encodes a Latin-1 string value to the buffer as UTF-8. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Count of written bytes. + */ + private static int encodeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + byte[] buf = out.array(); + + long off = out.position() + GridUnsafe.BYTE_ARR_OFF; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) + GridUnsafe.putByte(buf, off++, b); + else { + int c = b & 0xFF; + + GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); + } + } + + return (int)(off - GridUnsafe.BYTE_ARR_OFF - out.position()); + } + + /** + * Writes string chars UTF-8 encoded to the stream. Replicates {@code String#getBytes(UTF_8)} behavior exactly, + * including replacement of malformed surrogates with {@code '?'}. Stream capacity must be ensured by the caller. + * + * @param val Value. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeChars(String val, BinaryOutputStream out) { + int len = val.length(); + int utfLen = 0; + + for (int i = 0; i < len; i++) { + char c = val.charAt(i); + + if (c < 0x80) { + out.unsafeWriteByte((byte)c); + + utfLen++; Review Comment: Done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
