zstan commented on code in PR #13529:
URL: https://github.com/apache/ignite/pull/13529#discussion_r3948926977
##########
modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java:
##########
@@ -733,8 +735,12 @@ public void writeBooleanArray(boolean[] val) {
*/
public void writeString(String val) {
if (val != null) {
- if (curStrBackingArr == null)
- curStrBackingArr = val.getBytes();
+ if (curStrBackingArr == null) {
Review Comment:
latin1 zero-copy array, hasNegatives fallback to getBytes(UTF_8),
UTF-8 read) but no test covers it: every existing
DirectByteBufferStream/DirectMessageWriter test writes ASCII only
##########
modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java:
##########
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import org.apache.ignite.IgniteCommonsSystemProperties;
+import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY;
+
+/**
+ * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8
without allocation of temporary byte arrays.
+ *
+ * @see IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY
+ */
+public final class StringWriter {
+ /** Latin-1 value of the {@code java.lang.String#coder} field. */
+ private static final byte LATIN1 = 0;
+
+ /** Offset of the {@code java.lang.String#value} field, or {@code -1} if
the compact string fast path is unavailable. */
+ private static final long STR_VALUE_OFF;
+
+ /** Offset of the {@code java.lang.String#coder} field, or {@code -1} if
the compact string fast path is unavailable. */
+ private static final long STR_CODER_OFF;
+
+ static {
+ IgniteBiTuple<Long, Long> result = fieldsOffsets();
+
+ STR_VALUE_OFF = result.get1();
+ STR_CODER_OFF = result.get2();
+ }
+
+ /**
+ * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives},
or {@code null} if unavailable.
+ * The intrinsic scans the array with SIMD instructions, far faster than
any scalar loop.
+ */
+ private static final MethodHandle HAS_NEGATIVES = hasNegatives();
+
+ /** */
+ private StringWriter() {
+ // No-op.
+ }
+
+ /**
+ * Writes a string to the output stream.
+ *
+ * @param val Value.
+ * @param out Output stream.
+ */
+ public static void write(@NotNull String val, BinaryOutputStream out) {
+ // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for
length.
+ out.unsafeEnsure(1 + 4);
+ out.unsafeWriteByte(GridBinaryMarshaller.STRING);
+
+ int lenPos = out.position();
+
+ out.unsafePosition(out.position() + 4);
+
+ int written;
+
+ byte[] latin1 = latin1Value(val);
+
+ if (latin1 != null) {
+ if (out.hasArray()) {
+ if (!hasNegatives(latin1)) {
+ out.unsafeEnsure(latin1.length);
+ // Pure ASCII: UTF-8 representation matches the internal
array, copy it as-is.
+ System.arraycopy(latin1, 0, out.array(), out.position(),
latin1.length);
+
+ written = latin1.length;
+ }
+ else
+ written = encodeLatin1(latin1, out);
+
+ out.unsafePosition(out.position() + written);
+ }
+ else
+ written = writeLatin1(latin1, out);
+ }
+ else {
+ // Allocating memory for worst case - 3 bytes per char.
+ out.unsafeEnsure(Math.multiplyExact(3, val.length()));
+
+ if (out.hasArray()) {
+ written = encodeChars(val, out);
+
+ out.unsafePosition(out.position() + written);
+ }
+ else
+ written = writeChars(val, out);
+ }
+
+ out.unsafeWriteInt(lenPos, written);
+ }
+
+ /**
+ * Writes a Latin-1 encoded string value to the stream.
+ *
+ * @param val Internal Latin-1 array of the string.
+ * @param out Output stream.
+ * @return Number of bytes written.
+ */
+ private static int writeLatin1(byte[] val, BinaryOutputStream out) {
+ out.unsafeEnsure(Math.addExact(val.length, val.length));
+
+ int utfLen = 0;
+
+ for (int i = 0; i < val.length; i++) {
+ byte b = val[i];
+
+ if (b >= 0) {
+ out.unsafeWriteByte(b);
+
+ utfLen++;
+ }
+ else {
+ int c = b & 0b1111_1111;
+
+ out.unsafeWriteByte((byte)(0b1100_0000 | (c >> 6)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111)));
+
+ utfLen += 2;
+ }
+ }
+
+ return utfLen;
+ }
+
+ /**
+ * Encodes a Latin-1 string value to the buffer as UTF-8.
+ *
+ * @param val Internal Latin-1 array of the string.
+ * @param out Output stream.
+ * @return Count of written bytes.
+ */
+ private static int encodeLatin1(byte[] val, BinaryOutputStream out) {
+ out.unsafeEnsure(Math.addExact(val.length, val.length));
+
+ byte[] buf = out.array();
+
+ long off = out.position() + GridUnsafe.BYTE_ARR_OFF;
+
+ for (int i = 0; i < val.length; i++) {
+ byte b = val[i];
+
+ if (b >= 0)
+ GridUnsafe.putByte(buf, off++, b);
+ else {
+ int c = b & 0xFF;
+
+ GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6)));
+ GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c &
0b0011_1111)));
+ }
+ }
+
+ return (int)(off - GridUnsafe.BYTE_ARR_OFF - out.position());
+ }
+
+ /**
+ * Writes string chars UTF-8 encoded to the stream. Replicates {@code
String#getBytes(UTF_8)} behavior exactly,
+ * including replacement of malformed surrogates with {@code '?'}. Stream
capacity must be ensured by the caller.
+ *
+ * @param val Value.
+ * @param out Output stream.
+ * @return Number of bytes written.
+ */
+ private static int writeChars(String val, BinaryOutputStream out) {
+ int len = val.length();
+ int utfLen = 0;
+
+ for (int i = 0; i < len; i++) {
+ char c = val.charAt(i);
+
+ if (c < 0x80) {
+ out.unsafeWriteByte((byte)c);
+
+ utfLen++;
+ }
+ else if (c < 0x800) {
+ out.unsafeWriteByte((byte)(0b11_000000 | (c >> 6)));
+ out.unsafeWriteByte((byte)(0b10_000000 | (c & 0b00_111111)));
+
+ utfLen += 2;
+ }
+ else if (!Character.isSurrogate(c)) {
+ out.unsafeWriteByte((byte)(0b1110_0000 | (c >> 12)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | ((c >> 6) &
0b0011_1111)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111)));
+
+ utfLen += 3;
+ }
+ else {
+ char c2;
+
+ if (Character.isHighSurrogate(c) && i + 1 < len &&
Character.isLowSurrogate(c2 = val.charAt(i + 1))) {
+ int cp = Character.toCodePoint(c, c2);
+
+ out.unsafeWriteByte((byte)(0b1111_0000 | (cp >> 18)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 12) &
0b0011_1111)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 6) &
0b0011_1111)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | (cp &
0b0011_1111)));
+
+ utfLen += 4;
+ i++;
+ }
+ else {
+ out.unsafeWriteByte((byte)'?');
+
+ utfLen++;
+ }
+ }
+ }
+
+ return utfLen;
+ }
+
+ /**
+ * Encodes string chars to the buffer as UTF-8. Replicates {@code
String#getBytes(UTF_8)} behavior exactly,
+ * including replacement of malformed surrogates with {@code '?'}. Buffer
capacity must be ensured by the caller.
+ *
+ * @param val Value.
+ * @param out Output stream.
+ * @return Count of written bytes.
+ */
+ private static int encodeChars(String val, BinaryOutputStream out) {
+ byte[] buf = out.array();
+ int len = val.length();
+
+ // Unsafe writes skip the array bounds checks: capacity is ensured by
the caller.
+ long off = GridUnsafe.BYTE_ARR_OFF + out.position();
+
+ for (int i = 0; i < len; i++) {
+ char c = val.charAt(i);
+
+ if (c < 0x80)
+ GridUnsafe.putByte(buf, off++, (byte)c);
+ else if (c < 0x800) {
+ GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6)));
+ GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c &
0b0011_1111)));
+ }
+ else if (!Character.isSurrogate(c)) {
+ GridUnsafe.putByte(buf, off++, (byte)(0b1110_0000 | (c >>
12)));
+ GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((c >> 6)
& 0b0011_1111)));
+ GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c &
0b0011_1111)));
+ }
+ else {
+ char c2;
+
+ if (Character.isHighSurrogate(c) && i + 1 < len &&
Character.isLowSurrogate(c2 = val.charAt(i + 1))) {
+ int cp = Character.toCodePoint(c, c2);
+
+ GridUnsafe.putByte(buf, off++, (byte)(0b1111_0000 | (cp >>
18)));
+ GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((cp
>> 12) & 0b0011_1111)));
+ GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((cp
>> 6) & 0b0011_1111)));
+ GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (cp &
0b0011_1111)));
+
+ i++;
+ }
+ else
+ GridUnsafe.putByte(buf, off++, (byte)'?');
+ }
+ }
+
+ return (int)(off - GridUnsafe.BYTE_ARR_OFF - out.position());
+ }
+
+ /**
+ * @param val String.
+ * @return Internal Latin-1 array of the string,
+ * or {@code null} if the string is UTF-16 encoded or the internal
layout of {@link String} is unknown.
+ */
+ public static byte[] latin1Value(String val) {
Review Comment:
```suggestion
@Nullable public static byte[] latin1Value(String val) {
```
##########
modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java:
##########
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.util.Arrays;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.internal.binary.streams.BinaryStreams;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Tests that {@link StringWriter} output is byte-identical to serialization
of the {@link String#getBytes()} result,
+ * which was used before zero-copy string serialization was introduced.
+ */
+public class StringWriterSelfTest extends GridCommonAbstractTest {
+ /** */
+ public static final int ASCII_MAX = 0x80;
+
+ /** */
+ public static final int LATIN1_MAX = 0x100;
+
+ /** */
+ public static final int TWO_BYTES_MAX = 0x800;
+
+ /** */
+ public static final int THREE_BYTES_MAX = 0xD800;
+
+ /** */
+ public static final int FOUR_BYTES_MAX = 0xE000;
+
+ /** */
+ public static final int FOUR_BYTES_HIGH_BOUND = 0x10000;
+
+ /** Tests for all encoder paths: ASCII bulk copy, Latin-1, generic UTF-16
and malformed surrogates. */
+ @Test
+ public void testCorpus() {
+ String[] cases = {
+ "",
+ "a",
+ "?",
+ "abcdefghijklmnopqrstuvwxyz0123456789", // Long ASCII: exercises
the 8-byte stride scan and bulk copy.
+ "caf\u00e9", // Latin-1 with a negative
byte.
+ "\u00ff\u0080\u00a0", // Latin-1, negative bytes
only.
+ "\u041f\u0440\u0438\u0432\u0435\u0442", // Cyrillic: 2-byte UTF-8
sequences.
+ "\u0800\u1234\uffff", // 3-byte UTF-8 sequences.
+ "\ud83d\ude00", // Emoji: valid surrogate
pair.
+ "a\ud83d\ude00b\u00e9\u0416\u0001", // Mixed content.
+ "\ud800", // Lone high surrogate.
+ "\udc00", // Lone low surrogate.
+ "a\ud800", // High surrogate at the
end.
+ "\ud800a", // High surrogate followed
by a regular char.
+ "\ud800\ud800", // Two high surrogates.
+ "\udc00\ud800", // Low surrogate before a
high one.
+ "\u0000", // NUL char.
+ "nul\u0000nul"
+ };
+
+ for (String str : cases)
+ check(str);
+ }
+
+ /** Randomized differential test against {@link String#getBytes()}. */
+ @Test
+ public void testRandomStrings() {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int iter = 0; iter < 100; iter++) {
+ StringBuilder sb = new StringBuilder(1 + rnd.nextInt(42));
+
+ for (int i = 0; i < sb.capacity(); i++) {
+ int bucket = rnd.nextInt(100);
+
+ char c;
+
+ if (bucket < 40)
+ // ASCII.
+ c = (char)rnd.nextInt(ASCII_MAX);
+ else if (bucket < 55)
+ // Latin-1.
+ c = (char)(ASCII_MAX + rnd.nextInt(LATIN1_MAX -
ASCII_MAX));
+ else if (bucket < 65)
+ // Other 2-byte chars.
+ c = (char)(LATIN1_MAX + rnd.nextInt(TWO_BYTES_MAX -
LATIN1_MAX));
+ else if (bucket < 75)
+ // 3-byte chars.
+ c = (char)(TWO_BYTES_MAX + rnd.nextInt(THREE_BYTES_MAX -
TWO_BYTES_MAX));
+ else if (bucket < 90)
+ // Surrogates, mostly malformed.
+ c = (char)(THREE_BYTES_MAX + rnd.nextInt(FOUR_BYTES_MAX -
THREE_BYTES_MAX));
+ else
+ // 3-byte chars above the surrogate range.
+ c = (char)(FOUR_BYTES_MAX +
rnd.nextInt(FOUR_BYTES_HIGH_BOUND - FOUR_BYTES_MAX));
+
+ sb.append(c);
+ }
+
+ assertFalse(sb.isEmpty());
+
+ check(sb.toString());
+ }
+ }
+
+ /** Tests strings whose UTF-8 form is larger than the stream's minimal
capacity. */
+ @Test
+ public void testLargeStrings() {
+ int len = 100_000;
+
+ StringBuilder ascii = new StringBuilder(len);
+ StringBuilder latin1 = new StringBuilder(len);
+ StringBuilder cyrillic = new StringBuilder(len);
+ StringBuilder mixed = new StringBuilder(len);
+
+ for (int i = 0; i < len; i++) {
+ ascii.append((char)('a' + i % 26));
+ // Every char is a Latin-1 char with the sign bit set: worst case
for the 2-bytes-per-char reservation.
+ latin1.append((char)(ASCII_MAX + i % (LATIN1_MAX - ASCII_MAX)));
+ cyrillic.append((char)('\u0410' + i % 32));
+ mixed.append((char)('a' + i %
26)).append('\u00e9').append('\u0416').append('\u20ac').append("\ud83d\ude00");
+ }
+
+ check(ascii.toString());
+ check(latin1.toString());
+ check(cyrillic.toString());
+ check(mixed.toString());
+ }
+
+ /** Tests that the stream position is correct after a string write, so
surrounding values are not corrupted. */
+ @Test
+ public void testStreamPosition() {
+ int int1 = 0xDEADBEEF;
+ String str1 = "caf\u00e9";
+ String str2 = "\ud83d\ude00";
+ int int2 = 0xCAFEBABE;
+
+ // Small initial capacity to check buffer reallocation.
+ try (BinaryOutputStream out = BinaryStreams.outputStream(2)) {
+ out.writeInt(int1);
+ StringWriter.write(str1, out);
+ StringWriter.write(str2, out);
+ out.writeInt(int2);
+
+ byte[] strBytes1 = strBytes(str1);
+ byte[] strBytes2 = strBytes(str2);
+
+ byte[] exp = new byte[Integer.BYTES + strBytes1.length +
strBytes2.length + Integer.BYTES];
+
+ System.arraycopy(intBytes(int1), 0, exp, 0, Integer.BYTES);
+ System.arraycopy(strBytes1, 0, exp, Integer.BYTES,
strBytes1.length);
+ System.arraycopy(strBytes2, 0, exp, Integer.BYTES +
strBytes1.length, strBytes2.length);
+ System.arraycopy(intBytes(int2), 0, exp, Integer.BYTES +
strBytes1.length + strBytes2.length, Integer.BYTES);
+
+ assertTrue(Arrays.equals(exp, out.arrayCopy()));
+ }
+ }
+
+ /**
+ * Checks that serialized form of the given string is byte-identical to
serialization of the {@link String#getBytes()} result.
+ * @param str String to check.
+ */
+ private void check(String str) {
+ try (BinaryOutputStream out = BinaryStreams.outputStream(1)) {
+ StringWriter.write(str, out);
+
+ assertTrue("String serialization mismatch: " + str,
Arrays.equals(strBytes(str), out.arrayCopy()));
+ }
+ }
+
+ /**
+ * @param str String.
+ * @return Expected serialized form of the string: flag, UTF-8 length and
UTF-8 bytes.
+ */
+ private static byte[] strBytes(String str) {
+ byte[] bytes = str.getBytes(UTF_8);
Review Comment:
why we check only UTF-8, how to deal with UTF-16 or ignite not ready for
utf-16 and i miss smth ?
##########
modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java:
##########
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.benchmarks.jmh.binary;
+
+import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner;
+import org.apache.ignite.internal.binary.StringWriter;
+import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.internal.binary.streams.BinaryStreams;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static
org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY;
+import static org.openjdk.jmh.annotations.Mode.AverageTime;
+import static org.openjdk.jmh.annotations.Scope.Thread;
+
+/**
+ * Compares zero-copy string serialization with the legacy serialization.
+ * @see
org.apache.ignite.IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY
+ */
+@State(Thread)
+@OutputTimeUnit(NANOSECONDS)
+@BenchmarkMode(AverageTime)
+@Warmup(iterations = 5, time = 5, timeUnit = SECONDS)
+@Measurement(iterations = 5, time = 10, timeUnit = SECONDS)
+public class JmhBinaryStringWriteBenchmark {
+ /** */
+ @Param({"true", "false"})
+ private boolean zeroCopy;
+
+ /** */
+ @Param({"8", "64", "512", "4096"})
+ private int len;
+
+ /** */
+ @Param({"ascii", "latin1", "cyrillic", "mixed"})
+ private String content;
+
+ /** */
+ private BinaryOutputStream out;
+
+ /** */
+ private String str;
+
+ /** */
+ public static void main(String[] args) throws Exception {
+ OptionsBuilder builder = JmhIdeBenchmarkRunner.create()
+ .forks(1)
+ .benchmarks(JmhBinaryStringWriteBenchmark.class.getName())
+ .profilers(GCProfiler.class)
+ .optionsBuilder();
+
+ new Runner(builder.build()).run();
+ }
+
+ /** */
+ @Setup
+ public void setup() {
+ // Must be set before the first use of StringWriter in this JVM.
+ System.setProperty(IGNITE_BINARY_STRING_ZERO_COPY,
String.valueOf(zeroCopy));
+
+ StringBuilder sb = new StringBuilder(len);
+
+ for (int i = 0; sb.length() < len; i++) {
+ switch (content) {
+ case "ascii":
+ sb.append((char)('a' + i % 26));
+
+ break;
+
+ case "latin1":
+ // Every 8th char is a Latin-1 char with the sign bit set.
+ sb.append(i % 8 == 7 ? (char)(0xC0 + i % 0x20) :
(char)('a' + i % 26));
+
+ break;
+
+ case "cyrillic":
+ sb.append((char)('\u0410' + i % 32));
+
+ break;
+
+ case "mixed":
+ // ASCII, Latin-1, 2-byte, 3-byte chars and a surrogate
pair.
+ switch (i % 5) {
+ case 0:
+ sb.append((char)('a' + i % 26));
+
+ break;
+
+ case 1:
+ sb.append('\u00e9');
+
+ break;
+
+ case 2:
+ sb.append('\u0416');
+
+ break;
+
+ case 3:
+ sb.append('\u20ac');
+
+ break;
+
+ default:
+ sb.append("\ud83d\ude00");
+ }
+
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unknown content type:
" + content);
+ }
+ }
+
+ str = sb.toString();
+
+ out = BinaryStreams.outputStream(4 * len + 64);
+ }
+
+ /** */
+ @TearDown
+ public void tearDown() {
+ out.close();
+ }
+
+ /** */
+ @Benchmark
+ public void writeString(Blackhole bh) {
+ out.position(0);
+
+ StringWriter.write(str, out);
Review Comment:
Locally i obtain:
```
Benchmark (content) (len) (zeroCopy)
Mode Cnt Score Error Units
JmhBinaryStringWriteBenchmark.writeString ascii 512 true
avgt 5 16.607 ± 0.328 ns/op
JmhBinaryStringWriteBenchmark.writeString ascii 512 false
avgt 5 38.419 ± 1.346 ns/op
JmhBinaryStringWriteBenchmark.writeString cyrillic 512 true
avgt 5 298.396 ± 8.303 ns/op
JmhBinaryStringWriteBenchmark.writeString cyrillic 512 false
avgt 5 450.267 ± 10.969 ns/op
```
##########
modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java:
##########
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.benchmarks.jmh.binary;
+
+import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner;
+import org.apache.ignite.internal.binary.StringWriter;
+import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.internal.binary.streams.BinaryStreams;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static
org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY;
+import static org.openjdk.jmh.annotations.Mode.AverageTime;
+import static org.openjdk.jmh.annotations.Scope.Thread;
+
+/**
+ * Compares zero-copy string serialization with the legacy serialization.
+ * @see
org.apache.ignite.IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY
+ */
+@State(Thread)
+@OutputTimeUnit(NANOSECONDS)
+@BenchmarkMode(AverageTime)
+@Warmup(iterations = 5, time = 5, timeUnit = SECONDS)
+@Measurement(iterations = 5, time = 10, timeUnit = SECONDS)
+public class JmhBinaryStringWriteBenchmark {
+ /** */
+ @Param({"true", "false"})
+ private boolean zeroCopy;
+
+ /** */
+ @Param({"8", "64", "512", "4096"})
+ private int len;
+
+ /** */
+ @Param({"ascii", "latin1", "cyrillic", "mixed"})
+ private String content;
+
+ /** */
+ private BinaryOutputStream out;
+
+ /** */
+ private String str;
+
+ /** */
+ public static void main(String[] args) throws Exception {
+ OptionsBuilder builder = JmhIdeBenchmarkRunner.create()
+ .forks(1)
+ .benchmarks(JmhBinaryStringWriteBenchmark.class.getName())
+ .profilers(GCProfiler.class)
+ .optionsBuilder();
+
+ new Runner(builder.build()).run();
+ }
+
+ /** */
+ @Setup
+ public void setup() {
+ // Must be set before the first use of StringWriter in this JVM.
+ System.setProperty(IGNITE_BINARY_STRING_ZERO_COPY,
String.valueOf(zeroCopy));
+
+ StringBuilder sb = new StringBuilder(len);
+
+ for (int i = 0; sb.length() < len; i++) {
+ switch (content) {
+ case "ascii":
+ sb.append((char)('a' + i % 26));
+
+ break;
+
+ case "latin1":
+ // Every 8th char is a Latin-1 char with the sign bit set.
+ sb.append(i % 8 == 7 ? (char)(0xC0 + i % 0x20) :
(char)('a' + i % 26));
+
+ break;
+
+ case "cyrillic":
+ sb.append((char)('\u0410' + i % 32));
+
+ break;
+
+ case "mixed":
+ // ASCII, Latin-1, 2-byte, 3-byte chars and a surrogate
pair.
+ switch (i % 5) {
+ case 0:
+ sb.append((char)('a' + i % 26));
+
+ break;
+
+ case 1:
+ sb.append('\u00e9');
+
+ break;
+
+ case 2:
+ sb.append('\u0416');
+
+ break;
+
+ case 3:
+ sb.append('\u20ac');
+
+ break;
+
+ default:
+ sb.append("\ud83d\ude00");
+ }
+
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unknown content type:
" + content);
+ }
+ }
+
+ str = sb.toString();
+
+ out = BinaryStreams.outputStream(4 * len + 64);
+ }
+
+ /** */
+ @TearDown
+ public void tearDown() {
+ out.close();
+ }
+
+ /** */
+ @Benchmark
+ public void writeString(Blackhole bh) {
+ out.position(0);
+
+ StringWriter.write(str, out);
Review Comment:
This bench compares StringWriter reaction on IGNITE_BINARY_STRING_ZERO_COPY
flag, isn\`t it ?
I think that correct to compare new and old implementations ?
```
@Benchmark
public void writeString(Blackhole bh) {
out.position(0);
if (zeroCopy)
StringWriter.write(str, out);
else {
byte[] strArr;
if (BinaryUtils.USE_STR_SERIALIZATION_VER_2)
strArr = BinaryUtils.strToUtf8Bytes(str);
else
strArr = str.getBytes(UTF_8);
out.unsafeEnsure(1 + 4);
out.unsafeWriteByte(GridBinaryMarshaller.STRING);
out.unsafeWriteInt(strArr.length);
out.writeByteArray(strArr);
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
