raminqaf commented on code in PR #29023:
URL: https://github.com/apache/flink/pull/29023#discussion_r3880175073
##########
flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/BinaryStringDataTest.java:
##########
@@ -130,637 +116,811 @@ private BinaryStringData fromString(String str) {
}
}
- private void checkBasic(String str, int len) {
- BinaryStringData s1 = fromString(str);
+ /** Cases for the string-to-decimal conversion, shared by the string and
binary-row tests. */
+ private static final class DecimalCase {
+ private final String str;
+ private final int precision;
+ private final int scale;
+
+ private DecimalCase(String str, int precision, int scale) {
+ this.str = str;
+ this.precision = precision;
+ this.scale = scale;
+ }
+ }
+
+ private static final List<DecimalCase> DECIMAL_CASES =
+ List.of(
+ new DecimalCase("12.345", 5, 3),
+ new DecimalCase("-12.345", 5, 3),
+ new DecimalCase("+12345", 5, 0),
+ new DecimalCase("-12345", 5, 0),
+ new DecimalCase("12345.", 5, 0),
+ new DecimalCase("-12345.", 5, 0),
+ new DecimalCase(".12345", 5, 5),
+ new DecimalCase("-.12345", 5, 5),
+ new DecimalCase("+12.345E3", 5, 0),
+ new DecimalCase("-12.345e3", 5, 0),
+ new DecimalCase("12.345e-3", 6, 6),
+ new DecimalCase("-12.345E-3", 6, 6),
+ new DecimalCase("12345E3", 8, 0),
+ new DecimalCase("-12345e3", 8, 0),
+ new DecimalCase("12345e-3", 5, 3),
+ new DecimalCase("-12345E-3", 5, 3),
+ new DecimalCase("+.12345E3", 5, 2),
+ new DecimalCase("-.12345e3", 5, 2),
+ new DecimalCase(".12345e-3", 8, 8),
+ new DecimalCase("-.12345E-3", 8, 8),
+ new DecimalCase("1234512345.1234", 18, 8),
+ new DecimalCase("-1234512345.1234", 18, 8),
+ new DecimalCase("1234512345.1234", 12, 2),
+ new DecimalCase("-1234512345.1234", 12, 2),
+ new DecimalCase("1234512345.1299", 12, 2),
+ new DecimalCase("-1234512345.1299", 12, 2),
+ new DecimalCase("999999999999999999", 18, 0),
+ new DecimalCase("1234512345.1234512345", 20, 10),
+ new DecimalCase("-1234512345.1234512345", 20, 10),
+ new DecimalCase("1234512345.1234512345", 15, 5),
+ new DecimalCase("-1234512345.1234512345", 15, 5),
+ new DecimalCase("12345123451234512345E-10", 20, 10),
+ new DecimalCase("-12345123451234512345E-10", 20, 10),
+ new DecimalCase("12345123451234512345E-10", 15, 5),
+ new DecimalCase("-12345123451234512345E-10", 15, 5),
+ new DecimalCase("999999999999999999999", 21, 0),
+ new DecimalCase("-999999999999999999999", 21, 0),
+ new
DecimalCase("0.00000000000000000000123456789123456789", 38, 38),
+ new
DecimalCase("-0.00000000000000000000123456789123456789", 38, 38),
+ new
DecimalCase("0.00000000000000000000123456789123456789", 29, 29),
+ new
DecimalCase("-0.00000000000000000000123456789123456789", 29, 29),
+ new DecimalCase("123456789123E-27", 18, 18),
+ new DecimalCase("-123456789123E-27", 18, 18),
+ new DecimalCase("123456789999E-27", 18, 18),
+ new DecimalCase("-123456789999E-27", 18, 18),
+ new DecimalCase("123456789123456789E-36", 18, 18),
+ new DecimalCase("-123456789123456789E-36", 18, 18),
+ new DecimalCase("123456789999999999E-36", 18, 18),
+ new DecimalCase("-123456789999999999E-36", 18, 18));
+
+ private static Stream<Arguments> decimalCases() {
+ return Arrays.stream(Mode.values())
+ .flatMap(
+ mode ->
+ DECIMAL_CASES.stream()
+ .map(c -> arguments(mode, c.str,
c.precision, c.scale)));
+ }
+
+ /** Pairs every {@link Mode} with each of the given input strings. */
+ private static Stream<Arguments> withModes(String... inputs) {
+ return Arrays.stream(Mode.values())
+ .flatMap(mode -> Arrays.stream(inputs).map(input ->
arguments(mode, input)));
+ }
+
+ /** Decodes UTF-8 bytes through one of the {@link
StringUtf8Utils#decodeUTF8} entry points. */
+ @FunctionalInterface
+ private interface Utf8Decoder {
+ String decode(byte[] bytes);
+ }
+
+ private static void checkBasic(Mode mode, String str, int len) {
+ BinaryStringData s1 = fromString(mode, str);
BinaryStringData s2 = fromBytes(str.getBytes(StandardCharsets.UTF_8));
- assertThat(len).isEqualTo(s1.numChars());
- assertThat(len).isEqualTo(s2.numChars());
+ assertThat(s1.numChars()).isEqualTo(len);
+ assertThat(s2.numChars()).isEqualTo(len);
- assertThat(str).isEqualTo(s1.toString());
- assertThat(str).isEqualTo(s2.toString());
+ assertThat(s1).hasToString(str);
+ assertThat(s2).hasToString(str);
assertThat(s2).isEqualTo(s1);
assertThat(s2.hashCode()).isEqualTo(s1.hashCode());
- assertThat(s1.compareTo(s2)).isEqualTo(0);
+ assertThat(s1.compareTo(s2)).isZero();
assertThat(s1.contains(s2)).isTrue();
assertThat(s2.contains(s1)).isTrue();
assertThat(s1.startsWith(s1)).isTrue();
assertThat(s1.endsWith(s1)).isTrue();
}
- @TestTemplate
- void basicTest() {
- checkBasic("", 0);
- checkBasic(",", 1);
- checkBasic("hello", 5);
- checkBasic("hello world", 11);
- checkBasic("Flink中文社区", 9);
- checkBasic("中 文 社 区", 7);
-
- checkBasic("¡", 1); // 2 bytes char
- checkBasic("ку", 2); // 2 * 2 bytes chars
- checkBasic("︽﹋％", 3); // 3 * 3 bytes chars
- checkBasic("\uD83E\uDD19", 1); // 4 bytes char
- }
+ @Nested
+ @DisplayName("Basics")
+ class Basics {
+
+ @ParameterizedTest(name = "{0}")
+ @EnumSource(Mode.class)
+ void basicTest(Mode mode) {
+ checkBasic(mode, "", 0);
+ checkBasic(mode, ",", 1);
+ checkBasic(mode, "hello", 5);
+ checkBasic(mode, "hello world", 11);
+ checkBasic(mode, "Flink中文社区", 9);
+ checkBasic(mode, "中 文 社 区", 7);
+
+ checkBasic(mode, "¡", 1); // 2 bytes char
+ checkBasic(mode, "ку", 2); // 2 * 2 bytes chars
+ checkBasic(mode, "︽﹋％", 3); // 3 * 3 bytes chars
+ checkBasic(mode, "🤙", 1); // 4 bytes char
+ }
- @TestTemplate
- void emptyStringTest() {
- assertThat(fromString("")).isEqualTo(empty);
- assertThat(fromBytes(new byte[0])).isEqualTo(empty);
- assertThat(empty.numChars()).isEqualTo(0);
- assertThat(empty.getSizeInBytes()).isEqualTo(0);
- }
+ @ParameterizedTest(name = "{0}")
+ @EnumSource(Mode.class)
+ void emptyStringTest(Mode mode) {
+ BinaryStringData empty = fromString(mode, "");
+ assertThat(fromString(mode, "")).isEqualTo(empty);
+ assertThat(fromBytes(new byte[0])).isEqualTo(empty);
+ assertThat(empty.numChars()).isZero();
+ assertThat(empty.getSizeInBytes()).isZero();
+ }
- @TestTemplate
- void compareTo() {
- assertThat(fromString(" ").compareTo(blankString(3))).isEqualTo(0);
- assertThat(fromString("").compareTo(fromString("a"))).isLessThan(0);
-
assertThat(fromString("abc").compareTo(fromString("ABC"))).isGreaterThan(0);
-
assertThat(fromString("abc0").compareTo(fromString("abc"))).isGreaterThan(0);
-
assertThat(fromString("abcabcabc").compareTo(fromString("abcabcabc"))).isEqualTo(0);
-
assertThat(fromString("aBcabcabc").compareTo(fromString("Abcabcabc"))).isGreaterThan(0);
-
assertThat(fromString("Abcabcabc").compareTo(fromString("abcabcabC"))).isLessThan(0);
-
assertThat(fromString("abcabcabc").compareTo(fromString("abcabcabC"))).isGreaterThan(0);
-
-
assertThat(fromString("abc").compareTo(fromString("世界"))).isLessThan(0);
-
assertThat(fromString("你好").compareTo(fromString("世界"))).isGreaterThan(0);
-
assertThat(fromString("你好123").compareTo(fromString("你好122"))).isGreaterThan(0);
-
- MemorySegment segment1 =
MemorySegmentFactory.allocateUnpooledSegment(1024);
- MemorySegment segment2 =
MemorySegmentFactory.allocateUnpooledSegment(1024);
- SortUtil.putStringNormalizedKey(fromString("abcabcabc"), segment1, 0,
9);
- SortUtil.putStringNormalizedKey(fromString("abcabcabC"), segment2, 0,
9);
- assertThat(segment1.compare(segment2, 0, 0, 9)).isGreaterThan(0);
- SortUtil.putStringNormalizedKey(fromString("abcab"), segment1, 0, 9);
- assertThat(segment1.compare(segment2, 0, 0, 9)).isLessThan(0);
- }
+ @ParameterizedTest(name = "{0}")
+ @EnumSource(Mode.class)
+ void testIsEmpty(Mode mode) {
+ assertThat(isEmpty(fromString(mode, ""))).isTrue();
+ assertThat(isEmpty(BinaryStringData.fromBytes(new byte[]
{}))).isTrue();
+ assertThat(isEmpty(fromString(mode, "hello"))).isFalse();
+
assertThat(isEmpty(BinaryStringData.fromBytes("hello".getBytes()))).isFalse();
+ assertThat(isEmpty(fromString(mode, "中文"))).isFalse();
+
assertThat(isEmpty(BinaryStringData.fromBytes("中文".getBytes()))).isFalse();
+ assertThat(isEmpty(new BinaryStringData())).isTrue();
+ }
- @TestTemplate
- void testMultiSegments() {
-
- // prepare
- MemorySegment[] segments1 = new MemorySegment[2];
- segments1[0] = MemorySegmentFactory.wrap(new byte[10]);
- segments1[1] = MemorySegmentFactory.wrap(new byte[10]);
- segments1[0].put(5, "abcde".getBytes(UTF_8), 0, 5);
- segments1[1].put(0, "aaaaa".getBytes(UTF_8), 0, 5);
-
- MemorySegment[] segments2 = new MemorySegment[2];
- segments2[0] = MemorySegmentFactory.wrap(new byte[5]);
- segments2[1] = MemorySegmentFactory.wrap(new byte[5]);
- segments2[0].put(0, "abcde".getBytes(UTF_8), 0, 5);
- segments2[1].put(0, "b".getBytes(UTF_8), 0, 1);
-
- // test go ahead both
- BinaryStringData binaryString1 =
BinaryStringData.fromAddress(segments1, 5, 10);
- BinaryStringData binaryString2 =
BinaryStringData.fromAddress(segments2, 0, 6);
- assertThat(binaryString1.toString()).isEqualTo("abcdeaaaaa");
- assertThat(binaryString2.toString()).isEqualTo("abcdeb");
- assertThat(binaryString1.compareTo(binaryString2)).isEqualTo(-1);
-
- // test needCompare == len
- binaryString1 = BinaryStringData.fromAddress(segments1, 5, 5);
- binaryString2 = BinaryStringData.fromAddress(segments2, 0, 5);
- assertThat(binaryString1.toString()).isEqualTo("abcde");
- assertThat(binaryString2.toString()).isEqualTo("abcde");
- assertThat(binaryString1.compareTo(binaryString2)).isEqualTo(0);
-
- // test find the first segment of this string
- binaryString1 = BinaryStringData.fromAddress(segments1, 10, 5);
- binaryString2 = BinaryStringData.fromAddress(segments2, 0, 5);
- assertThat(binaryString1.toString()).isEqualTo("aaaaa");
- assertThat(binaryString2.toString()).isEqualTo("abcde");
- assertThat(binaryString1.compareTo(binaryString2)).isEqualTo(-1);
- assertThat(binaryString2.compareTo(binaryString1)).isEqualTo(1);
-
- // test go ahead single
- segments2 = new MemorySegment[] {MemorySegmentFactory.wrap(new
byte[10])};
- segments2[0].put(4, "abcdeb".getBytes(UTF_8), 0, 6);
- binaryString1 = BinaryStringData.fromAddress(segments1, 5, 10);
- binaryString2 = BinaryStringData.fromAddress(segments2, 4, 6);
- assertThat(binaryString1.toString()).isEqualTo("abcdeaaaaa");
- assertThat(binaryString2.toString()).isEqualTo("abcdeb");
- assertThat(binaryString1.compareTo(binaryString2)).isEqualTo(-1);
- assertThat(binaryString2.compareTo(binaryString1)).isEqualTo(1);
- }
+ @Test
+ void testLazy() {
+ String javaStr = "haha";
+ BinaryStringData str = BinaryStringData.fromString(javaStr);
+ str.ensureMaterialized();
- @TestTemplate
- void concatTest() {
- assertThat(concat()).isEqualTo(empty);
- assertThat(concat((BinaryStringData) null)).isNull();
- assertThat(concat(empty)).isEqualTo(empty);
- assertThat(concat(fromString("ab"))).isEqualTo(fromString("ab"));
- assertThat(concat(fromString("a"),
fromString("b"))).isEqualTo(fromString("ab"));
- assertThat(concat(fromString("a"), fromString("b"), fromString("c")))
- .isEqualTo(fromString("abc"));
- assertThat(concat(fromString("a"), null, fromString("c"))).isNull();
- assertThat(concat(fromString("a"), null, null)).isNull();
- assertThat(concat(null, null, null)).isNull();
- assertThat(concat(fromString("数据"),
fromString("砖头"))).isEqualTo(fromString("数据砖头"));
+ // check reference same.
+ assertThat(str.toString()).isSameAs(javaStr);
+ }
}
- @TestTemplate
- void concatWsTest() {
- // Returns empty if the separator is null
- assertThat(concatWs(null, (BinaryStringData) null)).isNull();
- assertThat(concatWs(null, fromString("a"))).isNull();
-
- // If separator is null, concatWs should skip all null inputs and
never return null.
- BinaryStringData sep = fromString("哈哈");
- assertThat(concatWs(sep, empty)).isEqualTo(empty);
- assertThat(concatWs(sep,
fromString("ab"))).isEqualTo(fromString("ab"));
- assertThat(concatWs(sep, fromString("a"),
fromString("b"))).isEqualTo(fromString("a哈哈b"));
- assertThat(concatWs(sep, fromString("a"), fromString("b"),
fromString("c")))
- .isEqualTo(fromString("a哈哈b哈哈c"));
- assertThat(concatWs(sep, fromString("a"), null, fromString("c")))
- .isEqualTo(fromString("a哈哈c"));
- assertThat(concatWs(sep, fromString("a"), null,
null)).isEqualTo(fromString("a"));
- assertThat(concatWs(sep, null, null, null)).isEqualTo(empty);
- assertThat(concatWs(sep, fromString("数据"), fromString("砖头")))
- .isEqualTo(fromString("数据哈哈砖头"));
- }
+ @Nested
+ @DisplayName("Comparison")
+ class Comparison {
+
+ @ParameterizedTest(name = "{0}")
+ @EnumSource(Mode.class)
+ void compareTo(Mode mode) {
+ assertThat(fromString(mode, "
").compareTo(blankString(3))).isZero();
+ assertThat(fromString(mode, "").compareTo(fromString(mode,
"a"))).isLessThan(0);
+ assertThat(fromString(mode, "abc").compareTo(fromString(mode,
"ABC"))).isGreaterThan(0);
+ assertThat(fromString(mode, "abc0").compareTo(fromString(mode,
"abc")))
+ .isGreaterThan(0);
+ assertThat(fromString(mode,
"abcabcabc").compareTo(fromString(mode, "abcabcabc")))
+ .isZero();
+ assertThat(fromString(mode,
"aBcabcabc").compareTo(fromString(mode, "Abcabcabc")))
+ .isGreaterThan(0);
+ assertThat(fromString(mode,
"Abcabcabc").compareTo(fromString(mode, "abcabcabC")))
+ .isLessThan(0);
+ assertThat(fromString(mode,
"abcabcabc").compareTo(fromString(mode, "abcabcabC")))
+ .isGreaterThan(0);
+
+ assertThat(fromString(mode, "abc").compareTo(fromString(mode,
"世界"))).isLessThan(0);
+ assertThat(fromString(mode, "你好").compareTo(fromString(mode,
"世界"))).isGreaterThan(0);
+ assertThat(fromString(mode, "你好123").compareTo(fromString(mode,
"你好122")))
+ .isGreaterThan(0);
+
+ MemorySegment segment1 =
MemorySegmentFactory.allocateUnpooledSegment(1024);
+ MemorySegment segment2 =
MemorySegmentFactory.allocateUnpooledSegment(1024);
+ SortUtil.putStringNormalizedKey(fromString(mode, "abcabcabc"),
segment1, 0, 9);
+ SortUtil.putStringNormalizedKey(fromString(mode, "abcabcabC"),
segment2, 0, 9);
+ assertThat(segment1.compare(segment2, 0, 0, 9)).isGreaterThan(0);
+ SortUtil.putStringNormalizedKey(fromString(mode, "abcab"),
segment1, 0, 9);
+ assertThat(segment1.compare(segment2, 0, 0, 9)).isLessThan(0);
+ }
- @TestTemplate
- void contains() {
- assertThat(empty.contains(empty)).isTrue();
- assertThat(fromString("hello").contains(fromString("ello"))).isTrue();
-
assertThat(fromString("hello").contains(fromString("vello"))).isFalse();
-
assertThat(fromString("hello").contains(fromString("hellooo"))).isFalse();
- assertThat(fromString("大千世界").contains(fromString("千世界"))).isTrue();
- assertThat(fromString("大千世界").contains(fromString("世千"))).isFalse();
- assertThat(fromString("大千世界").contains(fromString("大千世界好"))).isFalse();
- }
+ @Test
+ void testMultiSegments() {
+
+ // prepare
+ MemorySegment[] segments1 = new MemorySegment[2];
+ segments1[0] = MemorySegmentFactory.wrap(new byte[10]);
+ segments1[1] = MemorySegmentFactory.wrap(new byte[10]);
+ segments1[0].put(5, "abcde".getBytes(UTF_8), 0, 5);
+ segments1[1].put(0, "aaaaa".getBytes(UTF_8), 0, 5);
+
+ MemorySegment[] segments2 = new MemorySegment[2];
+ segments2[0] = MemorySegmentFactory.wrap(new byte[5]);
+ segments2[1] = MemorySegmentFactory.wrap(new byte[5]);
+ segments2[0].put(0, "abcde".getBytes(UTF_8), 0, 5);
+ segments2[1].put(0, "b".getBytes(UTF_8), 0, 1);
+
+ // test go ahead both
+ BinaryStringData binaryString1 =
BinaryStringData.fromAddress(segments1, 5, 10);
+ BinaryStringData binaryString2 =
BinaryStringData.fromAddress(segments2, 0, 6);
+ assertThat(binaryString1).hasToString("abcdeaaaaa");
+ assertThat(binaryString2).hasToString("abcdeb");
+ assertThat(binaryString1.compareTo(binaryString2)).isEqualTo(-1);
+
+ // test needCompare == len
+ binaryString1 = BinaryStringData.fromAddress(segments1, 5, 5);
+ binaryString2 = BinaryStringData.fromAddress(segments2, 0, 5);
+ assertThat(binaryString1).hasToString("abcde");
+ assertThat(binaryString2).hasToString("abcde");
+ assertThat(binaryString1.compareTo(binaryString2)).isZero();
+
+ // test find the first segment of this string
+ binaryString1 = BinaryStringData.fromAddress(segments1, 10, 5);
+ binaryString2 = BinaryStringData.fromAddress(segments2, 0, 5);
+ assertThat(binaryString1).hasToString("aaaaa");
+ assertThat(binaryString2).hasToString("abcde");
+ assertThat(binaryString1.compareTo(binaryString2)).isEqualTo(-1);
+ assertThat(binaryString2.compareTo(binaryString1)).isEqualTo(1);
+
+ // test go ahead single
+ segments2 = new MemorySegment[] {MemorySegmentFactory.wrap(new
byte[10])};
+ segments2[0].put(4, "abcdeb".getBytes(UTF_8), 0, 6);
+ binaryString1 = BinaryStringData.fromAddress(segments1, 5, 10);
+ binaryString2 = BinaryStringData.fromAddress(segments2, 4, 6);
+ assertThat(binaryString1).hasToString("abcdeaaaaa");
+ assertThat(binaryString2).hasToString("abcdeb");
+ assertThat(binaryString1.compareTo(binaryString2)).isEqualTo(-1);
+ assertThat(binaryString2.compareTo(binaryString1)).isEqualTo(1);
+ }
- @TestTemplate
- void startsWith() {
- assertThat(empty.startsWith(empty)).isTrue();
-
assertThat(fromString("hello").startsWith(fromString("hell"))).isTrue();
-
assertThat(fromString("hello").startsWith(fromString("ell"))).isFalse();
-
assertThat(fromString("hello").startsWith(fromString("hellooo"))).isFalse();
- assertThat(fromString("数据砖头").startsWith(fromString("数据"))).isTrue();
- assertThat(fromString("大千世界").startsWith(fromString("千"))).isFalse();
-
assertThat(fromString("大千世界").startsWith(fromString("大千世界好"))).isFalse();
- }
+ @ParameterizedTest(name = "{0}")
+ @EnumSource(Mode.class)
+ void testEmptyString(Mode mode) {
+ BinaryStringData str2 = fromString(mode, "hahahahah");
+ BinaryStringData str3;
+ {
+ MemorySegment[] segments = new MemorySegment[2];
+ segments[0] = MemorySegmentFactory.wrap(new byte[10]);
+ segments[1] = MemorySegmentFactory.wrap(new byte[10]);
+ str3 = BinaryStringData.fromAddress(segments, 15, 0);
+ }
Review Comment:
upsi. Removed them
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
