github-actions[bot] commented on code in PR #65281:
URL: https://github.com/apache/doris/pull/65281#discussion_r3534257903
##########
regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy:
##########
@@ -74,9 +373,200 @@ suite("test_iceberg_deletion_vector", "p0,external") {
qt_q19 """ SELECT count(value) FROM dv_test_1w where id%2 = 1; """
qt_q20 """ SELECT count(id) FROM dv_test_1w where id%3 = 1; """
qt_q21 """ SELECT count(ts) FROM dv_test_1w where id%3 != 1; """
+ qt_q22 """ SELECT * FROM dv_test ORDER BY data DESC LIMIT 3; """
+ qt_q23 """ SELECT * FROM dv_test_v2 ORDER BY data DESC LIMIT 3; """
+ qt_q24 """ SELECT batch, count(*), sum(id) FROM dv_test GROUP BY batch
ORDER BY batch; """
+ qt_q25 """ SELECT batch, count(*), sum(id) FROM dv_test_v2 GROUP BY batch
ORDER BY batch; """
+ qt_q26 """ SELECT batch, count(*), sum(id) FROM dv_test_orc GROUP BY batch
ORDER BY batch; """
+ qt_q27 """ SELECT id, data FROM dv_test WHERE id BETWEEN 2 AND 10 ORDER BY
id LIMIT 2; """
+ qt_q28 """ SELECT id, data FROM dv_test_v2 WHERE id BETWEEN 2 AND 10 ORDER
BY id LIMIT 2; """
+ qt_q29 """ SELECT id, data FROM dv_test_orc WHERE id BETWEEN 2 AND 10
ORDER BY id LIMIT 2; """
+ qt_q30 """ SELECT count(*) FROM (SELECT * FROM dv_test_1w WHERE id % 7 = 0
ORDER BY id LIMIT 100) t; """
+ qt_q31 """ SELECT count(*) FROM (SELECT * FROM dv_test_1w WHERE id % 11 =
0 ORDER BY id LIMIT 100) t; """
+
+ // Metadata matrix verifies the physical delete-file content types that
back the logical
+ // scenario checks above.
+ qt_delete_type_matrix """
+ SELECT 'dv_equality' scenario, content, file_format, count(*) files,
sum(record_count) records
+ FROM `dv_delete_matrix_equality_and_dv\$files`
+ GROUP BY content, file_format
+ UNION ALL
+ SELECT 'dv_only' scenario, content, file_format, count(*) files,
sum(record_count) records
+ FROM `dv_test\$files`
+ GROUP BY content, file_format
+ UNION ALL
+ SELECT 'dv_position' scenario, content, file_format, count(*) files,
sum(record_count) records
+ FROM `dv_delete_matrix_position_and_dv\$files`
+ GROUP BY content, file_format
+ UNION ALL
+ SELECT 'equality_only' scenario, content, file_format, count(*) files,
sum(record_count) records
+ FROM `dv_delete_matrix_equality_only\$files`
+ GROUP BY content, file_format
+ UNION ALL
+ SELECT 'no_delete' scenario, content, file_format, count(*) files,
sum(record_count) records
+ FROM `dv_test_no_delete\$files`
+ GROUP BY content, file_format
+ UNION ALL
+ SELECT 'position_only' scenario, content, file_format, count(*) files,
sum(record_count) records
+ FROM `dv_delete_matrix_position_only\$files`
+ GROUP BY content, file_format
+ ORDER BY scenario, content, file_format;
+ """
+ // Iceberg v3 semantics: when a data file has a DV, old position deletes
for that file must be
+ // ignored, while equality deletes still filter matching rows.
+ qt_position_delete_ignored_with_dv """
+ SELECT id, name
+ FROM dv_delete_matrix_position_and_dv
+ WHERE id IN (1, 2, 5, 10, 15)
+ ORDER BY id, name
+ LIMIT 20;
+ """
+ qt_equality_delete_active_with_dv """
+ SELECT id, batch, data
+ FROM dv_delete_matrix_equality_and_dv
+ ORDER BY id;
+ """
+
+ def normalizeExternalRows = { String output ->
+ return output.readLines()
+ .collect { it.trim() }
+ .findAll { it ==~ /^-?[0-9].*/ && it.contains("\t") }
+ .join("\n")
+ }
+ String expectedRows = ["2\t1\tb", "4\t1\td", "6\t1\tf", "8\t1\th",
"12\t2\tl"].join("\n")
+ String sparkRows = normalizeExternalRows(runSparkSql(
+ "use format_v3; select id, batch, data from
dv_delete_matrix_equality_and_dv order by id;",
+ 300
+ ))
+ assertEquals(expectedRows, sparkRows)
+ String javaRows = normalizeExternalRows(runInSparkContainer(
+ "java -cp \"/tmp:/opt/spark/jars/*\" ReadIcebergRows " +
+ "format_v3 dv_delete_matrix_equality_and_dv id batch data",
+ 300
+ ))
+ assertEquals(expectedRows, javaRows)
+
+ String trinoContainerName = findRequiredDockerContainer(
+ "Trino", "icebergTrinoContainer",
+ "test -d /etc/trino/catalog && command -v trino >/dev/null")
+ String trinoExternalEnvIp = externalEnvIp
+ if (trinoExternalEnvIp == "127.0.0.1" ||
trinoExternalEnvIp.equalsIgnoreCase("localhost")) {
+ trinoExternalEnvIp = executeCommand("hostname -I | cut -d' ' -f1",
true, 30)?.trim()
+ }
+ String trinoCatalogProps = """
+connector.name=iceberg
+iceberg.catalog.type=rest
+iceberg.rest-catalog.uri=http://${trinoExternalEnvIp}:${rest_port}
+fs.native-s3.enabled=true
+s3.endpoint=http://${trinoExternalEnvIp}:${minio_port}
+s3.aws-access-key=admin
+s3.aws-secret-key=password
+s3.region=us-east-1
+s3.path-style-access=true
+"""
+ String encodedTrinoCatalogProps = encodeBase64(trinoCatalogProps)
+ executeCommand(
+ "${dockerCommand} exec ${trinoContainerName} bash -lc " +
+ "'echo ${encodedTrinoCatalogProps} | " +
+ "base64 -d >/etc/trino/catalog/iceberg.properties'",
+ true,
+ 30
+ )
+ executeCommand("${dockerCommand} restart ${trinoContainerName}", true, 60)
+ String trinoRows = ""
+ for (int i = 0; i < 12; i++) {
+ Thread.sleep(5000)
+ trinoRows = normalizeExternalRows(executeCommand(
+ "${dockerCommand} exec ${trinoContainerName} trino
--output-format TSV " +
+ "--catalog iceberg --schema format_v3 --execute " +
+ "\"SELECT id, batch, data " +
+ "FROM dv_delete_matrix_equality_and_dv ORDER BY id\"",
+ false,
+ 120
+ ))
+ if (!trinoRows.isEmpty()) {
+ break
+ }
+ }
+ assertEquals(expectedRows, trinoRows)
+ def profileCounterMax = { String profileText, String counterName ->
+ long maxValue = Long.MIN_VALUE
+ profileText.readLines().findAll { it.contains(counterName + ":")
}.each { line ->
+ def matcher = line =~ /\(([0-9,]+)\)/
+ String rawValue
+ if (matcher.find()) {
+ rawValue = matcher.group(1)
+ } else {
+ matcher = line =~ /:\s*([0-9,]+)/
+ rawValue = matcher.find() ? matcher.group(1) : null
+ }
+ if (rawValue != null) {
+ maxValue = Math.max(maxValue,
Long.parseLong(rawValue.replace(",", "")))
+ }
+ }
+ return maxValue == Long.MIN_VALUE ? 0L : maxValue
+ }
+
+ def profileInfoValueCount = { String profileText, String infoName ->
+ def matcher = profileText =~ (java.util.regex.Pattern.quote(infoName)
+ ":\\s*\\[([^\\]]*)\\]")
+ if (!matcher.find()) {
+ return 0
+ }
+ return matcher.group(1).split(",").collect { it.trim() }.findAll {
!it.isEmpty() }.size()
+ }
+
+ // Split/cache scenario for 3.4: this query scans one data file with
file_split_size=1. The
+ // result is captured by qt_split_cache_single_file_count; the profile
assertion checks the DV
+ // rows were materialized once for the shared scan-node cache, not once
per split.
+ String splitCacheProfileTag = "iceberg_dv_split_cache_" +
UUID.randomUUID().toString()
+ profile(splitCacheProfileTag) {
+ run {
+ sql """
+ /* ${splitCacheProfileTag} */
+ SELECT count(*), sum(id)
+ FROM dv_split_cache_single_file;
+ """
+ // The detailed scanner counters are populated asynchronously
after the query returns.
+ Thread.sleep(5000)
+ }
+ check { profileString, exception ->
+ if (exception != null) {
+ throw exception
+ }
+ long numDeleteRows = profileCounterMax(profileString,
"NumDeleteRows")
Review Comment:
The profile assertion here does not prove the shared DV cache is used.
`profileCounterMax` only takes the largest `NumDeleteRows` value it sees. With
the intended behavior one scanner can parse the 2048-row DV and the other split
scanners hit the cache, so the max is 2048; with the regression this test is
trying to catch, every split can reparse the same 2048-row DV and the max is
still 2048. Since `scannerCount > 1` only proves multiple scanners existed,
this test would still pass if cache sharing broke. Please assert the number or
sum of nonzero `NumDeleteRows` counters, or add a direct cache read/hit
counter, so the test fails when the DV is materialized once per split.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
