github-actions[bot] commented on code in PR #70408:
URL: https://github.com/apache/doris/pull/70408#discussion_r4657733714
##########
be/src/exec/common/variant_util.cpp:
##########
@@ -183,6 +183,43 @@ PathToDataTypes collect_regular_types_outside_nested_group(
return regular_path_to_data_types;
}
+namespace {
+
+// The dynamic paths of a doc column, chosen once for all doc buckets: each
bucket writer sees only
+// its own paths. Typed paths do not count.
+void choose_doc_compaction_subpaths(const TabletSchema& target, const
TabletColumn& column,
+ const PathToNoneNullValues& path_stats,
+ VariantCompactionPaths& paths_set_info) {
+ PathToNoneNullValues dynamic_path_stats;
+ for (const auto& [path, size] : path_stats) {
+ TabletSchema::SubColumnInfo typed_info;
+ if (!generate_sub_column_info(target, column.unique_id(), path,
&typed_info)) {
+ dynamic_path_stats.emplace(path, size);
+ }
+ }
+ VariantCompactionUtil::get_subpaths(column.variant_max_subcolumns_count(),
Review Comment:
[P2] Use the writer's tie-break order for doc compaction path selection. The
new `choose_doc_compaction_subpaths()` calls `get_subpaths()`, whose
equal-count ordering is only raw path bytes. Initial writes use
`select_variant_paths()`, which prefers greater dotted depth before the path.
With `variant_max_subcolumns_count=1` and equal-frequency paths `z` and `a.b`,
a new segment extracts `a.b` while a later doc compaction extracts `z`, moving
the inherited subcolumn/index even though both values remain in the doc
snapshot. Share the depth-then-path ranking (and add an equal-frequency
doc-mode regression) so compaction preserves the writer's selection.
##########
regression-test/suites/variant_p0/test_variant_count_zero_keeps_paths_sparse.groovy:
##########
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_count_zero_keeps_paths_sparse") {
+ sql " set debug_variant_enable_doc_mode = false "
+ sql " set debug_variant_doc_materialization_min_rows = 0 "
+ sql """ set debug_variant_enable_typed_paths_to_sparse = false """
+ sql """ set debug_variant_max_sparse_column_statistics_size = 10 """
+ sql """ set debug_variant_doc_hash_shard_count = 0 """
+ sql """ set describe_extend_variant_column = true """
+
+ // a count of 0 keeps every dynamic path sparse, while template paths and
a positive count give subcolumns
+ sql "DROP TABLE IF EXISTS test_variant_count_zero"
+ sql """
+ CREATE TABLE test_variant_count_zero (
+ k INT,
+ zero VARIANT<PROPERTIES("variant_max_subcolumns_count" = "0")>,
+ typed VARIANT<'t' : BIGINT,
PROPERTIES("variant_max_subcolumns_count" = "0")>,
+ counted VARIANT<PROPERTIES("variant_max_subcolumns_count" =
"2048")>
+ )
+ DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES ("replication_num" = "1", "disable_auto_compaction" =
"true")
+ """
+ sql """INSERT INTO test_variant_count_zero VALUES
+ (1, parse_to_variant('{"a": 1, "b": "x"}'), parse_to_variant('{"t": 1,
"a": 1}'), parse_to_variant('{"a": 1}'))"""
+ sql """INSERT INTO test_variant_count_zero VALUES
+ (2, parse_to_variant('{"a": 2, "c": {"d": 2}}'),
parse_to_variant('{"t": 2, "b": "y"}'), parse_to_variant('{"b": 2}'))"""
+ qt_desc_load "DESC test_variant_count_zero"
+ order_qt_read_load """SELECT k, zero, zero['a'], zero['c']['d'],
typed['t'], typed['b'], counted['a']
+ FROM test_variant_count_zero"""
+
+ trigger_and_wait_compaction("test_variant_count_zero", "full")
+ qt_desc_compacted "DESC test_variant_count_zero"
+ order_qt_read_compacted """SELECT k, zero, zero['a'], zero['c']['d'],
typed['t'], typed['b'], counted['a']
+ FROM test_variant_count_zero"""
+
+ // a column added to the table has the same meaning, and a table created
from it too
+ sql """ALTER TABLE test_variant_count_zero
+ ADD COLUMN added VARIANT<PROPERTIES("variant_max_subcolumns_count" =
"0")>"""
+ waitForSchemaChangeDone {
+ sql """SHOW ALTER TABLE COLUMN WHERE TableName =
'test_variant_count_zero' ORDER BY CreateTime DESC LIMIT 1"""
+ time 600
+ }
+ sql """INSERT INTO test_variant_count_zero VALUES
+ (3, parse_to_variant('{"a": 3}'), parse_to_variant('{"t": 3}'),
parse_to_variant('{"a": 3}'), parse_to_variant('{"e": 3}'))"""
+ trigger_and_wait_compaction("test_variant_count_zero", "full")
+ qt_desc_added "DESC test_variant_count_zero"
+ order_qt_read_added "SELECT k, added, added['e'] FROM
test_variant_count_zero"
+
+ sql "DROP TABLE IF EXISTS test_variant_count_zero_ctas"
+ sql """
+ CREATE TABLE test_variant_count_zero_ctas
+ DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES ("replication_num" = "1") AS SELECT * FROM
test_variant_count_zero
+ """
+ qt_desc_ctas "DESC test_variant_count_zero_ctas"
+ order_qt_read_ctas "SELECT k, zero['a'], typed['t'], added['e'] FROM
test_variant_count_zero_ctas"
+
+ // CTAS creates a new column with the count of the query's type: the count
of 0 of a VARIANT expression
+ // then extracts no dynamic path
+ sql "DROP TABLE IF EXISTS test_variant_count_zero_ctas_expr"
+ sql """
+ CREATE TABLE test_variant_count_zero_ctas_expr
+ DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES ("replication_num" = "1") AS SELECT 1 AS k,
parse_to_variant('{"a": 1, "b": "x"}') AS v
+ """
+ qt_desc_ctas_expr "DESC test_variant_count_zero_ctas_expr"
+ order_qt_read_ctas_expr "SELECT k, v, v['a'] FROM
test_variant_count_zero_ctas_expr"
+
+ // the serialize type behind this is internal
+ test {
+ sql """
+ CREATE TABLE test_variant_serialize_type_rejected (
+ k INT,
+ v VARIANT<PROPERTIES("variant_serialize_type" = "v1")>
+ )
+ DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ exception "only support for"
+ }
+
+ // SHOW CREATE TABLE shows it in a comment, which the parser skips
+ def zeroDdl = sql("SHOW CREATE TABLE test_variant_count_zero")[0][1]
+ assertTrue(zeroDdl.contains("`zero` variant<PROPERTIES
(\"variant_max_subcolumns_count\" = \"0\","), zeroDdl)
Review Comment:
[P2] Keep the SHOW CREATE assertion in the regression output workflow. Lines
98-100 use raw `assertTrue` calls for deterministic SQL text, while the
repository test rules require `qt_sql`/`order_qt` (and generated `.out` output)
instead of Groovy assertions. Capture SHOW CREATE with a query/output block so
the expected text is generated and reviewed like the other regression results.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]