zy-kkk commented on code in PR #68707: URL: https://github.com/apache/doris/pull/68707#discussion_r4172706518
########## regression-test/suites/external_table_p0/lance/test_lance_search_snapshot.groovy: ########## @@ -0,0 +1,412 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lance_search_snapshot", "p0,external") { + /* + * vector_search() and full_text_search() with the version, timestamp, tag and branch + * properties, on a Lance filesystem catalog. + * + * search_snapshot.lance (see lance_build_search_snapshot.py), main chain: + * version 1 row_id 1..8 fragment 0 committed 2026-09-29 14:51:51.410834 UTC + * version 2 row_id 9..16 fragment 1 committed 2026-09-29 14:51:52.929307 UTC + * version 3 vec_idx over fragments 0, 1 committed 2026-09-29 14:51:54.457459 UTC + * version 4 body_idx over fragments 0, 1 committed 2026-09-29 14:51:55.985957 UTC + * version 5 row_id 17..24 fragment 2 committed 2026-09-29 14:51:57.507267 UTC + * version 6 delete row_id 3, 11 committed 2026-09-29 14:51:59.031043 UTC + * version 7 vec_idx rebuilt over 0, 1, 2 committed 2026-09-29 14:52:00.565680 UTC + * Tag rel points at version 5. Branch dev forks from version 4; its version 5 appends + * row_id 101..108 as fragment 2, and its version 6 rebuilds body_idx over all fragments. + * Tag dev_rel points at dev version 5. vec is [row_id, row_id + 1, row_id + 2, row_id + 3], + * so the squared L2 distance between rows r and n is 4 * (n - r)^2, and nprobes=2 makes + * the two-partition IVF_FLAT index exact. body is "doc <row_id> lance" for even row_ids and + * "doc <row_id> doris" for odd ones. + * + * search_snapshot_pruned.lance lost version 1 to cleanup; version 2, pinned by tag kept, + * still opens but its vector index files are gone; version 3 rebuilt the index. + * + * search_snapshot_evolved.lance changes its schema after its vector index: version 1 has + * row_id 1..8 as fragment 0, version 2 builds vec_idx over it, version 3 adds note + * ("note <row_id>"), version 4 renames vec to embedding (vec_idx still serves it), and + * version 5 appends row_id 9..16 as fragment 1, which vec_idx does not cover. + */ + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Lance test because the Iceberg MinIO environment is disabled.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_lance_search_snapshot" + String table = "${catalogName}.`default`.search_snapshot" + String pruned = "${catalogName}.`default`.search_snapshot_pruned" + String evolved = "${catalogName}.`default`.search_snapshot_evolved" + def vectorSearch = { String selector, String query = "[0,1,2,3]", String topK = "3", + String target = table -> + """vector_search( + "table"="${target}", + "column"="vec", + "query_vector"="${query}", + "top_k"="${topK}", + "metric"="l2", + "nprobes"="2"${selector})""" + } + def fullTextSearch = { String selector, String coverageMode = "index_only", String query = "lance" -> + """full_text_search( + "table"="${table}", + "column"="body", + "query"="${query}", + "top_k"="30", + "coverage_mode"="${coverageMode}"${selector})""" + } + // Queries sit a quarter step past a row, so no two rows tie. + def evolvedSearch = { String column, String selector, String query = "[2.25,3.25,4.25,5.25]" -> + """vector_search( + "table"="${evolved}", + "column"="${column}", + "query_vector"="${query}", + "top_k"="3", + "metric"="l2", + "nprobes"="2"${selector})""" + } + def scannerV2Rows = sql """SHOW VARIABLES LIKE 'enable_file_scanner_v2'""" + String originalScannerV2 = scannerV2Rows[0][1].toString() + String originalTimeZone = (sql """SHOW VARIABLES LIKE 'time_zone'""")[0][1].toString() + String originalLazy = (sql """SHOW VARIABLES LIKE 'enable_lance_lazy_materialization'""")[0][1].toString() + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + try { + sql """SET enable_file_scanner_v2 = true""" + sql """SET time_zone = 'UTC'""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + "type" = "lance", + "lance.catalog.type" = "filesystem", + "warehouse" = "s3://warehouse/lance", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}";, + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.region" = "us-east-1", + "use_path_style" = "true" + ) + """ + + // Without a selector: the latest main version, where rows 3 and 11 are deleted. + qt_latest """SELECT row_id, _distance FROM ${vectorSearch("")} ORDER BY _distance, row_id""" + explain { + sql("SELECT row_id FROM ${vectorSearch("")}") + contains "lanceCatalogType=filesystem" + contains "lanceVersion=7" + contains "lanceManagedVersioning=false" + notContains "lanceBranch=" + contains "lanceSearchIndexFragments=3" + contains "lanceSearchUnindexedFragments=0" + } + + // Before any index: a flat search over that version's two fragments. + qt_version_2 """SELECT row_id, _distance FROM ${vectorSearch(', "version"="2"')} ORDER BY _distance, row_id""" + explain { + sql("SELECT row_id FROM ${vectorSearch(', "version"="2"')}") + contains "lanceVersion=2" + contains "lanceVectorIndexStatus=NO_MATCH" + contains "lanceSearchIndexSegments=0" + contains "lanceSearchUnindexedFragments=2" + } + test { + sql """SELECT row_id FROM ${fullTextSearch(', "version"="3"')}""" + exception "No committed Lance FTS index exists for column 'body' at dataset version 3" + } + + // Version 5 appended fragment 2 after both indexes: the vector search combines the index + // over fragments 0 and 1 with a flat search of fragment 2, and full-text search covers + // only the indexed fragments. + qt_version_5 """ + SELECT row_id, _distance FROM ${vectorSearch(', "version"="5"', "[17,18,19,20]")} + ORDER BY _distance, row_id + """ + explain { + sql("SELECT row_id FROM ${vectorSearch(', "version"="5"')}") + contains "lanceVersion=5" + contains "lanceSearchIndexFragments=2" + contains "lanceSearchUnindexedFragments=1" + } + order_qt_version_5_fts """SELECT row_id FROM ${fullTextSearch(', "version"="5"')}""" + test { + sql """SELECT row_id FROM ${fullTextSearch(', "version"="5"', "strict")}""" + exception "requires every fragment at dataset version 5 to be indexed" + } + order_qt_version_4_fts_strict """SELECT row_id FROM ${fullTextSearch(', "version"="4"', "strict")}""" + + // Version 6 deleted rows 3 and 11 and still uses the original index; version 7 rebuilt it. + // The query sits a quarter step past row 3, so no two rows tie. + qt_version_6 """ + SELECT row_id, _distance FROM ${vectorSearch(', "version"="6"', "[3.25,4.25,5.25,6.25]")} + ORDER BY _distance, row_id + """ + qt_version_5_not_deleted """ + SELECT row_id, _distance FROM ${vectorSearch(', "version"="5"', "[3.25,4.25,5.25,6.25]")} + ORDER BY _distance, row_id + """ + explain { + sql("SELECT row_id FROM ${vectorSearch(', "version"="6"')}") + contains "lanceSearchIndexFragments=2" + contains "lanceSearchUnindexedFragments=1" + } + // Full-text search sees the deletion too: rows 3 and 11 are "doris" rows. + order_qt_version_5_fts_doris """SELECT row_id FROM ${fullTextSearch(', "version"="5"', "index_only", "doris")}""" + order_qt_version_6_fts_doris """SELECT row_id FROM ${fullTextSearch(', "version"="6"', "index_only", "doris")}""" + + // use_index=false loads metadata without indexes, from the selected snapshot as well. + qt_version_5_no_index """ + SELECT row_id, _distance FROM ${vectorSearch(', "version"="5", "use_index"="false"', "[17,18,19,20]")} + ORDER BY _distance, row_id + """ + explain { + sql("SELECT row_id FROM ${vectorSearch(', "version"="5", "use_index"="false"')}") + contains "lanceVersion=5" + contains "lanceVectorIndexStatus=DISABLED" + contains "lanceSearchUnindexedFragments=3" + } + + // A tag and main written as a branch select main versions. + qt_tag_rel """ + SELECT row_id, _distance FROM ${vectorSearch(', "tag"="rel"', "[17,18,19,20]")} + ORDER BY _distance, row_id + """ + explain { + sql("SELECT row_id FROM ${vectorSearch(', "tag"="rel"')}") + contains "lanceVersion=5" + notContains "lanceBranch=" + } + qt_main_branch """ + SELECT row_id, _distance FROM ${vectorSearch(', "branch"="main", "version"="2"')} + ORDER BY _distance, row_id + """ + + // Branch dev: its version 6 shares main's version numbers and fragment ids, not its rows. + qt_branch_latest """ + SELECT row_id, _distance FROM ${vectorSearch(', "branch"="dev"', "[100,101,102,103]")} + ORDER BY _distance, row_id + """ + qt_main_same_query """ + SELECT row_id, _distance FROM ${vectorSearch(', "version"="6"', "[100,101,102,103]")} + ORDER BY _distance, row_id + """ + explain { + sql("SELECT row_id FROM ${vectorSearch(', "branch"="dev"')}") + contains "lanceVersion=6" + contains "lanceBranch=dev" + contains "lanceSearchIndexFragments=2" + contains "lanceSearchUnindexedFragments=1" + } + order_qt_branch_fts_strict """SELECT row_id FROM ${fullTextSearch(', "branch"="dev"', "strict")}""" + qt_branch_no_index """ + SELECT row_id, _distance + FROM ${vectorSearch(', "branch"="dev", "use_index"="false"', "[100,101,102,103]")} + ORDER BY _distance, row_id + """ + explain { + sql("SELECT row_id FROM ${vectorSearch(', "branch"="dev", "use_index"="false"')}") + contains "lanceVersion=6" + contains "lanceBranch=dev" + contains "lanceVectorIndexStatus=DISABLED" + } + qt_branch_fork """ + SELECT row_id, _distance FROM ${vectorSearch(', "branch"="dev", "version"="4"', "[100,101,102,103]")} Review Comment: Agreed: the fork-point version and the future timestamp could not tell the branch from main, or the time from the head. Added in e55bd497a45: - `branch=dev, version=5` returns the branch's rows 101/102/103 (main version 5 would return 24/23/22), with EXPLAIN `lanceVersion=5` and `lanceBranch=dev`. - STRICT `full_text_search` succeeds on `branch=dev, version=6`, whose `body_idx` was rebuilt over fragment 2, and fails on main version 6. - `branch=dev, timestamp=2026-09-29 14:52:18`, between dev versions 5 and 6, gives EXPLAIN `lanceVersion=5`, and STRICT full-text search fails naming `dataset version 5 of branch 'dev'`. Ignoring the time would select dev version 6, and resolving it on main would select main version 7. All 29 suites in `external_table_p0/lance` pass locally. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
