github-actions[bot] commented on code in PR #68775:
URL: https://github.com/apache/doris/pull/68775#discussion_r4227702517
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java:
##########
@@ -9610,6 +9610,28 @@ public LogicalPlan
visitWarmUpSelect(DorisParser.WarmUpSelectContext ctx) {
}
LogicalProject project = new LogicalProject(projectList, filter);
+ Map<String, String> properties = visitPropertyClause(ctx.properties);
+ for (String key : properties.keySet()) {
+ if (!"read_index_only".equals(key)) {
+ throw new AnalysisException("Unknown WARM UP SELECT property:
" + key);
+ }
+ }
+ String indexOnly = properties.getOrDefault("read_index_only", "false");
+ if (!"true".equalsIgnoreCase(indexOnly) &&
!"false".equalsIgnoreCase(indexOnly)) {
+ throw new AnalysisException("read_index_only must be true or
false");
+ }
+ if (Boolean.parseBoolean(indexOnly)) {
+ if (ctx.whereClause() != null || ctx.explain() != null) {
+ throw new AnalysisException("Index-only WARM UP SELECT does
not support WHERE or EXPLAIN");
+ }
+ // Index cache fills must not execute the data scan or require the
data-file cache.
+ List<String> columns = indexWarmupColumns(ctx, projectList);
+ UnboundBlackholeSink<?> sink = new UnboundBlackholeSink<>(project,
+ new UnboundBlackholeSinkContext(true));
+ return new WarmupSelectCommand(sink,
+ new
TableNameInfo(visitMultipartIdentifier(ctx.warmUpSingleTableRef().multipartIdentifier())),
Review Comment:
[P2] Reject overqualified table names before constructing `TableNameInfo`.
`multipartIdentifier` accepts four or more components, but
`TableNameInfo(List)` silently keeps only the last three. For `WARM UP SELECT *
FROM ignored.lake.db.items PROPERTIES (read_index_only=true)`, this path
prewarms `lake.db.items` and reports success for that different table, while
normal relation binding rejects the four-part name. Validate one to three
components before taking the index-only branch.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexPrewarm.java:
##########
@@ -0,0 +1,274 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance;
+
+import org.apache.doris.analysis.ResourceTypeEnum;
+import org.apache.doris.analysis.TableName;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.lance.index.LanceIndexInspectionExecutor;
+import org.apache.doris.datasource.lance.index.LanceIndexSegmentGroup;
+import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
+import org.apache.doris.info.TableNameInfo;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.proto.InternalService.PLanceIndexPrewarmRequest;
+import org.apache.doris.proto.InternalService.PLanceIndexPrewarmResponse;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.resource.computegroup.ComputeGroup;
+import org.apache.doris.rpc.BackendServiceProxy;
+import org.apache.doris.system.Backend;
+import org.apache.doris.system.BeSelectionPolicy;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TStatusCode;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.OptionalInt;
+import java.util.Set;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.BooleanSupplier;
+
+/** Pins table access and coordinates bounded synchronous prewarm RPCs. */
+public final class LanceIndexPrewarm {
+ static final int MAX_IN_FLIGHT = 8;
+ private static final ShowResultSetMetaData RESULT_META =
ShowResultSetMetaData.builder()
+ .addColumn(new Column("Table", ScalarType.createStringType()))
+ .addColumn(new Column("Index", ScalarType.createStringType()))
+ .addColumn(new Column("DatasetVersion", ScalarType.BIGINT))
+ .addColumn(new Column("BackendCount", ScalarType.INT))
+ .addColumn(new Column("ElapsedMs", ScalarType.BIGINT)).build();
+
+ private LanceIndexPrewarm() {
+ }
+
+ public static ShowResultSetMetaData getResultSetMetaData() {
+ return RESULT_META;
+ }
+
+ public static void run(ConnectContext ctx, StmtExecutor executor,
TableNameInfo tableName,
+ List<String> columns, BooleanSupplier cancelled) throws Exception {
+ long started = System.nanoTime();
+ long elapsedMs = ctx.getStartTime() > 0 ? Math.max(0,
System.currentTimeMillis() - ctx.getStartTime()) : 0;
+ long remainingMs = TimeUnit.SECONDS.toMillis(ctx.getQueryTimeoutS()) -
elapsedMs;
+ long deadline = started + TimeUnit.MILLISECONDS.toNanos(Math.max(0,
remainingMs));
+ tableName.analyze(ctx);
+ checkPrivileges(ctx, tableName);
+ List<Backend> targets = selectBackends(resolveComputeGroup(ctx));
+ // Resolve catalog/table initialization and snapshot IO in the bounded
metadata pool.
+ // The worker owns native resources until it returns, even if this
caller stops waiting.
+ String catalogName = tableName.getCtl();
+ String databaseName = tableName.getDb();
+ String name = tableName.getTbl();
+ LanceTableMetadata metadata = LanceIndexInspectionExecutor.execute(()
-> {
+ CatalogIf<?> catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName);
+ if (!(catalog instanceof LanceExternalCatalog)) {
+ throw new AnalysisException("Index-only WARM UP SELECT
requires a Lance catalog table");
+ }
+ TableIf table =
catalog.getDbOrAnalysisException(databaseName).getTableOrAnalysisException(name);
+ if (!(table instanceof LanceExternalTable)) {
+ throw new AnalysisException("Index-only WARM UP SELECT
requires a Lance catalog table");
+ }
+ checkActive(deadline, cancelled);
+ // Use the query access path, including REST-vended credentials,
exactly once.
+ return ((LanceExternalTable) table).loadMetadata();
Review Comment:
[P2] Avoid loading every data fragment for index-only prewarm.
`loadMetadata()` uses the normal scan metadata scope, and
`LanceMetadataLoader.read()` calls `dataset.getFragments()` and creates one
`LanceFragmentInfo` per fragment before it reads indexes. This command never
uses the fragment list, so a table with many small fragments spends O(fragment
count) JNI work and FE heap under the prewarm deadline before any BE cache
fill. Read the pinned schema/index/access metadata without materializing scan
fragments for this command.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexPrewarm.java:
##########
@@ -0,0 +1,274 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance;
+
+import org.apache.doris.analysis.ResourceTypeEnum;
+import org.apache.doris.analysis.TableName;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.lance.index.LanceIndexInspectionExecutor;
+import org.apache.doris.datasource.lance.index.LanceIndexSegmentGroup;
+import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
+import org.apache.doris.info.TableNameInfo;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.proto.InternalService.PLanceIndexPrewarmRequest;
+import org.apache.doris.proto.InternalService.PLanceIndexPrewarmResponse;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.resource.computegroup.ComputeGroup;
+import org.apache.doris.rpc.BackendServiceProxy;
+import org.apache.doris.system.Backend;
+import org.apache.doris.system.BeSelectionPolicy;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TStatusCode;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.OptionalInt;
+import java.util.Set;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.BooleanSupplier;
+
+/** Pins table access and coordinates bounded synchronous prewarm RPCs. */
+public final class LanceIndexPrewarm {
+ static final int MAX_IN_FLIGHT = 8;
+ private static final ShowResultSetMetaData RESULT_META =
ShowResultSetMetaData.builder()
+ .addColumn(new Column("Table", ScalarType.createStringType()))
+ .addColumn(new Column("Index", ScalarType.createStringType()))
+ .addColumn(new Column("DatasetVersion", ScalarType.BIGINT))
+ .addColumn(new Column("BackendCount", ScalarType.INT))
+ .addColumn(new Column("ElapsedMs", ScalarType.BIGINT)).build();
+
+ private LanceIndexPrewarm() {
+ }
+
+ public static ShowResultSetMetaData getResultSetMetaData() {
+ return RESULT_META;
+ }
+
+ public static void run(ConnectContext ctx, StmtExecutor executor,
TableNameInfo tableName,
+ List<String> columns, BooleanSupplier cancelled) throws Exception {
+ long started = System.nanoTime();
+ long elapsedMs = ctx.getStartTime() > 0 ? Math.max(0,
System.currentTimeMillis() - ctx.getStartTime()) : 0;
+ long remainingMs = TimeUnit.SECONDS.toMillis(ctx.getQueryTimeoutS()) -
elapsedMs;
Review Comment:
[P2] Use the effective execution timeout for index prewarm. This command
extends `InsertIntoTableCommand`, so `StmtExecutor.isSyncLoadKindStmt()` makes
`ConnectContext.checkTimeout()` and `getExecTimeoutS()` allow
`max(insert_timeout, query_timeout)` (defaults: 4 h versus 15 min). This
separate deadline uses only `getQueryTimeoutS()`, so a valid index prewarm that
takes over 15 minutes fails while the statement still has hours remaining.
Derive the deadline from `ctx.getExecTimeoutS()` or classify this mode
consistently as a query.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
