This is an automated email from the ASF dual-hosted git repository.
liaoxin01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 058dfd3794a [opt](schema) Optimize tables schema scan status (#64933)
058dfd3794a is described below
commit 058dfd3794aca46617766173ced5ac5574efff94
Author: Refrain <[email protected]>
AuthorDate: Tue Jul 7 09:49:17 2026 +0800
[opt](schema) Optimize tables schema scan status (#64933)
Optimize information_schema.tables status scan for large OlapTables.
Before this PR, FE always filled the full table status payload for
information_schema.tables, so even queries that only projected
lightweight columns such as UPDATE_TIME still paid the cost of
tablet-derived status fields.
This PR includes the following optimizations:
| Optimization | Description |
| --- | --- |
| Column pruning for table status RPC | BE passes the required
information_schema.tables columns to FE, and FE only fills requested
optional status fields. Lightweight queries can skip TABLE_ROWS,
DATA_LENGTH, AVG_ROW_LENGTH, and INDEX_LENGTH. |
| Single tablet-status traversal | TABLE_ROWS, DATA_LENGTH,
AVG_ROW_LENGTH, and INDEX_LENGTH are computed together when any of them
is requested, avoiding repeated OlapTable tablet/replica traversal. |
This PR does not add a table-level cache for tablet-derived status
fields. The benchmark still shows the possible room for that future
optimization: the full status query remains slower than the
UPDATE_TIME-only query because it still needs to read tablet-derived
status, while the lightweight query can skip it completely.
Benchmark on a table with 3900 partitions and 21 buckets per partition,
about 81,900 tablets, workers=20, targetQps=100:
| Query | Before avgQps | After avgQps | Improvement | Notes |
| --- | ---: | ---: | ---: | --- |
| SHOW TABLE STATUS FROM regression_test_stress_load_usercase LIKE ... |
26.3 | 49.1 | +86.7% | Full status query. It still needs tablet-derived
status fields, but benefits from the single traversal optimization. |
| SELECT TABLE_NAME, UPDATE_TIME FROM information_schema.tables WHERE
TABLE_SCHEMA = ... | 29.3 | 92.2 | +214.7% | Lightweight projection. It
benefits from column pruning and skips tablet-derived status fields. |
---
be/src/exec/operator/schema_scan_operator.cpp | 3 +
be/src/information_schema/schema_scanner.h | 2 +
.../information_schema/schema_tables_scanner.cpp | 1 +
.../java/org/apache/doris/catalog/LocalTablet.java | 32 +++-
.../java/org/apache/doris/catalog/OlapTable.java | 48 +++---
.../java/org/apache/doris/catalog/TableIf.java | 34 ++++
.../apache/doris/service/FrontendServiceImpl.java | 60 +++++--
.../org/apache/doris/catalog/OlapTableTest.java | 36 +++++
.../java/org/apache/doris/catalog/TabletTest.java | 17 ++
.../doris/service/FrontendServiceImplTest.java | 176 +++++++++++++++++++++
gensrc/thrift/FrontendService.thrift | 3 +
.../test_tables_status_column_pruning.groovy | 97 ++++++++++++
12 files changed, 470 insertions(+), 39 deletions(-)
diff --git a/be/src/exec/operator/schema_scan_operator.cpp
b/be/src/exec/operator/schema_scan_operator.cpp
index 95ee374824c..3ede2483750 100644
--- a/be/src/exec/operator/schema_scan_operator.cpp
+++ b/be/src/exec/operator/schema_scan_operator.cpp
@@ -25,6 +25,7 @@
#include "core/data_type/data_type_factory.hpp"
#include "exec/operator/operator.h"
#include "runtime/runtime_profile.h"
+#include "util/string_util.h"
namespace doris {
class RuntimeState;
@@ -186,6 +187,8 @@ Status SchemaScanOperatorX::prepare(RuntimeState* state) {
for (; j < columns_desc.size(); ++j) {
if (boost::iequals(_dest_tuple_desc->slots()[i]->col_name(),
columns_desc[j].name)) {
_slot_offsets[i] = j;
+ _common_scanner_param->required_columns.insert(
+ to_upper(_dest_tuple_desc->slots()[i]->col_name()));
break;
}
}
diff --git a/be/src/information_schema/schema_scanner.h
b/be/src/information_schema/schema_scanner.h
index aa32f1639b3..31662be9391 100644
--- a/be/src/information_schema/schema_scanner.h
+++ b/be/src/information_schema/schema_scanner.h
@@ -24,6 +24,7 @@
#include <cstddef>
#include <cstdint>
#include <memory>
+#include <set>
#include <string>
#include <vector>
@@ -69,6 +70,7 @@ struct SchemaScannerCommonParam {
int64_t thread_id;
const std::string* catalog = nullptr;
std::set<TNetworkAddress> fe_addr_list;
+ std::set<std::string> required_columns;
};
// scanner parameter from frontend
diff --git a/be/src/information_schema/schema_tables_scanner.cpp
b/be/src/information_schema/schema_tables_scanner.cpp
index 947097906e1..ef94909fd73 100644
--- a/be/src/information_schema/schema_tables_scanner.cpp
+++ b/be/src/information_schema/schema_tables_scanner.cpp
@@ -114,6 +114,7 @@ Status SchemaTablesScanner::_get_new_table() {
if (nullptr != _param->common_param->table) {
table_params.__set_table(*(_param->common_param->table));
}
+
table_params.__set_required_columns(_param->common_param->required_columns);
if (nullptr != _param->common_param->current_user_ident) {
table_params.__set_current_user_ident(*(_param->common_param->current_user_ident));
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTablet.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTablet.java
index 5db0a1286b5..8c1c8016b33 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTablet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTablet.java
@@ -34,7 +34,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
-import java.util.stream.LongStream;
public class LocalTablet extends Tablet {
private static final Logger LOG = LogManager.getLogger(LocalTablet.class);
@@ -133,17 +132,34 @@ public class LocalTablet extends Tablet {
// due to dataSize not write to image
@Override
public long getDataSize(boolean singleReplica, boolean filterSizeZero) {
- LongStream s = getReplicas().stream().filter(r -> r.getState() ==
ReplicaState.NORMAL)
- .filter(r -> !filterSizeZero || r.getDataSize() > 0)
- .mapToLong(Replica::getDataSize);
- return singleReplica ?
Double.valueOf(s.average().orElse(0)).longValue() : s.sum();
+ long dataSize = 0;
+ int replicaNum = 0;
+ for (Replica replica : getReplicas()) {
+ if (replica.getState() != ReplicaState.NORMAL) {
+ continue;
+ }
+ long replicaDataSize = replica.getDataSize();
+ if (filterSizeZero && replicaDataSize <= 0) {
+ continue;
+ }
+ dataSize += replicaDataSize;
+ replicaNum++;
+ }
+ return singleReplica && replicaNum > 0 ? dataSize / replicaNum :
dataSize;
}
@Override
public long getRowCount(boolean singleReplica) {
- LongStream s = getReplicas().stream().filter(r -> r.getState() ==
ReplicaState.NORMAL)
- .mapToLong(Replica::getRowCount);
- return singleReplica ?
Double.valueOf(s.average().orElse(0)).longValue() : s.sum();
+ long rowCount = 0;
+ int replicaNum = 0;
+ for (Replica replica : getReplicas()) {
+ if (replica.getState() != ReplicaState.NORMAL) {
+ continue;
+ }
+ rowCount += replica.getRowCount();
+ replicaNum++;
+ }
+ return singleReplica && replicaNum > 0 ? rowCount / replicaNum :
rowCount;
}
// Get the least row count among all valid replicas.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 970999ef80f..c8d6c515b29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1911,37 +1911,43 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
@Override
public long getAvgRowLength() {
- long rowCount = 0;
- long dataSize = 0;
- for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
- rowCount += entry.getValue().getBaseIndex().getRowCount();
- dataSize += entry.getValue().getBaseIndex().getDataSize(false,
false);
- }
- if (rowCount > 0) {
- return dataSize / rowCount;
- } else {
- return 0;
- }
+ return getTableStatusStats().getAvgRowLength();
}
@Override
public long getDataLength() {
- long dataSize = 0;
- for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
- dataSize += entry.getValue().getBaseIndex().getLocalSegmentSize();
- dataSize += entry.getValue().getBaseIndex().getRemoteSegmentSize();
- }
- return dataSize;
+ return getTableStatusStats().getDataLength();
}
@Override
public long getIndexLength() {
- long indexSize = 0;
+ return getTableStatusStats().getIndexLength();
+ }
+
+ @Override
+ public TableStatusStats getTableStatusStats() {
+ long rowCount = 0;
+ long rowCountForAvg = 0;
+ long dataSizeForAvg = 0;
+ long dataLength = 0;
+ long indexLength = 0;
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
- indexSize += entry.getValue().getBaseIndex().getLocalIndexSize();
- indexSize += entry.getValue().getBaseIndex().getRemoteIndexSize();
+ MaterializedIndex baseIndex = entry.getValue().getBaseIndex();
+ long baseIndexRowCount = baseIndex.getRowCount();
+ rowCount += baseIndexRowCount == UNKNOWN_ROW_COUNT ? 0 :
baseIndexRowCount;
+ rowCountForAvg += baseIndexRowCount;
+ for (Tablet tablet : baseIndex.getTablets()) {
+ for (Replica replica : tablet.getReplicas()) {
+ if (replica.getState() == Replica.ReplicaState.NORMAL) {
+ dataSizeForAvg += replica.getDataSize();
+ }
+ dataLength += replica.getLocalSegmentSize() +
replica.getRemoteSegmentSize();
+ indexLength += replica.getLocalInvertedIndexSize() +
replica.getRemoteInvertedIndexSize();
+ }
+ }
}
- return indexSize;
+ long avgRowLength = rowCountForAvg > 0 ? dataSizeForAvg /
rowCountForAvg : 0;
+ return new TableStatusStats(rowCount, dataLength, avgRowLength,
indexLength);
}
// Get the signature string of this table with specified partitions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index 59d5c10045e..a201617b4f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -46,6 +46,36 @@ import java.util.concurrent.TimeUnit;
public interface TableIf {
Logger LOG = LogManager.getLogger(TableIf.class);
+ class TableStatusStats {
+ private final long rows;
+ private final long dataLength;
+ private final long avgRowLength;
+ private final long indexLength;
+
+ public TableStatusStats(long rows, long dataLength, long avgRowLength,
long indexLength) {
+ this.rows = rows;
+ this.dataLength = dataLength;
+ this.avgRowLength = avgRowLength;
+ this.indexLength = indexLength;
+ }
+
+ public long getRows() {
+ return rows;
+ }
+
+ public long getDataLength() {
+ return dataLength;
+ }
+
+ public long getAvgRowLength() {
+ return avgRowLength;
+ }
+
+ public long getIndexLength() {
+ return indexLength;
+ }
+ }
+
long UNKNOWN_ROW_COUNT = -1;
default void readLock() {
@@ -176,6 +206,10 @@ public interface TableIf {
long getIndexLength();
+ default TableStatusStats getTableStatusStats() {
+ return new TableStatusStats(getCachedRowCount(), getDataLength(),
getAvgRowLength(), getIndexLength());
+ }
+
long getLastCheckTime();
String getComment(boolean escapeQuota);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 434910398c3..34c0093c6b9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -350,6 +350,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
@@ -660,6 +661,11 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
TListTableStatusResult result = new TListTableStatusResult();
List<TTableStatus> tablesResult = Lists.newArrayList();
result.setTables(tablesResult);
+ Set<String> requiredColumns = params.isSetRequiredColumns()
+ ? params.getRequiredColumns().stream()
+ .map(column -> column.toUpperCase(Locale.ROOT))
+ .collect(Collectors.toSet())
+ : null;
PatternMatcher matcher = null;
String specifiedTable = null;
if (params.isSetPattern()) {
@@ -739,17 +745,40 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
TTableStatus status = new TTableStatus();
status.setName(table.getName());
status.setType(table.getMysqlType());
- status.setEngine(table.getEngine());
status.setComment(table.getComment());
- status.setCreateTime(table.getCreateTime());
- status.setLastCheckTime(lastCheckTime / 1000);
- status.setUpdateTime(table.getUpdateTime() / 1000);
- status.setCheckTime(lastCheckTime / 1000);
- status.setCollation("utf-8");
- status.setRows(table.getCachedRowCount());
- status.setDataLength(table.getDataLength());
- status.setAvgRowLength(table.getAvgRowLength());
- status.setIndexLength(table.getIndexLength());
+ if (needTableStatusColumn(requiredColumns,
"ENGINE")) {
+ status.setEngine(table.getEngine());
+ }
+ if (needTableStatusColumn(requiredColumns,
"CREATE_TIME")) {
+ status.setCreateTime(table.getCreateTime());
+ }
+ if (needTableStatusColumn(requiredColumns,
"LAST_CHECK_TIME")
+ || needTableStatusColumn(requiredColumns,
"CHECK_TIME")) {
+ status.setLastCheckTime(lastCheckTime / 1000);
+ }
+ if (needTableStatusColumn(requiredColumns,
"UPDATE_TIME")) {
+ status.setUpdateTime(table.getUpdateTime() /
1000);
+ }
+ if (needTableStatusColumn(requiredColumns,
"CHECK_TIME")) {
+ status.setCheckTime(lastCheckTime / 1000);
+ }
+ if (needTableStatusColumn(requiredColumns,
"TABLE_COLLATION")) {
+ status.setCollation("utf-8");
+ }
+ TableIf.TableStatusStats tableStatusStats =
+ needTableStatusStats(requiredColumns) ?
table.getTableStatusStats() : null;
+ if (needTableStatusColumn(requiredColumns,
"TABLE_ROWS")) {
+ status.setRows(tableStatusStats.getRows());
+ }
+ if (needTableStatusColumn(requiredColumns,
"DATA_LENGTH")) {
+
status.setDataLength(tableStatusStats.getDataLength());
+ }
+ if (needTableStatusColumn(requiredColumns,
"AVG_ROW_LENGTH")) {
+
status.setAvgRowLength(tableStatusStats.getAvgRowLength());
+ }
+ if (needTableStatusColumn(requiredColumns,
"INDEX_LENGTH")) {
+
status.setIndexLength(tableStatusStats.getIndexLength());
+ }
if (table instanceof View) {
status.setDdlSql(((View)
table).getInlineViewDef());
}
@@ -766,6 +795,17 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
return result;
}
+ private static boolean needTableStatusColumn(Set<String> requiredColumns,
String columnName) {
+ return requiredColumns == null || requiredColumns.contains(columnName);
+ }
+
+ private static boolean needTableStatusStats(Set<String> requiredColumns) {
+ return needTableStatusColumn(requiredColumns, "TABLE_ROWS")
+ || needTableStatusColumn(requiredColumns, "DATA_LENGTH")
+ || needTableStatusColumn(requiredColumns, "AVG_ROW_LENGTH")
+ || needTableStatusColumn(requiredColumns, "INDEX_LENGTH");
+ }
+
public TListTableMetadataNameIdsResult
listTableMetadataNameIds(TGetTablesParams params) throws TException {
if (LOG.isDebugEnabled()) {
LOG.debug("get list simple table request: {}", params);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index 87b4c7c78b7..f7f7bb88beb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -52,6 +52,42 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
public class OlapTableTest {
+ @Test
+ public void testGetTableStatusStatsUsesSinglePassSemantics() {
+ OlapTable olapTable = new OlapTable();
+ MaterializedIndex index = new MaterializedIndex(10,
MaterializedIndex.IndexState.NORMAL);
+ index.setRowCount(20);
+
+ List<Replica> replicas = Lists.newArrayList(
+ mockReplica(Replica.ReplicaState.NORMAL, 10, 100, 1, 20, 2),
+ mockReplica(Replica.ReplicaState.DECOMMISSION, 30, 300, 3, 40,
4),
+ mockReplica(Replica.ReplicaState.NORMAL, 50, 500, 5, 60, 6));
+ Tablet tablet = Mockito.mock(Tablet.class);
+ Mockito.when(tablet.getReplicas()).thenReturn(replicas);
+ index.appendTablets(Lists.newArrayList(tablet));
+
+ Partition partition = new Partition(11, "p1", index, null);
+ olapTable.addPartition(partition);
+
+ TableIf.TableStatusStats stats = olapTable.getTableStatusStats();
+ Assert.assertEquals(20L, stats.getRows());
+ Assert.assertEquals(909L, stats.getDataLength());
+ Assert.assertEquals(3L, stats.getAvgRowLength());
+ Assert.assertEquals(132L, stats.getIndexLength());
+ }
+
+ private Replica mockReplica(Replica.ReplicaState state, long dataSize,
long localSegmentSize,
+ long remoteSegmentSize, long localIndexSize, long remoteIndexSize)
{
+ Replica replica = Mockito.mock(Replica.class);
+ Mockito.when(replica.getState()).thenReturn(state);
+ Mockito.when(replica.getDataSize()).thenReturn(dataSize);
+
Mockito.when(replica.getLocalSegmentSize()).thenReturn(localSegmentSize);
+
Mockito.when(replica.getRemoteSegmentSize()).thenReturn(remoteSegmentSize);
+
Mockito.when(replica.getLocalInvertedIndexSize()).thenReturn(localIndexSize);
+
Mockito.when(replica.getRemoteInvertedIndexSize()).thenReturn(remoteIndexSize);
+ return replica;
+ }
+
@Test
public void test() throws IOException {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
index 2f89dcdeea8..69427f58a74 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
@@ -105,6 +105,23 @@ public class TabletTest {
Assert.assertEquals("tabletId=" + newTabletId, tablet.toString());
}
+ @Test
+ public void testGetReplicaStatsOnlyUsesNormalReplicas() {
+ Tablet statsTablet = new LocalTablet(2);
+ invertedIndex.addTablet(2, new TabletMeta(10, 20, 30, 40, 1,
TStorageMedium.HDD));
+ statsTablet.addReplica(new LocalReplica(11L, 1L, 100L, 0, 10L, 0L,
100L, ReplicaState.NORMAL, 0L, 100L));
+ statsTablet.addReplica(new LocalReplica(12L, 2L, 100L, 0, 20L, 0L,
200L, ReplicaState.NORMAL, 0L, 100L));
+ statsTablet.addReplica(new LocalReplica(13L, 3L, 100L, 0, 0L, 0L,
300L, ReplicaState.NORMAL, 0L, 100L));
+ statsTablet.addReplica(new LocalReplica(14L, 4L, 100L, 0, 40L, 0L,
400L, ReplicaState.DECOMMISSION,
+ 0L, 100L));
+
+ Assert.assertEquals(30L, statsTablet.getDataSize(false, false));
+ Assert.assertEquals(10L, statsTablet.getDataSize(true, false));
+ Assert.assertEquals(30L, statsTablet.getDataSize(false, true));
+ Assert.assertEquals(600L, statsTablet.getRowCount(false));
+ Assert.assertEquals(200L, statsTablet.getRowCount(true));
+ }
+
@Test
public void deleteReplicaTest() {
// delete replica1
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
index 5c442974eed..008c10d91df 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
@@ -22,6 +22,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.cloud.CacheHotspotManager;
import org.apache.doris.cloud.catalog.CloudEnv;
import org.apache.doris.common.AuthenticationException;
@@ -52,6 +53,7 @@ import org.apache.doris.thrift.TGetTablesParams;
import org.apache.doris.thrift.TGetTablesResult;
import org.apache.doris.thrift.TGetTabletReplicaInfosRequest;
import org.apache.doris.thrift.TGetTabletReplicaInfosResult;
+import org.apache.doris.thrift.TListTableStatusResult;
import org.apache.doris.thrift.TLoadTxnCommitRequest;
import org.apache.doris.thrift.TLoadTxnRollbackRequest;
import org.apache.doris.thrift.TMaxComputeBlockIdRequest;
@@ -65,10 +67,12 @@ import org.apache.doris.thrift.TSchemaTableRequestParams;
import org.apache.doris.thrift.TShowUserRequest;
import org.apache.doris.thrift.TShowUserResult;
import org.apache.doris.thrift.TStatusCode;
+import org.apache.doris.thrift.TTableStatus;
import org.apache.doris.transaction.GlobalTransactionMgrIface;
import org.apache.doris.transaction.TransactionState;
import org.apache.doris.utframe.UtFrameUtils;
+import com.google.common.collect.Sets;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -160,6 +164,178 @@ public class FrontendServiceImplTest {
Assert.assertTrue(result.getTables().isEmpty());
}
+ @Test
+ public void testListTableStatusPrunesUnrequestedExpensiveColumns() throws
Exception {
+ String createOlapTblStmt = "CREATE TABLE test.prune_status_columns(\n"
+ + " k1 INT,\n"
+ + " v1 INT\n"
+ + ")\n"
+ + "DUPLICATE KEY(k1)\n"
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ + "PROPERTIES(\"replication_num\" = \"1\");";
+ createTable(createOlapTblStmt);
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("prune_status_columns");
+ OlapTable spyTable = Mockito.spy(table);
+ db.unregisterTable(table.getName());
+ db.registerTable(spyTable);
+ try {
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TGetTablesParams params = new TGetTablesParams();
+ params.setCatalog(InternalCatalog.INTERNAL_CATALOG_NAME);
+ params.setDb("test");
+ params.setTable("prune_status_columns");
+ params.setRequiredColumns(Collections.singleton("UPDATE_TIME"));
+
params.setCurrentUserIdent(connectContext.getCurrentUserIdentity().toThrift());
+
+ TListTableStatusResult result = impl.listTableStatus(params);
+
+ Assert.assertEquals(1, result.getTablesSize());
+ TTableStatus status = result.getTables().get(0);
+ Assert.assertTrue(status.isSetUpdateTime());
+ Assert.assertFalse(status.isSetRows());
+ Assert.assertFalse(status.isSetDataLength());
+ Assert.assertFalse(status.isSetAvgRowLength());
+ Assert.assertFalse(status.isSetIndexLength());
+ Mockito.verify(spyTable, Mockito.never()).getCachedRowCount();
+ Mockito.verify(spyTable, Mockito.never()).getDataLength();
+ Mockito.verify(spyTable, Mockito.never()).getAvgRowLength();
+ Mockito.verify(spyTable, Mockito.never()).getIndexLength();
+ } finally {
+ db.unregisterTable(spyTable.getName());
+ db.registerTable(table);
+ }
+ }
+
+ @Test
+ public void testListTableStatusSetsLastCheckTimeForCheckTimeProjection()
throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.prune_check_time_status_column(\n"
+ + " k1 INT,\n"
+ + " v1 INT\n"
+ + ")\n"
+ + "DUPLICATE KEY(k1)\n"
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ + "PROPERTIES(\"replication_num\" = \"1\");";
+ createTable(createOlapTblStmt);
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("prune_check_time_status_column");
+ OlapTable spyTable = Mockito.spy(table);
+ Mockito.doReturn(10_000L).when(spyTable).getLastCheckTime();
+ db.unregisterTable(table.getName());
+ db.registerTable(spyTable);
+ try {
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TGetTablesParams params = new TGetTablesParams();
+ params.setCatalog(InternalCatalog.INTERNAL_CATALOG_NAME);
+ params.setDb("test");
+ params.setTable("prune_check_time_status_column");
+ params.setRequiredColumns(Collections.singleton("CHECK_TIME"));
+
params.setCurrentUserIdent(connectContext.getCurrentUserIdentity().toThrift());
+
+ TListTableStatusResult result = impl.listTableStatus(params);
+
+ Assert.assertEquals(1, result.getTablesSize());
+ TTableStatus status = result.getTables().get(0);
+ Assert.assertTrue(status.isSetLastCheckTime());
+ Assert.assertEquals(10L, status.getLastCheckTime());
+ Assert.assertTrue(status.isSetCheckTime());
+ Assert.assertEquals(10L, status.getCheckTime());
+ Assert.assertFalse(status.isSetUpdateTime());
+ Assert.assertFalse(status.isSetRows());
+ Mockito.verify(spyTable, Mockito.never()).getTableStatusStats();
+ } finally {
+ db.unregisterTable(spyTable.getName());
+ db.registerTable(table);
+ }
+ }
+
+ @Test
+ public void
testListTableStatusPrunesAllOptionalColumnsWhenRequiredColumnsIsEmpty() throws
Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.prune_all_optional_status_columns(\n"
+ + " k1 INT,\n"
+ + " v1 INT\n"
+ + ")\n"
+ + "DUPLICATE KEY(k1)\n"
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ + "PROPERTIES(\"replication_num\" = \"1\");";
+ createTable(createOlapTblStmt);
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("prune_all_optional_status_columns");
+ OlapTable spyTable = Mockito.spy(table);
+ db.unregisterTable(table.getName());
+ db.registerTable(spyTable);
+ try {
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TGetTablesParams params = new TGetTablesParams();
+ params.setCatalog(InternalCatalog.INTERNAL_CATALOG_NAME);
+ params.setDb("test");
+ params.setTable("prune_all_optional_status_columns");
+ params.setRequiredColumns(Collections.emptySet());
+
params.setCurrentUserIdent(connectContext.getCurrentUserIdentity().toThrift());
+
+ TListTableStatusResult result = impl.listTableStatus(params);
+
+ Assert.assertEquals(1, result.getTablesSize());
+ TTableStatus status = result.getTables().get(0);
+ Assert.assertFalse(status.isSetEngine());
+ Assert.assertFalse(status.isSetUpdateTime());
+ Assert.assertFalse(status.isSetRows());
+ Assert.assertFalse(status.isSetDataLength());
+ Assert.assertFalse(status.isSetAvgRowLength());
+ Assert.assertFalse(status.isSetIndexLength());
+ Mockito.verify(spyTable, Mockito.never()).getTableStatusStats();
+ } finally {
+ db.unregisterTable(spyTable.getName());
+ db.registerTable(table);
+ }
+ }
+
+ @Test
+ public void testListTableStatusUsesCombinedStatsForExpensiveColumns()
throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.combined_status_stats(\n"
+ + " k1 INT,\n"
+ + " v1 INT\n"
+ + ")\n"
+ + "DUPLICATE KEY(k1)\n"
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ + "PROPERTIES(\"replication_num\" = \"1\");";
+ createTable(createOlapTblStmt);
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("combined_status_stats");
+ OlapTable spyTable = Mockito.spy(table);
+ Mockito.doReturn(new TableIf.TableStatusStats(10L, 20L, 2L, 30L))
+ .when(spyTable).getTableStatusStats();
+ db.unregisterTable(table.getName());
+ db.registerTable(spyTable);
+ try {
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TGetTablesParams params = new TGetTablesParams();
+ params.setCatalog(InternalCatalog.INTERNAL_CATALOG_NAME);
+ params.setDb("test");
+ params.setTable("combined_status_stats");
+ params.setRequiredColumns(Sets.newHashSet("table_rows",
"Data_Length", "avg_row_length",
+ "INDEX_LENGTH"));
+
params.setCurrentUserIdent(connectContext.getCurrentUserIdentity().toThrift());
+
+ TListTableStatusResult result = impl.listTableStatus(params);
+
+ Assert.assertEquals(1, result.getTablesSize());
+ TTableStatus status = result.getTables().get(0);
+ Assert.assertEquals(10L, status.getRows());
+ Assert.assertEquals(20L, status.getDataLength());
+ Assert.assertEquals(2L, status.getAvgRowLength());
+ Assert.assertEquals(30L, status.getIndexLength());
+ Mockito.verify(spyTable).getTableStatusStats();
+ Mockito.verify(spyTable, Mockito.never()).getCachedRowCount();
+ Mockito.verify(spyTable, Mockito.never()).getDataLength();
+ Mockito.verify(spyTable, Mockito.never()).getAvgRowLength();
+ Mockito.verify(spyTable, Mockito.never()).getIndexLength();
+ } finally {
+ db.unregisterTable(spyTable.getName());
+ db.registerTable(table);
+ }
+ }
+
@Test
public void testCreatePartitionRange() throws Exception {
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index b7111df7066..13e8e57e055 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -128,6 +128,9 @@ struct TGetTablesParams {
// Reserved for downstream field `current_roles` to keep thrift field ids
// wire-compatible across maintained branches. Do not reuse this id.
9: optional set<string> reserved_field_9
+ // Columns needed by schema table callers. If unset, the callee returns the
+ // full table status for backward compatibility.
+ 10: optional set<string> required_columns
}
struct TTableStatus {
diff --git
a/regression-test/suites/information_schema_p0/test_tables_status_column_pruning.groovy
b/regression-test/suites/information_schema_p0/test_tables_status_column_pruning.groovy
new file mode 100644
index 00000000000..e6a3245d580
--- /dev/null
+++
b/regression-test/suites/information_schema_p0/test_tables_status_column_pruning.groovy
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_tables_status_column_pruning") {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+
+ def assertNoTabletStatusColumns = {
+ notContains "col=TABLE_ROWS"
+ notContains "col=DATA_LENGTH"
+ notContains "col=AVG_ROW_LENGTH"
+ notContains "col=INDEX_LENGTH"
+ }
+
+ explain {
+ verbose true
+ sql """
+ SELECT UPDATE_TIME
+ FROM information_schema.tables
+ WHERE TABLE_SCHEMA = 'information_schema'
+ """
+ contains "TABLE: information_schema.tables"
+ contains "final projections: UPDATE_TIME"
+ contains "col=TABLE_SCHEMA"
+ contains "col=UPDATE_TIME"
+ notContains "col=TABLE_NAME"
+ assertNoTabletStatusColumns.delegate = delegate
+ assertNoTabletStatusColumns.call()
+ }
+
+ explain {
+ verbose true
+ sql """
+ SELECT TABLE_NAME, UPDATE_TIME
+ FROM information_schema.tables
+ WHERE TABLE_SCHEMA = 'information_schema'
+ """
+ contains "TABLE: information_schema.tables"
+ contains "final projections: TABLE_NAME"
+ contains "UPDATE_TIME"
+ contains "col=TABLE_SCHEMA"
+ contains "col=TABLE_NAME"
+ contains "col=UPDATE_TIME"
+ assertNoTabletStatusColumns.delegate = delegate
+ assertNoTabletStatusColumns.call()
+ }
+
+ explain {
+ verbose true
+ sql """
+ SELECT TABLE_NAME, UPDATE_TIME
+ FROM information_schema.tables
+ WHERE TABLE_SCHEMA = 'information_schema'
+ AND TABLE_NAME = 'tables'
+ """
+ contains "TABLE: information_schema.tables"
+ contains "final projections: TABLE_NAME"
+ contains "UPDATE_TIME"
+ contains "col=TABLE_SCHEMA"
+ contains "col=TABLE_NAME"
+ contains "col=UPDATE_TIME"
+ assertNoTabletStatusColumns.delegate = delegate
+ assertNoTabletStatusColumns.call()
+ }
+
+ explain {
+ verbose true
+ sql """
+ SELECT TABLE_NAME, ENGINE, VERSION, ROW_FORMAT, TABLE_ROWS,
AVG_ROW_LENGTH,
+ DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE,
AUTO_INCREMENT,
+ CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION,
CHECKSUM,
+ CREATE_OPTIONS, TABLE_COMMENT
+ FROM information_schema.tables
+ WHERE TABLE_SCHEMA = 'information_schema'
+ AND TABLE_NAME LIKE 'tables'
+ """
+ contains "TABLE: information_schema.tables"
+ contains "col=TABLE_ROWS"
+ contains "col=DATA_LENGTH"
+ contains "col=AVG_ROW_LENGTH"
+ contains "col=INDEX_LENGTH"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
