This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e7a67786a56 [feature](information_schema) add rowsets table into
information_schema in cloud mode (#42367)
e7a67786a56 is described below
commit e7a67786a56585279e85bb56bc47fa9aef98c79f
Author: Uniqueyou <[email protected]>
AuthorDate: Wed Nov 6 20:53:04 2024 +0800
[feature](information_schema) add rowsets table into information_schema in
cloud mode (#42367)
## Proposed changes
Issue Number: close #32154
only query tablet **in cache** for cloud mode.
<!--Describe your changes.-->
```
mysql> CREATE TABLE user_info
-> (
-> id INT,
-> name VARCHAR(50),
-> age INT
-> )
-> DISTRIBUTED BY HASH(id) BUCKETS 10
-> PROPERTIES (
-> "replication_allocation" = "tag.location.default: 1",
-> "storage_vault_name" = "s3_vault"
-> );
mysql> select * from user_info;
+------+---------+------+
| id | name | age |
+------+---------+------+
| 1 | Alice | 25 |
| 4 | John | 29 |
| 3 | Charlie | 35 |
| 2 | Bob | 30 |
| 5 | Mars | 17 |
+------+---------+------+
5 rows in set (0.03 sec)
mysql> select * from information_schema.rowsets;
+------------+--------------------------------------------------+-----------+-----------------+-----------------+--------------+---------------+-------------+-----------------+----------------+---------------------+------------------------+----------------+
| BACKEND_ID | ROWSET_ID | TABLET_ID
| ROWSET_NUM_ROWS | TXN_ID | NUM_SEGMENTS | START_VERSION |
END_VERSION | INDEX_DISK_SIZE | DATA_DISK_SIZE | CREATION_TIME |
NEWEST_WRITE_TIMESTAMP | SCHEMA_VERSION |
+------------+--------------------------------------------------+-----------+-----------------+-----------------+--------------+---------------+-------------+-----------------+----------------+---------------------+------------------------+----------------+
| 10002 | 020000000000000a7c43ec9040f03f5ebddb5c07e1e26ea2 | 10201
| 1 | 179653390268416 | 1 | 2 |
2 | 152 | 627 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000fffffffffffffffefffffffffffee486 | 10201
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
| 10002 | 02000000000000097c43ec9040f03f5ebddb5c07e1e26ea2 | 10185
| 1 | 179653390268416 | 1 | 2 |
2 | 153 | 629 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000fffffffffe444511ffffffffffffffeb | 10185
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
| 10002 | 02000000000000067c43ec9040f03f5ebddb5c07e1e26ea2 | 10183
| 0 | 179653390268416 | 0 | 2 |
2 | 0 | 0 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000fffffffffffffffeffffffffffffffe9 | 10183
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
| 10002 | 02000000000000087c43ec9040f03f5ebddb5c07e1e26ea2 | 10189
| 0 | 179653390268416 | 0 | 2 |
2 | 0 | 0 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000ffffffffffffffefffffffffffffec86 | 10189
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
| 10002 | 02000000000000077c43ec9040f03f5ebddb5c07e1e26ea2 | 10199
| 0 | 179653390268416 | 0 | 2 |
2 | 0 | 0 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000fffffffffeb04512ffffffffffffffeb | 10199
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
| 10002 | 02000000000000057c43ec9040f03f5ebddb5c07e1e26ea2 | 10195
| 1 | 179653390268416 | 1 | 2 |
2 | 155 | 639 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000ffffffffffffea72ffffffffffffffc5 | 10195
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
| 10002 | 02000000000000017c43ec9040f03f5ebddb5c07e1e26ea2 | 10191
| 1 | 179653390268416 | 1 | 2 |
2 | 152 | 628 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000fffffffffffffffcffffffffffffffed | 10191
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
| 10002 | 02000000000000027c43ec9040f03f5ebddb5c07e1e26ea2 | 10193
| 1 | 179653390268416 | 1 | 2 |
2 | 151 | 622 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000fffffffffffffef1ffffffffffee2929 | 10193
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
| 10002 | 02000000000000047c43ec9040f03f5ebddb5c07e1e26ea2 | 10197
| 0 | 179653390268416 | 0 | 2 |
2 | 0 | 0 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000ffffffffffffffeafffffffffffffffb | 10197
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
| 10002 | 02000000000000037c43ec9040f03f5ebddb5c07e1e26ea2 | 10187
| 0 | 179653390268416 | 0 | 2 |
2 | 0 | 0 | 2024-10-29 21:05:59 | 2024-10-29
21:05:59 | 0 |
| 10002 | 0200000000000000fffffffffffffffffffffffffffffa80 | 10187
| 0 | 0 | 0 | 0 |
1 | 0 | 0 | 1970-01-01 08:00:00 | 1970-01-01
07:59:59 | 0 |
+------------+--------------------------------------------------+-----------+-----------------+-----------------+--------------+---------------+-------------+-----------------+----------------+---------------------+------------------------+----------------+
20 rows in set (0.04 sec)
mysql> show tablet 10201\G;
*************************** 1. row ***************************
DbName: wyx
TableName: user_info
PartitionName: user_info
IndexName: user_info
DbId: 10102
TableId: 10161
PartitionId: 10160
IndexId: 10162
IsSync: true
Order: 9
QueryHits: 0
DetailCmd: SHOW PROC '/dbs/10102/10161/partitions/10160/10162/10201';
1 row in set (0.00 sec)
mysql> SHOW PROC '/dbs/10102/10161/partitions/10160/10162/10201'\G;
*************************** 1. row ***************************
ReplicaId: 10202
BackendId: 10002
Version: 1
LstSuccessVersion: 1
LstFailedVersion: -1
LstFailedTime: NULL
SchemaHash: 1263548704
LocalDataSize: 627
RemoteDataSize: 0
RowCount: 1
State: NORMAL
IsBad: false
IsUserDrop: false
VisibleVersionCount: -1
VersionCount: -1
PathHash: -1
Path:
MetaUrl: http://192.168.31.122:8440/api/meta/header/10201
CompactionStatus:
http://192.168.31.122:8440/api/compaction/show?tablet_id=10201
CooldownReplicaId: -1
CooldownMetaId:
QueryHits: 0
PrimaryBackendId: 10002
1 row in set (0.00 sec)
```
---
.../exec/schema_scanner/schema_rowsets_scanner.cpp | 18 +++++-
.../system/test_query_sys_scan_rowsets.out | 34 +++++++++++
.../system/test_query_sys_scan_rowsets.groovy | 71 ++++++++++++++++++++++
3 files changed, 122 insertions(+), 1 deletion(-)
diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
index 16d5f2daba6..3aa0e944a82 100644
--- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
@@ -26,6 +26,9 @@
#include <string>
#include <utility>
+#include "cloud/cloud_storage_engine.h"
+#include "cloud/cloud_tablet.h"
+#include "cloud/cloud_tablet_mgr.h"
#include "cloud/config.h"
#include "common/status.h"
#include "olap/olap_common.h"
@@ -35,6 +38,7 @@
#include "olap/tablet.h"
#include "olap/tablet_manager.h"
#include "runtime/define_primitive_type.h"
+#include "runtime/exec_env.h"
#include "runtime/runtime_state.h"
#include "util/runtime_profile.h"
#include "vec/common/string_ref.h"
@@ -78,7 +82,19 @@ Status SchemaRowsetsScanner::start(RuntimeState* state) {
Status SchemaRowsetsScanner::_get_all_rowsets() {
if (config::is_cloud_mode()) {
- return Status::NotSupported("SchemaRowsetsScanner::_get_all_rowsets is
not implemented");
+ // only query cloud tablets in lru cache instead of all tablets
+ std::vector<std::weak_ptr<CloudTablet>> tablets =
+
ExecEnv::GetInstance()->storage_engine().to_cloud().tablet_mgr().get_weak_tablets();
+ for (const std::weak_ptr<CloudTablet>& tablet : tablets) {
+ if (!tablet.expired()) {
+ auto t = tablet.lock();
+ std::shared_lock rowset_ldlock(t->get_header_lock());
+ for (const auto& it : t->rowset_map()) {
+ rowsets_.emplace_back(it.second);
+ }
+ }
+ }
+ return Status::OK();
}
std::vector<TabletSharedPtr> tablets =
ExecEnv::GetInstance()->storage_engine().to_local().tablet_manager()->get_all_tablet();
diff --git
a/regression-test/data/query_p0/system/test_query_sys_scan_rowsets.out
b/regression-test/data/query_p0/system/test_query_sys_scan_rowsets.out
new file mode 100644
index 00000000000..b034ebe1bbd
--- /dev/null
+++ b/regression-test/data/query_p0/system/test_query_sys_scan_rowsets.out
@@ -0,0 +1,34 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !desc_rowsets --
+BACKEND_ID bigint Yes false \N
+ROWSET_ID varchar(64) Yes false \N
+TABLET_ID bigint Yes false \N
+ROWSET_NUM_ROWS bigint Yes false \N
+TXN_ID bigint Yes false \N
+NUM_SEGMENTS bigint Yes false \N
+START_VERSION bigint Yes false \N
+END_VERSION bigint Yes false \N
+INDEX_DISK_SIZE bigint Yes false \N
+DATA_DISK_SIZE bigint Yes false \N
+CREATION_TIME datetime Yes false \N
+NEWEST_WRITE_TIMESTAMP datetime Yes false \N
+SCHEMA_VERSION int Yes false \N
+
+-- !rowsets1 --
+
+-- !rowsets2 --
+0 1
+2 2
+
+-- !rowsets3 --
+0 1
+2 2
+3 3
+4 4
+
+-- !rowsets4 --
+2 2
+3 3
+4 4
+5 5
+
diff --git
a/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy
b/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy
new file mode 100644
index 00000000000..48281198cfd
--- /dev/null
+++ b/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+suite("test_query_sys_scan_rowsets", "query,p0") {
+ def dbName1 = "test_query_sys_scan_rowsets"
+
+ if (!isCloudMode()) {
+ log.info("not cloud mode")
+ return
+ }
+
+
+ sql("CREATE DATABASE IF NOT EXISTS ${dbName1}")
+
+ // test rowsets
+ qt_desc_rowsets """ desc information_schema.rowsets """
+ def rowsets_table_name = """ test_query_sys_scan_rowsets.test_query_rowset
"""
+ sql """ drop table if exists ${rowsets_table_name} """
+
+ sql """
+ create table ${rowsets_table_name}(
+ a int ,
+ b boolean ,
+ c string )
+ DISTRIBUTED BY HASH(`a`) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "disable_auto_compaction" = "true",
+ "enable_single_replica_compaction"="true"
+ );
+ """
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+ def now = sdf.format(new Date()).toString();
+
+ def rowsets_table_name_tablets = sql_return_maparray """ show tablets from
${rowsets_table_name}; """
+ def tablet_id = rowsets_table_name_tablets[0].TabletId
+
+ sql """ select * from ${rowsets_table_name}; """
+ order_qt_rowsets1 """ select START_VERSION,END_VERSION from
information_schema.rowsets where TABLET_ID=${tablet_id} group by
START_VERSION,END_VERSION order by START_VERSION,END_VERSION; """
+
+ sql """ insert into ${rowsets_table_name} values (1,0,"abc"); """
+ sql """ select * from ${rowsets_table_name}; """
+ order_qt_rowsets2 """ select START_VERSION,END_VERSION from
information_schema.rowsets where TABLET_ID=${tablet_id} group by
START_VERSION,END_VERSION order by START_VERSION,END_VERSION; """
+
+ sql """ insert into ${rowsets_table_name} values (2,1,"hello world");
"""
+ sql """ insert into ${rowsets_table_name} values (3,0,"dssadasdsafafdf");
"""
+ sql """ select * from ${rowsets_table_name}; """
+ order_qt_rowsets3 """ select START_VERSION,END_VERSION from
information_schema.rowsets where TABLET_ID=${tablet_id} group by
START_VERSION,END_VERSION order by START_VERSION,END_VERSION; """
+
+ sql """ insert into ${rowsets_table_name} values (4,0,"abcd"); """
+ sql """ select * from ${rowsets_table_name}; """
+ order_qt_rowsets4 """ select START_VERSION,END_VERSION from
information_schema.rowsets where TABLET_ID=${tablet_id} and
NEWEST_WRITE_TIMESTAMP>='${now}' group by START_VERSION,END_VERSION order by
START_VERSION,END_VERSION; """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
