This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new c0e6d1e7b15 [improvement](test)Use show_nested_index_file to check
inverted index (#38443 #39024 #39077) (#39232)
c0e6d1e7b15 is described below
commit c0e6d1e7b157a482153f312535565b093e121267
Author: qiye <[email protected]>
AuthorDate: Mon Aug 12 16:28:42 2024 +0800
[improvement](test)Use show_nested_index_file to check inverted index
(#38443 #39024 #39077) (#39232)
---
.../plugins/plugin_curl_requester.groovy | 131 +++++++++++++++++++++
.../test_add_build_index_with_format_v2.groovy | 36 +-----
.../test_create_table_with_format_v2.groovy | 13 +-
...est_cumulative_compaction_with_format_v2.groovy | 27 +----
.../test_drop_column_with_format_v2.groovy | 42 +------
.../test_drop_index_with_format_v2.groovy | 38 +-----
.../test_mor_table_with_format_v2.groovy | 30 +----
.../test_mow_table_with_format_v2.groovy | 29 +----
.../test_recover_with_format_v2.groovy | 16 +--
.../test_rename_column_with_format_v2.groovy | 39 +-----
...single_replica_compaction_with_format_v2.groovy | 31 +----
11 files changed, 164 insertions(+), 268 deletions(-)
diff --git a/regression-test/plugins/plugin_curl_requester.groovy
b/regression-test/plugins/plugin_curl_requester.groovy
index 77698a6a88f..531b2c192fb 100644
--- a/regression-test/plugins/plugin_curl_requester.groovy
+++ b/regression-test/plugins/plugin_curl_requester.groovy
@@ -18,8 +18,99 @@
import org.apache.doris.regression.suite.Suite
import org.apache.doris.regression.util.Http
import org.apache.doris.regression.util.NodeType
+@Grab(group='org.apache.httpcomponents', module='httpclient', version='4.5.13')
+import org.apache.http.client.methods.*
+import org.apache.http.impl.client.*
+import org.apache.http.util.EntityUtils
+import org.apache.http.client.config.RequestConfig
+import org.apache.http.conn.ConnectTimeoutException
+import org.apache.http.conn.HttpHostConnectException
import org.codehaus.groovy.runtime.IOGroovyMethods
+Suite.metaClass.http_client = { String method, String url /* param */ ->
+ Suite suite = delegate as Suite
+ if (method != "GET" && method != "POST") {
+ throw new Exception("Invalid method: ${method}")
+ }
+ if (!url || !(url =~ /^https?:\/\/.+/)) {
+ throw new Exception("Invalid url: ${url}")
+ }
+
+ Integer timeout = 300 // seconds
+ Integer maxRetries = 10
+ Integer retryCount = 0
+ Integer sleepTime = 1000 // milliseconds
+
+ logger.info("HTTP request: ${method} ${url}")
+
+ CloseableHttpClient httpClient = HttpClients.custom()
+ .setRetryHandler(new DefaultHttpRequestRetryHandler(3, true))
+ .build()
+
+ int code
+ String err
+ String out
+
+ try {
+ while (retryCount < maxRetries) {
+ HttpRequestBase request
+ if (method == "GET") {
+ request = new HttpGet(url)
+ } else if (method == "POST") {
+ request = new HttpPost(url)
+ }
+
+ RequestConfig requestConfig = RequestConfig.custom()
+ .setConnectTimeout(timeout * 1000)
+ .setSocketTimeout(timeout * 1000)
+ .build()
+ request.setConfig(requestConfig)
+
+ try {
+ CloseableHttpResponse response = httpClient.execute(request)
+ try {
+ code = response.getStatusLine().getStatusCode()
+ out = EntityUtils.toString(response.getEntity())
+
+ if (code >= 200 && code < 300) {
+ code = 0 // to be compatible with the old curl function
+ err = ""
+ return [code, out, err]
+ } else if (code == 500) {
+ return [code, out, "Internal Server Error"]
+ } else {
+ logger.warn("HTTP request failed with status code
${code}, response ${out}, retrying (${++retryCount}/${maxRetries})")
+ }
+ } finally {
+ response.close()
+ }
+ } catch (ConnectTimeoutException | HttpHostConnectException e) {
+ logger.warn("Connection failed, retrying
(${++retryCount}/${maxRetries}): ${e.message}")
+ } catch (SocketTimeoutException e) {
+ timeout = timeout + 10
+ logger.warn("Read timed out, retrying
(${++retryCount}/${maxRetries}): ${e.message}")
+ } catch (Exception e) {
+ code = 500 // Internal Server Error
+ logger.error("Error executing HTTP request: ${e.message}")
+ err = e.message
+ return [code, out, err]
+ }
+
+ sleep(sleepTime)
+ sleepTime = Math.min(sleepTime * 2, 60000)
+ }
+
+ logger.error("HTTP request failed after ${maxRetries} attempts")
+ err = "Failed after ${maxRetries} attempts"
+ code = 500 // Internal Server Error
+ return [code, out, err]
+ } finally {
+ httpClient.close()
+ }
+}
+
+logger.info("Added 'http_client' function to Suite")
+
Suite.metaClass.curl = { String method, String url /* param */->
Suite suite = delegate as Suite
if (method != "GET" && method != "POST")
@@ -161,3 +252,43 @@ Suite.metaClass.be_report_task = { String ip, int port ->
}
logger.info("Added 'be_report_task' function to Suite")
+
+// check nested index file api
+Suite.metaClass.check_nested_index_file = { ip, port, tablet_id,
expected_rowsets_count, expected_indices_count, format ->
+ def (code, out, err) = http_client("GET",
String.format("http://%s:%s/api/show_nested_index_file?tablet_id=%s";, ip, port,
tablet_id))
+ logger.info("Run show_nested_index_file_on_tablet: code=" + code + ",
out=" + out + ", err=" + err)
+ // only when the expected_indices_count is 0, the tablet may not have the
index file.
+ if (code == 500 && expected_indices_count == 0) {
+ assertEquals("E-6003", parseJson(out.trim()).status)
+ assertTrue(parseJson(out.trim()).msg.contains("not found"))
+ return
+ }
+ assertTrue(code == 0)
+ assertEquals(tablet_id, parseJson(out.trim()).tablet_id.toString())
+ def rowsets_count = parseJson(out.trim()).rowsets.size();
+ assertEquals(expected_rowsets_count, rowsets_count)
+ def index_files_count = 0
+ def segment_files_count = 0
+ for (def rowset in parseJson(out.trim()).rowsets) {
+ assertEquals(format, rowset.index_storage_format)
+ for (int i = 0; i < rowset.segments.size(); i++) {
+ def segment = rowset.segments[i]
+ assertEquals(i, segment.segment_id)
+ def indices_count = segment.indices.size()
+ assertEquals(expected_indices_count, indices_count)
+ if (format == "V1") {
+ index_files_count += indices_count
+ } else {
+ index_files_count++
+ }
+ }
+ segment_files_count += rowset.segments.size()
+ }
+ if (format == "V1") {
+ assertEquals(index_files_count, segment_files_count *
expected_indices_count)
+ } else {
+ assertEquals(index_files_count, segment_files_count)
+ }
+}
+
+logger.info("Added 'check_nested_index_file' function to Suite")
\ No newline at end of file
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy
index 430c3a67e0a..dabb3534e32 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy
@@ -20,9 +20,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_add_build_index_with_format_v2", "inverted_index_format_v2"){
def tableName = "test_add_build_index_with_format_v2"
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
def backendId_to_backendIP = [:]
def backendId_to_backendHttpPort = [:]
getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
@@ -115,22 +112,15 @@ suite("test_add_build_index_with_format_v2",
"inverted_index_format_v2"){
String backend_id = tablets[0].BackendId
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
+
// cloud mode is directly schema change, local mode is light schema change.
// cloud mode is 12, local mode is 6
if (isCloudMode()) {
- assertEquals("12", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 2, "V2")
qt_sql "SELECT * FROM $tableName WHERE name match 'andy' order by id,
name, score;"
return
} else {
- assertEquals("6", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 0, "V2")
}
// build index
@@ -139,15 +129,7 @@ suite("test_add_build_index_with_format_v2",
"inverted_index_format_v2"){
"""
wait_for_build_index_on_partition_finish(tableName, timeout)
- (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("12", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 1, "V2")
// build index
sql """
@@ -155,15 +137,7 @@ suite("test_add_build_index_with_format_v2",
"inverted_index_format_v2"){
"""
wait_for_build_index_on_partition_finish(tableName, timeout)
- (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("12", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 2, "V2")
qt_sql "SELECT * FROM $tableName WHERE name match 'andy' order by id,
name, score;"
}
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_create_table_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_create_table_with_format_v2.groovy
index dd87a214e2d..cf31e2ca80f 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_create_table_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_create_table_with_format_v2.groovy
@@ -20,9 +20,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_create_table_with_format_v2", "inverted_index_format_v2"){
def tableName = "test_create_table_with_format_v2"
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
def backendId_to_backendIP = [:]
def backendId_to_backendHttpPort = [:]
getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
@@ -60,13 +57,5 @@ suite("test_create_table_with_format_v2",
"inverted_index_format_v2"){
String backend_id = tablets[0].BackendId
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("12", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 2, "V2")
}
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy
index f8ae36175a5..65270ebe597 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy
@@ -40,10 +40,6 @@ suite("test_cumulative_compaction_with_format_v2",
"inverted_index_format_v2") {
assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
}
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
-
def calc_segment_count = { tablet ->
int segment_count = 0
String tablet_id = tablet.TabletId
@@ -173,16 +169,7 @@ suite("test_cumulative_compaction_with_format_v2",
"inverted_index_format_v2") {
String port = backendId_to_backendHttpPort.get(backend_id)
int segment_count = calc_segment_count(tablet)
logger.info("TabletId: " + tablet_id + ", segment_count: " +
segment_count)
- def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" +
o + ", err=" + e)
- assertTrue(c == 0)
- assertTrue(o.contains("crc_value"))
- assertTrue(o.contains("used_time_ms"))
- assertEquals("0", parseJson(o.trim()).start_version)
- assertEquals("9", parseJson(o.trim()).end_version)
- assertEquals("9", parseJson(o.trim()).rowset_count)
- int file_count = segment_count * 2
- assertEquals(file_count,
Integer.parseInt(parseJson(o.trim()).file_count))
+ check_nested_index_file(ip, port, tablet_id, 9, 3, "V2")
StringBuilder sb = new StringBuilder();
sb.append("curl -X POST http://";)
@@ -242,17 +229,7 @@ suite("test_cumulative_compaction_with_format_v2",
"inverted_index_format_v2") {
String port = backendId_to_backendHttpPort.get(backend_id)
int segment_count = calc_segment_count(tablet)
logger.info("TabletId: " + tablet_id + ", segment_count: " +
segment_count)
- def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" +
o + ", err=" + e)
- assertTrue(c == 0)
- assertTrue(o.contains("crc_value"))
- assertTrue(o.contains("used_time_ms"))
- assertEquals("0", parseJson(o.trim()).start_version)
- assertEquals("9", parseJson(o.trim()).end_version)
- // after compaction, there are 2 rwosets.
- assertEquals("2", parseJson(o.trim()).rowset_count)
- int file_count = segment_count * 2
- assertEquals(file_count,
Integer.parseInt(parseJson(o.trim()).file_count))
+ check_nested_index_file(ip, port, tablet_id, 2, 3, "V2")
}
int segmentsCount = 0
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy
index 24ac5c6ee6f..1a75137e1e4 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy
@@ -25,9 +25,6 @@ suite("test_drop_column_with_format_v2",
"inverted_index_format_v2"){
}
def tableName = "test_drop_column_with_format_v2"
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
def backendId_to_backendIP = [:]
def backendId_to_backendHttpPort = [:]
getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
@@ -106,50 +103,23 @@ suite("test_drop_column_with_format_v2",
"inverted_index_format_v2"){
String backend_id = tablets[0].BackendId
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("12", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 2, "V2")
// drop column
sql """ ALTER TABLE ${tableName} DROP COLUMN score; """
wait_for_latest_op_on_table_finish(tableName, timeout)
- // select to sync rowset meta in cloud mode
- sql """ select * from ${tableName} limit 1; """
-
tablets = sql_return_maparray """ show tablets from ${tableName}; """
tablet_id = tablets[0].TabletId
- (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("12", parseJson(out.trim()).file_count)
+ // when drop column, the index files will not be deleted, so the index
files count is still 2
+ check_nested_index_file(ip, port, tablet_id, 7, 2, "V2")
sql """ ALTER TABLE ${tableName} DROP COLUMN name; """
wait_for_latest_op_on_table_finish(tableName, timeout)
- // select to sync rowset meta in cloud mode
- sql """ select * from ${tableName} limit 1; """
-
tablets = sql_return_maparray """ show tablets from ${tableName}; """
tablet_id = tablets[0].TabletId
- (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("6", parseJson(out.trim()).file_count)
+ // when drop column, the index files will not be deleted, so the index
files count is still 2
+ // when all index columns are dropped, the index files will be deleted by
GC later
+ check_nested_index_file(ip, port, tablet_id, 7, 2, "V2")
}
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy
index c2ce451e857..12877099f38 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy
@@ -25,9 +25,6 @@ suite("test_drop_index_with_format_v2",
"inverted_index_format_v2"){
}
def tableName = "test_drop_index_with_format_v2"
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
def backendId_to_backendIP = [:]
def backendId_to_backendHttpPort = [:]
getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
@@ -85,44 +82,15 @@ suite("test_drop_index_with_format_v2",
"inverted_index_format_v2"){
String backend_id = tablets[0].BackendId
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("12", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 2, "V2")
// drop index
sql """ DROP INDEX index_name on ${tableName}; """
wait_for_latest_op_on_table_finish(tableName, timeout)
-
- (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("12", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 1, "V2")
// drop index
sql """ DROP INDEX index_score on ${tableName}; """
wait_for_latest_op_on_table_finish(tableName, timeout)
-
- // select to sync rowset meta in cloud mode
- sql """ select * from ${tableName} limit 1; """
-
- (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("6", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 0, "V2")
}
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy
index 700cb6abc32..b20148e8bd5 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy
@@ -40,10 +40,6 @@ suite("test_mor_table_with_format_v2",
"inverted_index_format_v2") {
assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
}
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
-
def calc_segment_count = { tablet ->
int segment_count = 0
String tablet_id = tablet.TabletId
@@ -175,16 +171,7 @@ suite("test_mor_table_with_format_v2",
"inverted_index_format_v2") {
String port = backendId_to_backendHttpPort.get(backend_id)
int segment_count = calc_segment_count(tablet)
logger.info("TabletId: " + tablet_id + ", segment_count: " +
segment_count)
- def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" +
o + ", err=" + e)
- assertTrue(c == 0)
- assertTrue(o.contains("crc_value"))
- assertTrue(o.contains("used_time_ms"))
- assertEquals("0", parseJson(o.trim()).start_version)
- assertEquals("10", parseJson(o.trim()).end_version)
- assertEquals("10", parseJson(o.trim()).rowset_count)
- int file_count = segment_count * 2
- assertEquals(file_count,
Integer.parseInt(parseJson(o.trim()).file_count))
+ check_nested_index_file(ip, port, tablet_id, 10, 3, "V2")
StringBuilder sb = new StringBuilder();
sb.append("curl -X POST http://";)
@@ -242,23 +229,12 @@ suite("test_mor_table_with_format_v2",
"inverted_index_format_v2") {
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- int segment_count = calc_segment_count(tablet)
- logger.info("TabletId: " + tablet_id + ", segment_count: " +
segment_count)
- def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" +
o + ", err=" + e)
- assertTrue(c == 0)
- assertTrue(o.contains("crc_value"))
- assertTrue(o.contains("used_time_ms"))
- assertEquals("0", parseJson(o.trim()).start_version)
- assertEquals("10", parseJson(o.trim()).end_version)
// after compaction, there are 1 rwoset in local mode and 2
rowsets in cloud mode.
if (isCloudMode()) {
- assertEquals("2", parseJson(o.trim()).rowset_count)
+ check_nested_index_file(ip, port, tablet_id, 2, 3, "V2")
} else {
- assertEquals("1", parseJson(o.trim()).rowset_count)
+ check_nested_index_file(ip, port, tablet_id, 1, 3, "V2")
}
- int file_count = segment_count * 2
- assertEquals(file_count,
Integer.parseInt(parseJson(o.trim()).file_count))
}
int segmentsCount = 0
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy
index e38f7bfe5e0..568990c8b4d 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy
@@ -40,10 +40,6 @@ suite("test_mow_table_with_format_v2",
"inverted_index_format_v2") {
assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
}
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
-
def calc_segment_count = { tablet ->
int segment_count = 0
String tablet_id = tablet.TabletId
@@ -173,16 +169,7 @@ suite("test_mow_table_with_format_v2",
"inverted_index_format_v2") {
String port = backendId_to_backendHttpPort.get(backend_id)
int segment_count = calc_segment_count(tablet)
logger.info("TabletId: " + tablet_id + ", segment_count: " +
segment_count)
- def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" +
o + ", err=" + e)
- assertTrue(c == 0)
- assertTrue(o.contains("crc_value"))
- assertTrue(o.contains("used_time_ms"))
- assertEquals("0", parseJson(o.trim()).start_version)
- assertEquals("9", parseJson(o.trim()).end_version)
- assertEquals("9", parseJson(o.trim()).rowset_count)
- int file_count = segment_count * 2
- assertEquals(file_count,
Integer.parseInt(parseJson(o.trim()).file_count))
+ check_nested_index_file(ip, port, tablet_id, 9, 3, "V2")
StringBuilder sb = new StringBuilder();
sb.append("curl -X POST http://";)
@@ -240,19 +227,7 @@ suite("test_mow_table_with_format_v2",
"inverted_index_format_v2") {
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- int segment_count = calc_segment_count(tablet)
- logger.info("TabletId: " + tablet_id + ", segment_count: " +
segment_count)
- def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" +
o + ", err=" + e)
- assertTrue(c == 0)
- assertTrue(o.contains("crc_value"))
- assertTrue(o.contains("used_time_ms"))
- assertEquals("0", parseJson(o.trim()).start_version)
- assertEquals("9", parseJson(o.trim()).end_version)
- // after compaction, there are 2 rwosets.
- assertEquals("2", parseJson(o.trim()).rowset_count)
- int file_count = segment_count * 2
- assertEquals(file_count,
Integer.parseInt(parseJson(o.trim()).file_count))
+ check_nested_index_file(ip, port, tablet_id, 2, 3, "V2")
}
int segmentsCount = 0
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_recover_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_recover_with_format_v2.groovy
index ee109817c4c..8de145d7921 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_recover_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_recover_with_format_v2.groovy
@@ -20,9 +20,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_recover_with_format_v2", "inverted_index_format_v2"){
def tableName = "test_recover_with_format_v2"
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
def backendId_to_backendIP = [:]
def backendId_to_backendHttpPort = [:]
getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
@@ -34,15 +31,7 @@ suite("test_recover_with_format_v2",
"inverted_index_format_v2"){
String backend_id = tablets[i].BackendId
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out="
+ out + ", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("3", parseJson(out.trim()).end_version)
- assertEquals("3", parseJson(out.trim()).rowset_count)
- assertEquals("4", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 3, 2, "V2")
}
}
@@ -78,6 +67,9 @@ suite("test_recover_with_format_v2",
"inverted_index_format_v2"){
sql """ INSERT INTO ${tableName} VALUES (3, "andy", 100); """
sql """ INSERT INTO ${tableName} VALUES (3, "bason", 99); """
+ // select to sync tablet meta in cloud mode
+ sql """ SELECT * FROM ${tableName}; """
+
check_index_file()
// drop table and recover
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy
index 663eb8434a3..38b03510a0c 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy
@@ -20,9 +20,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_rename_column_with_format_v2", "inverted_index_format_v2"){
def tableName = "test_rename_column_with_format_v2"
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
def backendId_to_backendIP = [:]
def backendId_to_backendHttpPort = [:]
getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
@@ -78,15 +75,7 @@ suite("test_rename_column_with_format_v2",
"inverted_index_format_v2"){
String backend_id = tablets[0].BackendId
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("12", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 1, "V2")
// rename column
sql """ ALTER TABLE ${tableName} RENAME COLUMN name name_new; """
@@ -94,30 +83,12 @@ suite("test_rename_column_with_format_v2",
"inverted_index_format_v2"){
qt_sql "SELECT * FROM $tableName WHERE name_new match 'andy' order by id,
name_new, score;"
- (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- assertEquals("12", parseJson(out.trim()).file_count)
+ check_nested_index_file(ip, port, tablet_id, 7, 1, "V2")
// drop column
sql """ ALTER TABLE ${tableName} DROP COLUMN name_new; """
wait_for_latest_op_on_table_finish(tableName, timeout)
-
- (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out +
", err=" + err)
- assertTrue(code == 0)
- assertTrue(out.contains("crc_value"))
- assertTrue(out.contains("used_time_ms"))
- assertEquals("0", parseJson(out.trim()).start_version)
- assertEquals("7", parseJson(out.trim()).end_version)
- assertEquals("7", parseJson(out.trim()).rowset_count)
- // 6 files are expected after dropping the column
- // Since we do not support change light_schema_change from true to false,
so we can not do directly schema change.
- // As a result, the rowset schema will not remove the index info, so the
file count will not decrease.
- assertEquals("12", parseJson(out.trim()).file_count)
+ // when drop column, the index files will not be deleted, so the index
files count is still 1
+ // when all index columns are dropped, the index files will be deleted by
GC later
+ check_nested_index_file(ip, port, tablet_id, 7, 1, "V2")
}
diff --git
a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy
b/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy
index 453fcd91226..7826702fa7e 100644
---
a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy
+++
b/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy
@@ -46,10 +46,6 @@ suite("test_single_replica_compaction_with_format_v2",
"inverted_index_format_v2
assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
}
- def calc_file_crc_on_tablet = { ip, port, tablet ->
- return curl("GET",
String.format("http://%s:%s/api/calc_crc?tablet_id=%s";, ip, port, tablet))
- }
-
def calc_segment_count = { tablet ->
int segment_count = 0
String tablet_id = tablet.TabletId
@@ -175,18 +171,7 @@ suite("test_single_replica_compaction_with_format_v2",
"inverted_index_format_v2
backend_id = tablet.BackendId
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- int segment_count = calc_segment_count(tablet)
- logger.info("TabletId: " + tablet_id + ", segment_count: " +
segment_count)
- def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" +
o + ", err=" + e)
- assertTrue(c == 0)
- assertTrue(o.contains("crc_value"))
- assertTrue(o.contains("used_time_ms"))
- assertEquals("0", parseJson(o.trim()).start_version)
- assertEquals("9", parseJson(o.trim()).end_version)
- assertEquals("9", parseJson(o.trim()).rowset_count)
- int file_count = segment_count * 2
- assertEquals(file_count,
Integer.parseInt(parseJson(o.trim()).file_count))
+ check_nested_index_file(ip, port, tablet_id, 9, 3, "V2")
StringBuilder sb = new StringBuilder();
sb.append("curl -X POST http://";)
@@ -244,19 +229,7 @@ suite("test_single_replica_compaction_with_format_v2",
"inverted_index_format_v2
String ip = backendId_to_backendIP.get(backend_id)
String port = backendId_to_backendHttpPort.get(backend_id)
- int segment_count = calc_segment_count(tablet)
- logger.info("TabletId: " + tablet_id + ", segment_count: " +
segment_count)
- def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id)
- logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" +
o + ", err=" + e)
- assertTrue(c == 0)
- assertTrue(o.contains("crc_value"))
- assertTrue(o.contains("used_time_ms"))
- assertEquals("0", parseJson(o.trim()).start_version)
- assertEquals("9", parseJson(o.trim()).end_version)
- // after compaction, there are 2 rwosets.
- assertEquals("2", parseJson(o.trim()).rowset_count)
- int file_count = segment_count * 2
- assertEquals(file_count,
Integer.parseInt(parseJson(o.trim()).file_count))
+ check_nested_index_file(ip, port, tablet_id, 2, 3, "V2")
}
int segmentsCount = 0
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
