This is an automated email from the ASF dual-hosted git repository.
jiacai2050 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-horaedb.git
The following commit(s) were added to refs/heads/main by this push:
new 75af5b31 feat: add sst level config for benchmark (#1482)
75af5b31 is described below
commit 75af5b3142e6b56d11d22a72267b1a5aeed36496
Author: MianChen <[email protected]>
AuthorDate: Tue Feb 20 21:40:10 2024 +0800
feat: add sst level config for benchmark (#1482)
## Rationale
Add sst level configuration for the benchmark test.
## Detailed Changes
Add sst level configuration for the benchmark test.
## Test Plan
Manual test
---------
Co-authored-by: jiacai2050 <[email protected]>
---
.github/workflows/ci.yml | 7 +------
Cargo.toml | 2 +-
src/benchmarks/bench.toml | 1 +
src/benchmarks/config/bench.toml | 1 +
src/benchmarks/src/config.rs | 1 +
src/benchmarks/src/merge_sst_bench.rs | 4 +++-
src/benchmarks/src/util.rs | 4 ++--
src/df_engine_extensions/Cargo.toml | 2 +-
src/horaedb/Cargo.toml | 5 +----
src/meta_client/Cargo.toml | 2 +-
src/wal/Cargo.toml | 6 +++---
11 files changed, 16 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3c71912d..8af668ea 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -70,17 +70,12 @@ jobs:
run: |
rustup component add clippy
rustup component add rustfmt
- cargo install cargo-sort
+ cargo install --git https://github.com/DevinR528/cargo-sort --rev
55ec890 --locked
- name: Run Style Check
run: |
make clippy
make fmt
make check-cargo-toml
- - name: Report Disk Usage
- if: always()
- run: |
- du -sh ./target
- df -h
unit-test:
name: unit-test
diff --git a/Cargo.toml b/Cargo.toml
index cee30e63..d195a121 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -80,7 +80,7 @@ members = [
"src/system_catalog",
"src/table_engine",
"src/tools",
- "src/wal",
+ "src/wal"
]
[workspace.dependencies]
diff --git a/src/benchmarks/bench.toml b/src/benchmarks/bench.toml
index 093898d5..9e9e0331 100644
--- a/src/benchmarks/bench.toml
+++ b/src/benchmarks/bench.toml
@@ -40,6 +40,7 @@ runtime_thread_num = 1
bench_measurement_time = "120s"
max_projections = 5
read_batch_row_num = 500
+sst_level = 0
[merge_sst_bench.predicate]
start_time_ms = 0
diff --git a/src/benchmarks/config/bench.toml b/src/benchmarks/config/bench.toml
index b3e707a9..41a08af7 100644
--- a/src/benchmarks/config/bench.toml
+++ b/src/benchmarks/config/bench.toml
@@ -42,6 +42,7 @@ bench_measurement_time = "30s"
bench_sample_size = 30
max_projections = 5
read_batch_row_num = 500
+sst_level = 0
[merge_sst_bench.predicate]
start_time_ms = 0
diff --git a/src/benchmarks/src/config.rs b/src/benchmarks/src/config.rs
index 63c3c329..b90a7cb5 100644
--- a/src/benchmarks/src/config.rs
+++ b/src/benchmarks/src/config.rs
@@ -85,6 +85,7 @@ pub struct MergeSstBenchConfig {
pub max_projections: usize,
pub num_rows_per_row_group: usize,
pub predicate: BenchPredicate,
+ pub sst_level: u16,
}
#[derive(Deserialize)]
diff --git a/src/benchmarks/src/merge_sst_bench.rs
b/src/benchmarks/src/merge_sst_bench.rs
index c71da73e..86fe3587 100644
--- a/src/benchmarks/src/merge_sst_bench.rs
+++ b/src/benchmarks/src/merge_sst_bench.rs
@@ -59,6 +59,7 @@ pub struct MergeSstBench {
file_handles: Vec<FileHandle>,
_receiver: UnboundedReceiver<Request>,
dedup: bool,
+ sst_level: Level,
}
impl MergeSstBench {
@@ -121,6 +122,7 @@ impl MergeSstBench {
file_handles,
_receiver: rx,
dedup: true,
+ sst_level: config.sst_level.into(),
}
}
@@ -168,7 +170,7 @@ impl MergeSstBench {
builder
// TODO: make level configurable
- .mut_ssts_of_level(Level::MIN)
+ .mut_ssts_of_level(self.sst_level)
.extend_from_slice(&self.file_handles);
self.runtime.block_on(async {
diff --git a/src/benchmarks/src/util.rs b/src/benchmarks/src/util.rs
index c173935f..e8fe664c 100644
--- a/src/benchmarks/src/util.rs
+++ b/src/benchmarks/src/util.rs
@@ -192,8 +192,8 @@ pub async fn file_handles_from_ssts(
let file_meta = FileMeta {
id: *file_id,
- size: 0,
- row_num: 0,
+ size: store.head(&path).await.unwrap().size as u64,
+ row_num: parquet_metadata.file_metadata().num_rows() as u64,
time_range: sst_meta.time_range,
max_seq: sst_meta.max_sequence,
storage_format: StorageFormat::Columnar,
diff --git a/src/df_engine_extensions/Cargo.toml
b/src/df_engine_extensions/Cargo.toml
index 940e6b1a..ec00b7a5 100644
--- a/src/df_engine_extensions/Cargo.toml
+++ b/src/df_engine_extensions/Cargo.toml
@@ -38,7 +38,7 @@ catalog = { workspace = true, features = ["test"] }
common_types = { workspace = true, features = ["test"] }
datafusion = { workspace = true }
datafusion-proto = { workspace = true }
-futures = { workspace = true }
+futures = { workspace = true }
generic_error = { workspace = true }
horaedbproto = { workspace = true }
lazy_static = { workspace = true }
diff --git a/src/horaedb/Cargo.toml b/src/horaedb/Cargo.toml
index 4abce863..38f22464 100644
--- a/src/horaedb/Cargo.toml
+++ b/src/horaedb/Cargo.toml
@@ -33,10 +33,7 @@ workspace = true
[features]
default = ["wal-rocksdb", "wal-table-kv", "wal-message-queue"]
wal-table-kv = ["wal/wal-table-kv", "analytic_engine/wal-table-kv"]
-wal-message-queue = [
- "wal/wal-message-queue",
- "analytic_engine/wal-message-queue",
-]
+wal-message-queue = ["wal/wal-message-queue",
"analytic_engine/wal-message-queue"]
wal-rocksdb = ["wal/wal-rocksdb", "analytic_engine/wal-rocksdb"]
[dependencies]
diff --git a/src/meta_client/Cargo.toml b/src/meta_client/Cargo.toml
index 727ae110..3ff32077 100644
--- a/src/meta_client/Cargo.toml
+++ b/src/meta_client/Cargo.toml
@@ -43,7 +43,7 @@ reqwest = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
snafu = { workspace = true }
-table_engine = { workspace = true }
+table_engine = { workspace = true }
time_ext = { workspace = true }
tokio = { workspace = true }
tonic = { workspace = true }
diff --git a/src/wal/Cargo.toml b/src/wal/Cargo.toml
index 3d11f0e1..6c8be8c1 100644
--- a/src/wal/Cargo.toml
+++ b/src/wal/Cargo.toml
@@ -27,15 +27,15 @@ workspace = true
[package.authors]
workspace = true
+[package.edition]
+workspace = true
+
[dependencies.rocksdb]
git = "https://github.com/tikv/rust-rocksdb.git";
rev = "f04f4dd8eacc30e67c24bc2529a6d9c6edb85f8f"
features = ["portable"]
optional = true
-[package.edition]
-workspace = true
-
[features]
wal-message-queue = ["dep:message_queue"]
wal-table-kv = ["dep:table_kv"]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
