This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new a71131ffb chore: clean up cgo flags and conditionally compile grocksdb
(#2885)
a71131ffb is described below
commit a71131ffb445e953458918579ea62ae0bca0a8d1
Author: Twice <[email protected]>
AuthorDate: Sat Apr 19 19:03:39 2025 +0800
chore: clean up cgo flags and conditionally compile grocksdb (#2885)
Signed-off-by: PragmaTwice <[email protected]>
---
.github/workflows/kvrocks.yaml | 7 ++--
src/storage/storage.cc | 31 +++++------------
tests/gocase/go.mod | 4 +--
tests/gocase/unit/sst/sst_load_test.go | 2 +-
x.py | 61 ++++++----------------------------
5 files changed, 25 insertions(+), 80 deletions(-)
diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml
index de4e26752..570577991 100644
--- a/.github/workflows/kvrocks.yaml
+++ b/.github/workflows/kvrocks.yaml
@@ -143,11 +143,11 @@ jobs:
- name: Ubuntu GCC
os: ubuntu-22.04
compiler: gcc
+ enable_grocksdb: -tags=enable_grocksdb
- name: SonarCloud with Coverage
os: ubuntu-22.04
compiler: gcc
sonarcloud: -DCMAKE_CXX_FLAGS=--coverage
- with_coverage: "true"
- name: Ubuntu Clang
os: ubuntu-22.04
compiler: clang
@@ -321,9 +321,6 @@ jobs:
export PATH=$PATH:$HOME/local/bin/
export CGO_ENABLED=1
GOCASE_RUN_ARGS=""
- if [[ -n "${{ matrix.with_coverage }}" ]]; then
- export WITH_COVERAGE="true"
- fi
if [[ -n "${{ matrix.with_openssl }}" ]] && [[ "${{ matrix.os }}" ==
ubuntu* ]]; then
git clone https://github.com/jsha/minica
cd minica && git checkout v1.1.0 && go build && cd ..
@@ -333,7 +330,7 @@ jobs:
cp minica.pem tests/gocase/tls/cert/ca.crt
GOCASE_RUN_ARGS="-tlsEnable"
fi
- ./x.py test go build -parallel 2 $GOCASE_RUN_ARGS ${{
matrix.ignore_when_tsan}} ${{ matrix.ignore_when_asan}} ${{
matrix.ignore_when_ubsan}}
+ ./x.py test go build -parallel 2 $GOCASE_RUN_ARGS ${{
matrix.ignore_when_tsan}} ${{ matrix.ignore_when_asan}} ${{
matrix.ignore_when_ubsan}} ${{ matrix.enable_grocksdb }}
- name: Install redis-py
run: pip3 install redis==5.2.0
diff --git a/src/storage/storage.cc b/src/storage/storage.cc
index b880f86d7..4738c1a3f 100644
--- a/src/storage/storage.cc
+++ b/src/storage/storage.cc
@@ -20,7 +20,6 @@
#include "storage.h"
-#include <dirent.h>
#include <event2/buffer.h>
#include <fcntl.h>
#include <glog/logging.h>
@@ -783,24 +782,19 @@ StatusOr<int> Storage::IngestSST(const std::string
&sst_dir, const rocksdb::Inge
filtered_files.push_back(sst_dir + "/" + filename);
}
}
- sst_files = std::move(filtered_files);
+ sst_files = std::move(filtered_files);
if (sst_files.empty()) {
LOG(WARNING) << "No SST files found in " << sst_dir;
return 0;
}
- std::unordered_map<std::string_view, std::vector<std::string>> cf_files;
- std::vector<std::string> cf_names;
- for (const auto &cf : ColumnFamilyConfigs::ListAllColumnFamilies()) {
- cf_names.emplace_back(cf.Name());
- }
-
+ std::unordered_map<ColumnFamilyID, std::vector<std::string>> cf_files;
for (const auto &file : sst_files) {
bool matched = false;
- for (const auto &cf_name : cf_names) {
- if (file.find(cf_name) != std::string::npos) {
- cf_files[cf_name].push_back(file);
+ for (const auto &cf : ColumnFamilyConfigs::ListAllColumnFamilies()) {
+ if (file.find(cf.Name()) != std::string::npos) {
+ cf_files[cf.Id()].push_back(file);
matched = true;
break;
}
@@ -816,18 +810,11 @@ StatusOr<int> Storage::IngestSST(const std::string
&sst_dir, const rocksdb::Inge
// if the metadata import fails, the imported data will be deleted by the
compaction.
rocksdb::Status status;
// Process files for each column family except metadata
- for (const auto &[cf_name, files] : cf_files) {
- if (cf_name == kMetadataColumnFamilyName) continue;
+ for (const auto &[cf, files] : cf_files) {
+ if (cf == ColumnFamilyID::Metadata) continue;
if (files.empty()) continue;
- rocksdb::ColumnFamilyHandle *cf_handle = nullptr;
- // Find the correct CF handle
- for (auto handle : cf_handles_) {
- if (handle->GetName() == cf_name) {
- cf_handle = handle;
- break;
- }
- }
+ rocksdb::ColumnFamilyHandle *cf_handle = GetCFHandle(cf);
status = ingestSST(cf_handle, ingest_options, files);
if (!status.ok()) {
@@ -835,7 +822,7 @@ StatusOr<int> Storage::IngestSST(const std::string
&sst_dir, const rocksdb::Inge
}
}
// Process metadata files
- const auto &metadata_files = cf_files[kMetadataColumnFamilyName];
+ const auto &metadata_files = cf_files[ColumnFamilyID::Metadata];
if (!metadata_files.empty()) {
status = ingestSST(GetCFHandle(ColumnFamilyID::Metadata), ingest_options,
metadata_files);
if (!status.ok()) {
diff --git a/tests/gocase/go.mod b/tests/gocase/go.mod
index 4160e3919..37ca0f112 100644
--- a/tests/gocase/go.mod
+++ b/tests/gocase/go.mod
@@ -3,10 +3,12 @@ module github.com/apache/kvrocks/tests/gocase
go 1.23.0
require (
+ github.com/linxGnu/grocksdb v1.9.9
github.com/redis/go-redis/v9 v9.7.3
github.com/shirou/gopsutil/v4 v4.25.2
github.com/stretchr/testify v1.10.0
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394
+ golang.org/x/sync v0.12.0
)
require (
@@ -15,14 +17,12 @@ require (
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f //
indirect
github.com/ebitengine/purego v0.8.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
- github.com/linxGnu/grocksdb v1.9.9
github.com/lufia/plan9stats v0.0.0-20250303091104-876f3ea5145d //
indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 //
indirect
github.com/tklauser/go-sysconf v0.3.15 // indirect
github.com/tklauser/numcpus v0.10.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
- golang.org/x/sync v0.12.0
golang.org/x/sys v0.31.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/tests/gocase/unit/sst/sst_load_test.go
b/tests/gocase/unit/sst/sst_load_test.go
index cd0e264a6..d2c20bc11 100644
--- a/tests/gocase/unit/sst/sst_load_test.go
+++ b/tests/gocase/unit/sst/sst_load_test.go
@@ -1,4 +1,4 @@
-//go:build !(ignore_when_tsan || ignore_when_asan || ignore_when_ubsan)
+//go:build enable_grocksdb
/*
* Licensed to the Apache Software Foundation (ASF) under one
diff --git a/x.py b/x.py
index 5ab66c07c..787eecade 100755
--- a/x.py
+++ b/x.py
@@ -223,56 +223,17 @@ def clang_tidy(dir: str, jobs: Optional[int],
clang_tidy_path: str, run_clang_ti
run(run_command, *options, *regexes, verbose=True, cwd=basedir)
-def is_rocky_linux():
- try:
- with open('/etc/os-release') as f:
- lines = f.readlines()
- data = {}
- for line in lines:
- if '=' in line:
- key, value = line.strip().split('=', 1)
- data[key] = value.strip('"')
-
- is_rocky = data.get('ID') == 'rocky'
- version_id = data.get('VERSION_ID', '').split('.')[0]
- version_match = version_id in ('8', '9')
-
- return is_rocky and version_match
- except FileNotFoundError:
- print("File /etc/os-release not found.")
- return False
- except Exception as e:
- print(f"Unexpected error: {e}")
- return False
-
-
-def get_custom_env():
- rocksdb =
Path(__file__).parent.absolute().joinpath('build/_deps/rocksdb-src/include')
- rocksdb_lib =
Path(__file__).parent.absolute().joinpath('build/_deps/rocksdb-build')
- zlib =
Path(__file__).parent.absolute().joinpath('build/_deps/zstd-src/lib')
- z4lib =
Path(__file__).parent.absolute().joinpath('build/_deps/lz4-src/lib')
- snappy_lib =
Path(__file__).parent.absolute().joinpath('build/_deps/snappy-build')
-
- additional_flags = ""
- libstdc_folder = ""
- env = os.environ.copy()
-
- if is_rocky_linux():
- output = run_pipe("find", "/opt/rh/gcc-toolset-12/root/", "-name",
"libstdc++.so*")
- output = run_pipe("grep", "-v", "32",stdin=output)
- output = run_pipe("xargs", "dirname", stdin=output)
- libstdc_folder = output.read().strip() + "/"
-
- if libstdc_folder != "":
- additional_flags = f"-L{libstdc_folder} -lstdc++"
-
- with_cov = env.get("WITH_COVERAGE","false")
- if with_cov == "true":
- additional_flags += "-lgcov"
+def env_with_cgo_flags(build_dir: Path):
+ rocksdb_inc = build_dir.joinpath('_deps/rocksdb-src/include')
+ rocksdb_lib = build_dir.joinpath('_deps/rocksdb-build')
+ zlib_lib = build_dir.joinpath('_deps/zstd-src/lib')
+ lz4_lib = build_dir.joinpath('_deps/lz4-src/lib')
+ snappy_lib = build_dir.joinpath('_deps/snappy-build')
+ env = os.environ.copy()
env.update({
- "CGO_CFLAGS": f"-I{rocksdb}",
- "CGO_LDFLAGS": f"-L{rocksdb_lib} -L{zlib} -L{z4lib} -L{snappy_lib}
{additional_flags}",
+ "CGO_CFLAGS": f"-I{rocksdb_inc} {env.get('CGO_CFLAGS', '')}",
+ "CGO_LDFLAGS": f"-L{rocksdb_lib} -L{zlib_lib} -L{lz4_lib}
-L{snappy_lib} {env.get('CGO_LDFLAGS', '')}",
})
return env
@@ -312,7 +273,7 @@ def golangci_lint(golangci_lint_path: str) -> None:
check_version(version_str, GOLANGCI_LINT_REQUIRED_VERSION,
"golangci-lint")
basedir = Path(__file__).parent.absolute() / 'tests' / 'gocase'
- run(binpath_str, 'run', '-v', './...', cwd=str(basedir), verbose=True,
env=get_custom_env())
+ run(binpath_str, 'run', '-v', './...', cwd=str(basedir), verbose=True)
def write_version(release_version: str) -> str:
@@ -376,7 +337,7 @@ def test_go(dir: str, cli_path: str, rest: List[str]) ->
None:
*rest
]
- run(go, *args, cwd=str(basedir), verbose=True, env=get_custom_env())
+ run(go, *args, cwd=str(basedir), verbose=True,
env=env_with_cgo_flags(Path(dir).absolute()))
if __name__ == '__main__':
