This is an automated email from the ASF dual-hosted git repository.
csun5285 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 56a9fa7a093 [improvement](beut) add a temp file framework (#68494)
56a9fa7a093 is described below
commit 56a9fa7a093952b7bc7cb60a3ba92fe73755ecd6
Author: yiguolei <[email protected]>
AuthorDate: Fri Sep 25 15:21:12 2026 +0800
[improvement](beut) add a temp file framework (#68494)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
Currently, many be ut will create a temp file. But if two developers are
running beut concurrently on same machine, then then will try to access
the same file. For example, one process under user A may create temp
file /tmp/a and another process may want to delete it. In this case,
another developer will failed because the process does not have the
correct privilege.
I add temp dir framework to create temp file dir randomly to avoid this
problem.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../format/table/iceberg/iceberg_reader_test.cpp | 7 ++-
be/test/format_v2/json/json_reader_test.cpp | 6 +--
be/test/format_v2/table_reader_test.cpp | 8 ++-
.../kuromoji/kuromoji_dictionary_builder_test.cpp | 29 ++++++-----
.../analyzer/kuromoji/kuromoji_dictionary_test.cpp | 16 +++---
.../analyzer/kuromoji/kuromoji_real_dict_test.cpp | 13 +++--
.../analyzer/kuromoji/kuromoji_viterbi_test.cpp | 20 ++++----
be/test/testutil/scoped_temp_dir.h | 60 ++++++++++++++++++++++
8 files changed, 111 insertions(+), 48 deletions(-)
diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp
b/be/test/format/table/iceberg/iceberg_reader_test.cpp
index 0dee309fb7f..d736d2429b9 100644
--- a/be/test/format/table/iceberg/iceberg_reader_test.cpp
+++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp
@@ -72,6 +72,7 @@
#include "runtime/descriptors.h"
#include "runtime/runtime_state.h"
#include "storage/olap_scan_common.h"
+#include "testutil/scoped_temp_dir.h"
#include "util/timezone_utils.h"
namespace doris {
@@ -1753,10 +1754,8 @@ TEST_F(IcebergReaderTest,
v1_position_delete_consumes_delete_file_size) {
// An inflated size must break the read, proving the v1 equality-delete path
consumes the FE file_size.
TEST_F(IcebergReaderTest, v1_equality_delete_consumes_delete_file_size) {
- const auto test_dir = std::filesystem::temp_directory_path() /
"doris_v1_eq_delete_size_test";
- std::filesystem::remove_all(test_dir);
- std::filesystem::create_directories(test_dir);
- const auto delete_file_path = (test_dir /
"equality-delete.parquet").string();
+ const doris::test::ScopedTempDirectory
test_dir("doris_v1_eq_delete_size_test");
+ const auto delete_file_path = (test_dir.path() /
"equality-delete.parquet").string();
write_iceberg_int_equality_delete_parquet_file(delete_file_path, "id", 0,
2);
RuntimeState runtime_state = RuntimeState(TQueryOptions(),
TQueryGlobals());
diff --git a/be/test/format_v2/json/json_reader_test.cpp
b/be/test/format_v2/json/json_reader_test.cpp
index e8d9aec40af..c55bb3f512e 100644
--- a/be/test/format_v2/json/json_reader_test.cpp
+++ b/be/test/format_v2/json/json_reader_test.cpp
@@ -45,6 +45,7 @@
#include "runtime/descriptors.h"
#include "runtime/runtime_profile.h"
#include "testutil/mock/mock_runtime_state.h"
+#include "testutil/scoped_temp_dir.h"
namespace doris::format::json {
namespace {
@@ -136,9 +137,8 @@ std::unique_ptr<io::FileDescription> file_description(const
std::string& path) {
}
std::filesystem::path write_json_file(const std::string& name, const
std::string& content) {
- const auto test_dir = std::filesystem::temp_directory_path() /
"doris_format_v2_json_reader";
- std::filesystem::create_directories(test_dir);
- const auto file_path = test_dir / name;
+ static const doris::test::ScopedTempDirectory
test_dir("doris_format_v2_json_reader");
+ const auto file_path = test_dir.path() / name;
std::ofstream out(file_path);
out << content;
return file_path;
diff --git a/be/test/format_v2/table_reader_test.cpp
b/be/test/format_v2/table_reader_test.cpp
index e4741a63467..334ea1ce72b 100644
--- a/be/test/format_v2/table_reader_test.cpp
+++ b/be/test/format_v2/table_reader_test.cpp
@@ -66,6 +66,7 @@
#include "runtime/runtime_profile.h"
#include "runtime/runtime_state.h"
#include "storage/segment/condition_cache.h"
+#include "testutil/scoped_temp_dir.h"
namespace doris::format {
namespace {
@@ -2095,11 +2096,8 @@ TEST(TableReaderTest,
PendingRuntimeFilterDisablesTableLevelCount) {
}
TEST(TableReaderTest, CountStarFallbackKeepsLateRuntimeFilterCarrierValues) {
- const auto test_dir =
- std::filesystem::temp_directory_path() /
"doris_table_reader_count_star_late_rf_test";
- std::filesystem::remove_all(test_dir);
- std::filesystem::create_directories(test_dir);
- const auto file_path = (test_dir / "split.parquet").string();
+ const doris::test::ScopedTempDirectory
test_dir("doris_table_reader_count_star_late_rf_test");
+ const auto file_path = (test_dir.path() / "split.parquet").string();
write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5, 6}, {10, 20, 30,
40, 50, 60},
{"one", "two", "three", "four", "five",
"six"}, 2);
diff --git
a/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_dictionary_builder_test.cpp
b/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_dictionary_builder_test.cpp
index 8ee710dc597..86cd6f68d82 100644
---
a/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_dictionary_builder_test.cpp
+++
b/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_dictionary_builder_test.cpp
@@ -23,6 +23,7 @@
#include <string>
#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dict_format.h"
+#include "testutil/scoped_temp_dir.h"
namespace doris::segment_v2::inverted_index::kuromoji {
@@ -31,44 +32,48 @@ static bool file_nonempty(const std::string& p) {
return ::stat(p.c_str(), &st) == 0 && st.st_size >
static_cast<off_t>(sizeof(KmjFileHeader));
}
-TEST(KuromojiDictionaryBuilderTest, WritesFourFiles) {
- std::string dir = std::string(::testing::TempDir()) + "/kmj_build_test";
- ::mkdir(dir.c_str(), 0755);
+class KuromojiDictionaryBuilderTest : public ::testing::Test {
+protected:
+ doris::test::ScopedTempDirectory _temp_dir {"kuromoji_builder"};
+ std::string _dir = _temp_dir.path().string();
+};
+TEST_F(KuromojiDictionaryBuilderTest, WritesFourFiles) {
SystemDictInput sys;
sys.surfaces.push_back({"\xE6\x9D\xB1", {{1, 1, 100, "POS,East"}}});
// 東
sys.surfaces.push_back({"\xE6\x9D\xB1\xE4\xBA\xAC", {{2, 2, 50,
"POS,Tokyo"}}}); // 東京
- ASSERT_TRUE(KuromojiDictionaryBuilder::write_system(dir + "/system.bin",
sys).ok());
+ auto st = KuromojiDictionaryBuilder::write_system(_dir + "/system.bin",
sys);
+ ASSERT_TRUE(st.ok()) << st.to_string();
MatrixInput m;
m.forward_size = 3;
m.backward_size = 3;
m.cells.assign(9, 7);
- ASSERT_TRUE(KuromojiDictionaryBuilder::write_matrix(dir + "/matrix.bin",
m).ok());
+ ASSERT_TRUE(KuromojiDictionaryBuilder::write_matrix(_dir + "/matrix.bin",
m).ok());
CharDefInput cd;
cd.catmap.fill(CAT_DEFAULT);
cd.catmap[0x6771] = CAT_KANJI; // 東
cd.defs.assign(CAT_CLASS_COUNT, CategoryDef {0, 0, 0});
cd.defs[CAT_DEFAULT] = CategoryDef {1, 1, 0};
- ASSERT_TRUE(KuromojiDictionaryBuilder::write_chardef(dir + "/chardef.bin",
cd).ok());
+ ASSERT_TRUE(KuromojiDictionaryBuilder::write_chardef(_dir +
"/chardef.bin", cd).ok());
UnkDictInput unk;
unk.per_category.resize(CAT_CLASS_COUNT);
unk.per_category[CAT_DEFAULT].push_back({5, 5, 4769, "SYMBOL"});
- ASSERT_TRUE(KuromojiDictionaryBuilder::write_unkdict(dir + "/unkdict.bin",
unk).ok());
+ ASSERT_TRUE(KuromojiDictionaryBuilder::write_unkdict(_dir +
"/unkdict.bin", unk).ok());
- EXPECT_TRUE(file_nonempty(dir + "/system.bin"));
- EXPECT_TRUE(file_nonempty(dir + "/matrix.bin"));
- EXPECT_TRUE(file_nonempty(dir + "/chardef.bin"));
- EXPECT_TRUE(file_nonempty(dir + "/unkdict.bin"));
+ EXPECT_TRUE(file_nonempty(_dir + "/system.bin"));
+ EXPECT_TRUE(file_nonempty(_dir + "/matrix.bin"));
+ EXPECT_TRUE(file_nonempty(_dir + "/chardef.bin"));
+ EXPECT_TRUE(file_nonempty(_dir + "/unkdict.bin"));
// matrix rejects a wrong cell count.
MatrixInput bad;
bad.forward_size = 2;
bad.backward_size = 2;
bad.cells.assign(3, 0);
- EXPECT_FALSE(KuromojiDictionaryBuilder::write_matrix(dir + "/bad.bin",
bad).ok());
+ EXPECT_FALSE(KuromojiDictionaryBuilder::write_matrix(_dir + "/bad.bin",
bad).ok());
}
} // namespace doris::segment_v2::inverted_index::kuromoji
diff --git
a/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_dictionary_test.cpp
b/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_dictionary_test.cpp
index 4c7543eff47..8488529fe9d 100644
---
a/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_dictionary_test.cpp
+++
b/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_dictionary_test.cpp
@@ -18,10 +18,10 @@
#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary.h"
#include <gtest/gtest.h>
-#include <sys/stat.h>
#include <cstddef>
#include <cstring>
+#include <filesystem>
#include <fstream>
#include <iterator>
#include <memory>
@@ -30,21 +30,21 @@
#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dict_format.h"
#include
"storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary_builder.h"
+#include "testutil/scoped_temp_dir.h"
namespace doris::segment_v2::inverted_index::kuromoji {
class KuromojiDictionaryTest : public ::testing::Test {
protected:
- std::string _dir;
+ doris::test::ScopedTempDirectory _temp_dir {"kuromoji_dictionary"};
+ std::string _dir = _temp_dir.path().string();
void SetUp() override {
- _dir = std::string(::testing::TempDir()) + "/kmj_dict_rt";
- ::mkdir(_dir.c_str(), 0755);
-
SystemDictInput sys;
sys.surfaces.push_back({"\xE6\x9D\xB1", {{1, 1, 100, "f-east"}}});
// 東
sys.surfaces.push_back({"\xE6\x9D\xB1\xE4\xBA\xAC", {{2, 2, 50,
"f-tokyo"}}}); // 東京
- ASSERT_TRUE(KuromojiDictionaryBuilder::write_system(_dir +
"/system.bin", sys).ok());
+ auto st = KuromojiDictionaryBuilder::write_system(_dir +
"/system.bin", sys);
+ ASSERT_TRUE(st.ok()) << st.to_string();
MatrixInput m;
m.forward_size = 3;
@@ -108,8 +108,8 @@ TEST_F(KuromojiDictionaryTest, LoadAndQuery) {
TEST_F(KuromojiDictionaryTest, RejectsMissingFiles) {
// Point at a directory with no dictionary files -> must fail, not crash.
- std::string bad = std::string(::testing::TempDir()) + "/kmj_dict_bad";
- ::mkdir(bad.c_str(), 0755);
+ const std::string bad = _dir + "/missing_files";
+ ASSERT_TRUE(std::filesystem::create_directory(bad));
std::unique_ptr<KuromojiDictionary> dict;
EXPECT_FALSE(KuromojiDictionary::load(bad, &dict).ok());
}
diff --git
a/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_real_dict_test.cpp
b/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_real_dict_test.cpp
index 5f3950f2809..c140a669424 100644
---
a/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_real_dict_test.cpp
+++
b/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_real_dict_test.cpp
@@ -20,6 +20,7 @@
#include <algorithm>
#include <cstdlib>
+#include <filesystem>
#include <iostream>
#include <memory>
#include <string>
@@ -31,6 +32,7 @@
#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary.h"
#include "storage/index/inverted/analyzer/kuromoji/kuromoji_viterbi.h"
#include "storage/index/inverted/inverted_index_parser.h"
+#include "util/defer_op.h"
// End-to-end against the REAL IPADIC dictionary generated under
be/dict/kuromoji.
// Skips if the dictionary has not been generated on this host (e.g. plain CI),
@@ -125,11 +127,14 @@ TEST(KuromojiRealDictTest,
AnalyzerSegmentsViaBuiltinParser) {
if (!real_dict_present()) {
GTEST_SKIP() << "real IPADIC dictionary not generated at " <<
real_dict_dir();
}
- const char* home = std::getenv("DORIS_HOME");
const std::string saved = doris::config::inverted_index_dict_path;
- doris::config::inverted_index_dict_path =
- std::string(home != nullptr ? home : ".") + "/be/dict";
const bool saved_enable = doris::config::enable_kuromoji_analyzer;
+ Defer restore_config {[&] {
+ doris::config::inverted_index_dict_path = saved;
+ doris::config::enable_kuromoji_analyzer = saved_enable;
+ }};
+ doris::config::inverted_index_dict_path =
+ std::filesystem::path(real_dict_dir()).parent_path().string();
doris::config::enable_kuromoji_analyzer = true;
std::vector<std::string> toks;
@@ -152,8 +157,6 @@ TEST(KuromojiRealDictTest,
AnalyzerSegmentsViaBuiltinParser) {
toks.emplace_back(t.termBuffer<char>(),
static_cast<std::size_t>(t.termLength<char>()));
}
}
- doris::config::inverted_index_dict_path = saved;
- doris::config::enable_kuromoji_analyzer = saved_enable;
std::cout << "analyzer tokens for 東京都に住んでいます:";
for (const auto& t : toks) {
diff --git
a/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_viterbi_test.cpp
b/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_viterbi_test.cpp
index 553651aa5df..c2ec9aa5a0f 100644
--- a/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_viterbi_test.cpp
+++ b/be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_viterbi_test.cpp
@@ -18,7 +18,6 @@
#include "storage/index/inverted/analyzer/kuromoji/kuromoji_viterbi.h"
#include <gtest/gtest.h>
-#include <sys/stat.h>
#include <memory>
#include <string>
@@ -30,6 +29,7 @@
#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dict_format.h"
#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary.h"
#include
"storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary_builder.h"
+#include "testutil/scoped_temp_dir.h"
namespace doris::segment_v2::inverted_index::kuromoji {
@@ -40,20 +40,19 @@ static const std::string FU = "\xE5\xBA\x9C";
class KuromojiViterbiTest : public ::testing::Test {
protected:
- std::string _dir;
+ doris::test::ScopedTempDirectory _temp_dir {"kuromoji_viterbi"};
+ std::string _dir = _temp_dir.path().string();
std::unique_ptr<KuromojiDictionary> _dict;
void SetUp() override {
- _dir = std::string(::testing::TempDir()) + "/kmj_viterbi";
- ::mkdir(_dir.c_str(), 0755);
-
// Lexicon: single chars cost 1000 each; the compound "東京" costs 100.
// With an all-zero connection matrix, the min-cost path must pick
"東京".
SystemDictInput sys;
sys.surfaces.push_back({TOU, {{1, 1, 1000, "noun,East"}}});
sys.surfaces.push_back({KYO, {{1, 1, 1000, "noun,Capital"}}});
sys.surfaces.push_back({TOU + KYO, {{1, 1, 100, "noun,Tokyo"}}});
- ASSERT_TRUE(KuromojiDictionaryBuilder::write_system(_dir +
"/system.bin", sys).ok());
+ auto st = KuromojiDictionaryBuilder::write_system(_dir +
"/system.bin", sys);
+ ASSERT_TRUE(st.ok()) << st.to_string();
MatrixInput m;
m.forward_size = 2; // context ids: 0 = BOS/EOS, 1 = word
@@ -159,13 +158,11 @@ static const std::string KAWA = "\xE5\xB7\x9D";
// short (<= 2 kanji) compounds are never split.
class KuromojiSearchModeTest : public ::testing::Test {
protected:
- std::string _dir;
+ doris::test::ScopedTempDirectory _temp_dir {"kuromoji_search"};
+ std::string _dir = _temp_dir.path().string();
std::unique_ptr<KuromojiDictionary> _dict;
void SetUp() override {
- _dir = std::string(::testing::TempDir()) + "/kmj_search";
- ::mkdir(_dir.c_str(), 0755);
-
// Single kanji cost 1000 each; compounds "東京都" and "山川" cost 100.
SystemDictInput sys;
sys.surfaces.push_back({TOU, {{1, 1, 1000, "noun"}}});
@@ -175,7 +172,8 @@ protected:
sys.surfaces.push_back({KAWA, {{1, 1, 1000, "noun"}}});
sys.surfaces.push_back({TOU + KYO + TO, {{1, 1, 100, "noun"}}});
sys.surfaces.push_back({YAMA + KAWA, {{1, 1, 100, "noun"}}});
- ASSERT_TRUE(KuromojiDictionaryBuilder::write_system(_dir +
"/system.bin", sys).ok());
+ auto st = KuromojiDictionaryBuilder::write_system(_dir +
"/system.bin", sys);
+ ASSERT_TRUE(st.ok()) << st.to_string();
MatrixInput m;
m.forward_size = 2;
diff --git a/be/test/testutil/scoped_temp_dir.h
b/be/test/testutil/scoped_temp_dir.h
new file mode 100644
index 00000000000..0c5e02a95ed
--- /dev/null
+++ b/be/test/testutil/scoped_temp_dir.h
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <unistd.h>
+
+#include <cerrno>
+#include <filesystem>
+#include <string>
+#include <string_view>
+#include <system_error>
+#include <utility>
+
+namespace doris::test {
+
+class ScopedTempDirectory {
+public:
+ explicit ScopedTempDirectory(std::string_view prefix) {
+ std::string path_template =
+ (std::filesystem::temp_directory_path() / (std::string(prefix)
+ "_XXXXXX"))
+ .string();
+ if (::mkdtemp(path_template.data()) == nullptr) {
+ const int error = errno;
+ throw std::filesystem::filesystem_error(
+ "cannot create temporary directory", path_template,
+ std::error_code(error, std::generic_category()));
+ }
+ _path = std::move(path_template);
+ }
+
+ ~ScopedTempDirectory() {
+ std::error_code error;
+ std::filesystem::remove_all(_path, error);
+ }
+
+ ScopedTempDirectory(const ScopedTempDirectory&) = delete;
+ ScopedTempDirectory& operator=(const ScopedTempDirectory&) = delete;
+
+ const std::filesystem::path& path() const { return _path; }
+
+private:
+ std::filesystem::path _path;
+};
+
+} // namespace doris::test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
