This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new b0033c83d feat: Add some useful time and filesystem utilies (#2020)
b0033c83d is described below
commit b0033c83d140ef9af514816b2664f5305ff80259
Author: Yingchun Lai <[email protected]>
AuthorDate: Thu May 23 19:31:58 2024 +0800
feat: Add some useful time and filesystem utilies (#2020)
---
src/utils/filesystem.cpp | 39 +++++++++++++++++++++++++++++++++++++
src/utils/filesystem.h | 10 ++++++++++
src/utils/test/file_system_test.cpp | 34 ++++++++++++++++++++++++++++++++
src/utils/test/time_utils_test.cpp | 25 +++++++++++++++++++-----
src/utils/time_utils.cpp | 15 ++++++++++++--
src/utils/time_utils.h | 5 ++++-
6 files changed, 120 insertions(+), 8 deletions(-)
diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp
index ec414767c..1a7b3dd21 100644
--- a/src/utils/filesystem.cpp
+++ b/src/utils/filesystem.cpp
@@ -25,10 +25,12 @@
*/
#include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/path.hpp>
#include <boost/system/error_code.hpp>
#include <errno.h>
#include <fmt/core.h>
#include <ftw.h>
+#include <glob.h>
#include <limits.h>
#include <openssl/md5.h>
#include <rocksdb/env.h>
@@ -42,6 +44,7 @@
#include <memory>
#include "absl/strings/string_view.h"
+#include "errors.h"
#include "utils/defer.h"
#include "utils/env.h"
#include "utils/fail_point.h"
@@ -492,6 +495,12 @@ bool create_file(const std::string &path)
return true;
}
+bool is_absolute_path(const std::string &path)
+{
+ boost::filesystem::path p(path);
+ return p.is_absolute();
+}
+
bool get_absolute_path(const std::string &path1, std::string &path2)
{
bool succ;
@@ -901,6 +910,36 @@ bool check_dir_rw(const std::string &path, std::string
&err_msg)
return true;
}
+error_s glob(const std::string &path_pattern, std::vector<std::string>
&path_list)
+{
+ glob_t result;
+ auto cleanup = dsn::defer([&] { ::globfree(&result); });
+
+ errno = 0;
+ int ret = ::glob(path_pattern.c_str(), GLOB_TILDE | GLOB_ERR, NULL,
&result);
+ switch (ret) {
+ case 0:
+ break;
+
+ case GLOB_NOMATCH:
+ return error_s::ok();
+
+ case GLOB_NOSPACE:
+ return error_s::make(ERR_FS_INTERNAL, "glob out of memory");
+
+ default:
+ std::string error(errno == 0 ? "unknown error" : safe_strerror(errno));
+ return error_s::make(ERR_FS_INTERNAL,
+ fmt::format("glob failed for '{}': {}",
path_pattern, error));
+ }
+
+ for (size_t i = 0; i < result.gl_pathc; ++i) {
+ path_list.emplace_back(result.gl_pathv[i]);
+ }
+
+ return error_s::ok();
+}
+
} // namespace filesystem
} // namespace utils
} // namespace dsn
diff --git a/src/utils/filesystem.h b/src/utils/filesystem.h
index 29524c9c3..3c370044c 100644
--- a/src/utils/filesystem.h
+++ b/src/utils/filesystem.h
@@ -32,6 +32,7 @@
#include <utility>
#include <vector>
+#include "utils/errors.h"
#include "utils/error_code.h"
#ifndef _XOPEN_SOURCE
@@ -69,6 +70,8 @@ namespace filesystem {
void get_normalized_path(const std::string &path, std::string &npath);
+bool is_absolute_path(const std::string &path);
+
bool get_absolute_path(const std::string &path1, std::string &path2);
std::string remove_file_name(const std::string &path);
@@ -165,6 +168,13 @@ bool create_directory(const std::string &path,
// call `create_directory` before to make `path` exist
bool check_dir_rw(const std::string &path, /*out*/ std::string &err_msg);
+// Finds paths on the filesystem matching a pattern.
+//
+// The found pathnames are added to the 'paths' vector. If no pathnames are
+// found matching the pattern, no paths are added to the vector and an OK
+// status is returned.
+error_s glob(const std::string &path_pattern, std::vector<std::string>
&path_list);
+
} // namespace filesystem
} // namespace utils
} // namespace dsn
diff --git a/src/utils/test/file_system_test.cpp
b/src/utils/test/file_system_test.cpp
index 66a224f9e..dbe0b901a 100644
--- a/src/utils/test/file_system_test.cpp
+++ b/src/utils/test/file_system_test.cpp
@@ -18,9 +18,12 @@
#include <rocksdb/env.h>
#include <rocksdb/slice.h>
#include <rocksdb/status.h>
+#include <stddef.h>
#include <stdint.h>
#include <set>
#include <string>
+#include <utility>
+#include <vector>
#include "gtest/gtest.h"
#include "utils/env.h"
@@ -175,6 +178,37 @@ TEST(filesystem_test, verify_file_test)
remove_path(fname);
}
+TEST(filesystem_test, absolute_path_test)
+{
+ const std::string kTestDir = "absolute_path_test";
+ ASSERT_TRUE(create_directory(kTestDir));
+ ASSERT_FALSE(is_absolute_path(kTestDir));
+
+ std::string abs_path;
+ ASSERT_TRUE(get_absolute_path(kTestDir, abs_path));
+ ASSERT_TRUE(is_absolute_path(abs_path));
+}
+
+TEST(filesystem_test, glob_test)
+{
+ const std::string kTestDir = "glob_test";
+ ASSERT_TRUE(create_directory(kTestDir));
+ std::vector<std::string> filenames = {"fuzz", "fuzzy", "fuzzyiest",
"buzz"};
+ std::vector<std::pair<std::string, size_t>> matchers = {
+ {"file", 0}, {"fuzz", 1}, {"fuzz*", 3}, {"?uzz", 2},
+ };
+
+ for (const auto &name : filenames) {
+ ASSERT_TRUE(create_file(path_combine(kTestDir, name)));
+ }
+
+ for (const auto & [ path_pattern, matched_count ] : matchers) {
+ std::vector<std::string> matches;
+ ASSERT_TRUE(glob(path_combine(kTestDir, path_pattern), matches)) <<
path_pattern;
+ ASSERT_EQ(matched_count, matches.size()) << path_pattern;
+ }
+}
+
} // namespace filesystem
} // namespace utils
} // namespace dsn
diff --git a/src/utils/test/time_utils_test.cpp
b/src/utils/test/time_utils_test.cpp
index 35838aaf7..ec952637c 100644
--- a/src/utils/test/time_utils_test.cpp
+++ b/src/utils/test/time_utils_test.cpp
@@ -90,16 +90,16 @@ TEST(time_utils, get_current_physical_time_ns)
template <typename T>
void test_time_ms_to_string(T &str)
{
- time_ms_to_string(1605091506136, str);
+ time_ms_to_string(1605091506036, str);
std::string actual_str(str);
- // Time differ between time zones.
+ // Time differs between time zones.
//
- // The real time 2020-11-11 18:45:06.136 (UTC+8)
- // so it must be 2020-11-1x xx:45:06.136.
+ // The real time 2020-11-11 18:45:06.036 (UTC+8)
+ // so it must be 2020-11-1x xx:45:06.036.
ASSERT_EQ(std::string("2020-11-1"), actual_str.substr(0, 9));
- ASSERT_EQ(std::string(":45:06.136"), actual_str.substr(13, 10));
+ ASSERT_EQ(std::string(":45:06.036"), actual_str.substr(13, 10));
}
TEST(time_utils, time_ms_to_buf)
@@ -114,5 +114,20 @@ TEST(time_utils, time_ms_to_str)
test_time_ms_to_string(str);
}
+TEST(time_utils, time_ms_to_sequent_str)
+{
+ std::string str;
+ time_ms_to_sequent_string(1605091506036, str);
+
+ std::string actual_str(str);
+
+ // Time differs between time zones.
+ //
+ // The real time 20201111_184506_036 (UTC+8)
+ // so it must be 2020111x_xx4506_036.
+ ASSERT_EQ(std::string("2020111"), actual_str.substr(0, 7));
+ ASSERT_EQ(std::string("4506_036"), actual_str.substr(11, 8));
+}
+
} // namespace utils
} // namespace dsn
diff --git a/src/utils/time_utils.cpp b/src/utils/time_utils.cpp
index 34504fc99..e8d564814 100644
--- a/src/utils/time_utils.cpp
+++ b/src/utils/time_utils.cpp
@@ -36,7 +36,7 @@ namespace utils {
auto ret = get_localtime(ts_ms, &tmp);
// NOTE: format_to() does not append a terminating null character, so
remember to initialize
// str's memory as zero before.
- fmt::format_to(str, "{:%Y-%m-%d %H:%M:%S}.{}", *ret,
static_cast<uint32_t>(ts_ms % 1000));
+ fmt::format_to(str, "{:%Y-%m-%d %H:%M:%S}.{:03}", *ret,
static_cast<uint32_t>(ts_ms % 1000));
}
/*extern*/ void time_ms_to_string(uint64_t ts_ms, std::string &str)
@@ -45,7 +45,18 @@ namespace utils {
struct tm tmp;
auto ret = get_localtime(ts_ms, &tmp);
fmt::format_to(std::back_inserter(str),
- "{:%Y-%m-%d %H:%M:%S}.{}",
+ "{:%Y-%m-%d %H:%M:%S}.{:03}",
+ *ret,
+ static_cast<uint32_t>(ts_ms % 1000));
+}
+
+/*extern*/ void time_ms_to_sequent_string(uint64_t ts_ms, std::string &str)
+{
+ str.clear();
+ struct tm tmp;
+ auto ret = get_localtime(ts_ms, &tmp);
+ fmt::format_to(std::back_inserter(str),
+ "{:%Y%m%d_%H%M%S}_{:03}",
*ret,
static_cast<uint32_t>(ts_ms % 1000));
}
diff --git a/src/utils/time_utils.h b/src/utils/time_utils.h
index f89eebf8f..546bea0ef 100644
--- a/src/utils/time_utils.h
+++ b/src/utils/time_utils.h
@@ -47,13 +47,16 @@ static struct tm *get_localtime(uint64_t ts_ms, struct tm
*tm_buf)
return localtime_r(&t, tm_buf);
}
-// get time string, which format is yyyy-MM-dd hh:mm:ss.SSS
+// Get time string, which format is yyyy-MM-dd hh:mm:ss.SSS
// NOTE: using char* as output is usually unsafe, remember to initialize its
memory as zero before
// calling 'time_ms_to_string'. Please use std::string as the output argument
as long as it's
// possible.
extern void time_ms_to_string(uint64_t ts_ms, char *str);
extern void time_ms_to_string(uint64_t ts_ms, std::string &str);
+// Get time string, which format is yyyyMMdd_hhmmss_SSS
+extern void time_ms_to_sequent_string(uint64_t ts_ms, std::string &str);
+
// get date string with format of 'yyyy-MM-dd' from given timestamp
inline void time_ms_to_date(uint64_t ts_ms, char *str, int len)
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
