This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new b75bd0388 feat(json_file): load and dump functions of JSON objects
support encryption (#1842)
b75bd0388 is described below
commit b75bd038869730de4e84b2597875452868c4ff53
Author: Yingchun Lai <[email protected]>
AuthorDate: Wed Jan 10 11:40:30 2024 +0800
feat(json_file): load and dump functions of JSON objects support encryption
(#1842)
---
src/utils/load_dump_object.h | 64 +++++++++++++++++++++-----------
src/utils/test/load_dump_object_test.cpp | 55 ++++++++++++++++-----------
2 files changed, 75 insertions(+), 44 deletions(-)
diff --git a/src/utils/load_dump_object.h b/src/utils/load_dump_object.h
index a2ee03167..a4127ddb9 100644
--- a/src/utils/load_dump_object.h
+++ b/src/utils/load_dump_object.h
@@ -65,11 +65,18 @@ error_code write_data_to_file(const std::string &file_path,
const T &data, const
return ERR_OK;
}
-// Load a object from the file in 'file_path' to 'obj', the file content must
be in JSON format.
+// Load a object from the file in 'file_path' to 'obj', while 'type' decide
whether the data is
+// encrypted, the file content must be in JSON format.
// The object T must use the marco DEFINE_JSON_SERIALIZATION(...) when
defining, which implement
// the RapidJson APIs.
template <class T>
error_code load_rjobj_from_file(const std::string &file_path, T *obj)
+{
+ return load_rjobj_from_file(file_path, FileDataType::kSensitive, obj);
+}
+
+template <class T>
+error_code load_rjobj_from_file(const std::string &file_path, FileDataType
type, T *obj)
{
LOG_AND_RETURN_NOT_TRUE(ERROR,
filesystem::path_exists(file_path),
@@ -77,12 +84,11 @@ error_code load_rjobj_from_file(const std::string
&file_path, T *obj)
"file '{}' not exist",
file_path);
std::string data;
- LOG_AND_RETURN_CODE_NOT_RDB_OK(
- ERROR,
- rocksdb::ReadFileToString(PegasusEnv(FileDataType::kSensitive),
file_path, &data),
- ERR_FILE_OPERATION_FAILED,
- "read file '{}' failed",
- file_path);
+ LOG_AND_RETURN_CODE_NOT_RDB_OK(ERROR,
+ rocksdb::ReadFileToString(PegasusEnv(type),
file_path, &data),
+ ERR_FILE_OPERATION_FAILED,
+ "read file '{}' failed",
+ file_path);
LOG_AND_RETURN_NOT_TRUE(
ERROR,
json::json_forwarder<T>::decode(blob::create_from_bytes(std::move(data)), *obj),
@@ -92,18 +98,23 @@ error_code load_rjobj_from_file(const std::string
&file_path, T *obj)
return ERR_OK;
}
-// Dump the object to the file in 'file_path' according to 'obj', the file
content will be in JSON
+// Dump the object to the file in 'file_path' according to 'obj', while 'type'
decide whether the
+// data is encrypted, the file content will be in JSON
// format.
// The object T must use the marco DEFINE_JSON_SERIALIZATION(...) when
defining, which implement
// the RapidJson APIs.
template <class T>
error_code dump_rjobj_to_file(const T &obj, const std::string &file_path)
+{
+ return dump_rjobj_to_file(obj, FileDataType::kSensitive, file_path);
+}
+
+template <class T>
+error_code dump_rjobj_to_file(const T &obj, FileDataType type, const
std::string &file_path)
{
const auto data = json::json_forwarder<T>::encode(obj);
- LOG_AND_RETURN_NOT_OK(ERROR,
- write_data_to_file(file_path, data,
FileDataType::kSensitive),
- "dump content to '{}' failed",
- file_path);
+ LOG_AND_RETURN_NOT_OK(
+ ERROR, write_data_to_file(file_path, data, type), "dump content to
'{}' failed", file_path);
return ERR_OK;
}
@@ -112,6 +123,12 @@ error_code dump_rjobj_to_file(const T &obj, const
std::string &file_path)
// which implement the NlohmannJson APIs.
template <class T>
error_code load_njobj_from_file(const std::string &file_path, T *obj)
+{
+ return load_njobj_from_file(file_path, FileDataType::kSensitive, obj);
+}
+
+template <class T>
+error_code load_njobj_from_file(const std::string &file_path, FileDataType
type, T *obj)
{
LOG_AND_RETURN_NOT_TRUE(ERROR,
filesystem::path_exists(file_path),
@@ -119,12 +136,11 @@ error_code load_njobj_from_file(const std::string
&file_path, T *obj)
"file '{}' not exist",
file_path);
std::string data;
- LOG_AND_RETURN_CODE_NOT_RDB_OK(
- ERROR,
- rocksdb::ReadFileToString(PegasusEnv(FileDataType::kSensitive),
file_path, &data),
- ERR_FILE_OPERATION_FAILED,
- "read file '{}' failed",
- file_path);
+ LOG_AND_RETURN_CODE_NOT_RDB_OK(ERROR,
+ rocksdb::ReadFileToString(PegasusEnv(type),
file_path, &data),
+ ERR_FILE_OPERATION_FAILED,
+ "read file '{}' failed",
+ file_path);
try {
nlohmann::json::parse(data).get_to(*obj);
} catch (nlohmann::json::exception &exp) {
@@ -142,12 +158,16 @@ error_code load_njobj_from_file(const std::string
&file_path, T *obj)
// which implement the NlohmannJson APIs.
template <class T>
error_code dump_njobj_to_file(const T &obj, const std::string &file_path)
+{
+ return dump_njobj_to_file(obj, FileDataType::kSensitive, file_path);
+}
+
+template <class T>
+error_code dump_njobj_to_file(const T &obj, FileDataType type, const
std::string &file_path)
{
const auto data = nlohmann::json(obj).dump();
- LOG_AND_RETURN_NOT_OK(ERROR,
- write_data_to_file(file_path, data,
FileDataType::kSensitive),
- "dump content to '{}' failed",
- file_path);
+ LOG_AND_RETURN_NOT_OK(
+ ERROR, write_data_to_file(file_path, data, type), "dump content to
'{}' failed", file_path);
return ERR_OK;
}
diff --git a/src/utils/test/load_dump_object_test.cpp
b/src/utils/test/load_dump_object_test.cpp
index 4b84de33c..0b6a33d89 100644
--- a/src/utils/test/load_dump_object_test.cpp
+++ b/src/utils/test/load_dump_object_test.cpp
@@ -43,35 +43,47 @@ struct nlohmann_json_struct
};
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(nlohmann_json_struct, a, b, c);
-TEST(load_dump_object, nlohmann_json_struct_normal_test)
+class load_dump_object : public testing::TestWithParam<FileDataType>
+{
+public:
+ load_dump_object() : type_(GetParam()) {}
+
+protected:
+ FileDataType type_;
+};
+
+INSTANTIATE_TEST_CASE_P(,
+ load_dump_object,
+ ::testing::Values(FileDataType::kNonSensitive,
FileDataType::kSensitive));
+
+TEST_P(load_dump_object, nlohmann_json_struct_normal_test)
{
const std::string path("nlohmann_json_struct_test");
nlohmann_json_struct obj;
obj.a = 123;
obj.b = "hello world";
obj.c = std::vector<int64_t>({1, 3, 5, 2, 4});
- ASSERT_EQ(ERR_OK, dump_njobj_to_file(obj, path));
+ ASSERT_EQ(ERR_OK, dump_njobj_to_file(obj, type_, path));
nlohmann_json_struct obj2;
- ASSERT_EQ(ERR_OK, load_njobj_from_file(path, &obj2));
+ ASSERT_EQ(ERR_OK, load_njobj_from_file(path, type_, &obj2));
ASSERT_EQ(obj, obj2);
}
-TEST(load_dump_object, nlohmann_json_struct_load_failed_test)
+TEST_P(load_dump_object, nlohmann_json_struct_load_failed_test)
{
const std::string path("nlohmann_json_struct_test_bad");
ASSERT_TRUE(filesystem::remove_path(path));
nlohmann_json_struct obj;
- ASSERT_EQ(ERR_PATH_NOT_FOUND, load_njobj_from_file(path, &obj));
+ ASSERT_EQ(ERR_PATH_NOT_FOUND, load_njobj_from_file(path, type_, &obj));
- auto s =
-
rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive),
- rocksdb::Slice("invalid data"),
- path,
- /* should_sync */ true);
+ auto s = rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(type_),
+ rocksdb::Slice("invalid data"),
+ path,
+ /* should_sync */ true);
ASSERT_TRUE(s.ok()) << s.ToString();
- ASSERT_EQ(ERR_CORRUPTION, load_njobj_from_file(path, &obj));
+ ASSERT_EQ(ERR_CORRUPTION, load_njobj_from_file(path, type_, &obj));
}
struct rapid_json_struct
@@ -80,35 +92,34 @@ struct rapid_json_struct
DEFINE_JSON_SERIALIZATION(a, b, c);
};
-TEST(load_dump_object, rapid_json_struct_test)
+TEST_P(load_dump_object, rapid_json_struct_test)
{
const std::string path("rapid_json_struct_test");
rapid_json_struct obj;
obj.a = 123;
obj.b = "hello world";
obj.c = std::vector<int64_t>({1, 3, 5, 2, 4});
- ASSERT_EQ(ERR_OK, dump_rjobj_to_file(obj, path));
+ ASSERT_EQ(ERR_OK, dump_rjobj_to_file(obj, type_, path));
rapid_json_struct obj2;
- ASSERT_EQ(ERR_OK, load_rjobj_from_file(path, &obj2));
+ ASSERT_EQ(ERR_OK, load_rjobj_from_file(path, type_, &obj2));
ASSERT_EQ(obj, obj2);
}
-TEST(load_dump_object, rapid_json_struct_load_failed_test)
+TEST_P(load_dump_object, rapid_json_struct_load_failed_test)
{
const std::string path("rapid_json_struct_test_bad");
ASSERT_TRUE(filesystem::remove_path(path));
rapid_json_struct obj;
- ASSERT_EQ(ERR_PATH_NOT_FOUND, load_rjobj_from_file(path, &obj));
+ ASSERT_EQ(ERR_PATH_NOT_FOUND, load_rjobj_from_file(path, type_, &obj));
- auto s =
-
rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive),
- rocksdb::Slice("invalid data"),
- path,
- /* should_sync */ true);
+ auto s = rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(type_),
+ rocksdb::Slice("invalid data"),
+ path,
+ /* should_sync */ true);
ASSERT_TRUE(s.ok()) << s.ToString();
- ASSERT_EQ(ERR_CORRUPTION, load_rjobj_from_file(path, &obj));
+ ASSERT_EQ(ERR_CORRUPTION, load_rjobj_from_file(path, type_, &obj));
}
} // namespace utils
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
