This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 40bc3b16ad8516fa83e13eb826415600daabca03
Author: Ferenc Gerlits <fgerl...@gmail.com>
AuthorDate: Thu Apr 11 18:40:57 2024 +0200
MINIFICPP-2282 Support re-encryption of sensitive properties
Added a new --re-encrypt option to the flow-config action of the
encrypt-config binary, which can decrypt the sensitive properties using
the .old key and re-encrypt them with a new key (either supported by
the user or auto-generated).
Closes #1739
Signed-off-by: Marton Szasz <sza...@apache.org>
---
encrypt-config/EncryptConfig.cpp | 26 ++--
encrypt-config/EncryptConfig.h | 5 +-
encrypt-config/EncryptConfigMain.cpp | 20 ++-
encrypt-config/FlowConfigEncryptor.cpp | 156 +++++++++++++--------
encrypt-config/FlowConfigEncryptor.h | 20 ++-
.../include/core/flow/AdaptiveConfiguration.h | 3 +-
libminifi/include/core/flow/FlowSerializer.h | 17 ++-
libminifi/include/core/json/JsonFlowSerializer.h | 4 +-
libminifi/include/core/yaml/YamlFlowSerializer.h | 4 +-
.../include/utils/crypto/EncryptionProvider.h | 2 +-
libminifi/include/utils/crypto/EncryptionUtils.h | 4 +
libminifi/include/utils/crypto/ciphers/XSalsa20.h | 6 +-
libminifi/src/core/flow/AdaptiveConfiguration.cpp | 7 +-
libminifi/src/core/flow/FlowSerializer.cpp | 52 +++++++
.../src/core/flow/StructuredConfiguration.cpp | 3 +
libminifi/src/core/json/JsonFlowSerializer.cpp | 25 ++--
libminifi/src/core/yaml/YamlFlowSerializer.cpp | 23 +--
libminifi/src/utils/crypto/EncryptionUtils.cpp | 14 +-
libminifi/test/unit/JsonFlowSerializerTests.cpp | 127 ++++++++++++++---
libminifi/test/unit/YamlFlowSerializerTests.cpp | 121 +++++++++++++---
20 files changed, 489 insertions(+), 150 deletions(-)
diff --git a/encrypt-config/EncryptConfig.cpp b/encrypt-config/EncryptConfig.cpp
index a1a3c89c9..6e0b07296 100644
--- a/encrypt-config/EncryptConfig.cpp
+++ b/encrypt-config/EncryptConfig.cpp
@@ -57,7 +57,7 @@ EncryptConfig::EncryptConfig(const std::string& minifi_home)
: minifi_home_(mini
std::filesystem::current_path(minifi_home_);
}
-bool EncryptConfig::isReencrypting() const {
+bool EncryptConfig::isReEncrypting() const {
encrypt_config::ConfigFile
bootstrap_file{std::ifstream{bootstrapFilePath()}};
std::string decryption_key_name =
utils::string::join_pack(ENCRYPTION_KEY_PROPERTY_NAME, ".old");
@@ -200,16 +200,20 @@ void
EncryptConfig::encryptSensitiveValuesInMinifiProperties() const {
}
void EncryptConfig::encryptSensitiveValuesInFlowConfig(
- const std::optional<std::string>& component_id, const
std::optional<std::string>& property_name, const std::optional<std::string>&
property_value) const {
- if (!component_id && !property_name && !property_value) {
- EncryptionKeys keys =
getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME);
- flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys,
minifi_home_, flowConfigPath());
- } else if (component_id && property_name && property_value) {
- EncryptionKeys keys =
getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME);
- flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys,
minifi_home_, flowConfigPath(), *component_id, *property_name, *property_value);
- } else {
- throw std::runtime_error("either all of --component-id, --property-name
and --property-value should be given (for batch mode) or none of them (for
interactive mode)");
- }
+ bool re_encrypt, const std::optional<std::string>& component_id, const
std::optional<std::string>& property_name, const std::optional<std::string>&
property_value) const {
+ EncryptionKeys keys =
getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME);
+ flow_config_encryptor::EncryptionRequest request_type = [&] {
+ if (re_encrypt) {
+ return
flow_config_encryptor::EncryptionRequest{flow_config_encryptor::EncryptionType::ReEncrypt};
+ } else if (!component_id && !property_name && !property_value) {
+ return
flow_config_encryptor::EncryptionRequest{flow_config_encryptor::EncryptionType::Interactive};
+ } else if (component_id && property_name && property_value) {
+ return flow_config_encryptor::EncryptionRequest{*component_id,
*property_name, *property_value};
+ } else {
+ throw std::runtime_error("either all of --component-id, --property-name
and --property-value should be given (for batch mode) or none of them (for
interactive mode)");
+ }
+ }();
+ flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys,
minifi_home_, flowConfigPath(), request_type);
}
} // namespace org::apache::nifi::minifi::encrypt_config
diff --git a/encrypt-config/EncryptConfig.h b/encrypt-config/EncryptConfig.h
index 1f48f48fb..5d32fa500 100644
--- a/encrypt-config/EncryptConfig.h
+++ b/encrypt-config/EncryptConfig.h
@@ -28,10 +28,11 @@ class EncryptConfig {
explicit EncryptConfig(const std::string& minifi_home);
void encryptSensitiveValuesInMinifiProperties() const;
- void encryptSensitiveValuesInFlowConfig(const std::optional<std::string>&
component_id, const std::optional<std::string>& property_name, const
std::optional<std::string>& property_value) const;
+ void encryptSensitiveValuesInFlowConfig(
+ bool re_encrypt, const std::optional<std::string>& component_id, const
std::optional<std::string>& property_name, const std::optional<std::string>&
property_value) const;
void encryptWholeFlowConfigFile() const;
- [[nodiscard]] bool isReencrypting() const;
+ [[nodiscard]] bool isReEncrypting() const;
private:
[[nodiscard]] std::filesystem::path bootstrapFilePath() const;
diff --git a/encrypt-config/EncryptConfigMain.cpp
b/encrypt-config/EncryptConfigMain.cpp
index 1faa91007..a18e8d075 100644
--- a/encrypt-config/EncryptConfigMain.cpp
+++ b/encrypt-config/EncryptConfigMain.cpp
@@ -48,6 +48,9 @@ int main(int argc, char* argv[]) try {
argument_parser.add_argument("--property-value")
.metavar("VALUE")
.help(minifi::utils::string::join_pack("The new value of the sensitive
property (", OPERATION_FLOW_CONFIG, " only)"));
+ argument_parser.add_argument("--re-encrypt")
+ .flag()
+ .help(minifi::utils::string::join_pack("Decrypt all properties with the
old key and re-encrypt them with a new key (", OPERATION_FLOW_CONFIG, "
only)"));
try {
argument_parser.parse_args(argc, argv);
@@ -57,15 +60,22 @@ int main(int argc, char* argv[]) try {
}
EncryptConfig encrypt_config{argument_parser.get("-m")};
+
std::string operation = argument_parser.get("operation");
+ const auto re_encrypt = argument_parser.get<bool>("--re-encrypt");
+ const auto component_id = argument_parser.present("--component-id");
+ const auto property_name = argument_parser.present("--property-name");
+ const auto property_value = argument_parser.present("--property-value");
+
+ if (operation != OPERATION_FLOW_CONFIG && (re_encrypt || component_id ||
property_name || property_value)) {
+ std::cerr << "Unsupported option for operation '" << operation << "'\n\n"
<< argument_parser;
+ return 5;
+ }
if (operation == OPERATION_MINIFI_PROPERTIES) {
encrypt_config.encryptSensitiveValuesInMinifiProperties();
} else if (operation == OPERATION_FLOW_CONFIG) {
- auto component_id = argument_parser.present("--component-id");
- auto property_name = argument_parser.present("--property-name");
- auto property_value = argument_parser.present("--property-value");
- encrypt_config.encryptSensitiveValuesInFlowConfig(component_id,
property_name, property_value);
+ encrypt_config.encryptSensitiveValuesInFlowConfig(re_encrypt,
component_id, property_name, property_value);
} else if (operation == OPERATION_WHOLE_FLOW_CONFIG_FILE) {
encrypt_config.encryptWholeFlowConfigFile();
} else {
@@ -73,7 +83,7 @@ int main(int argc, char* argv[]) try {
return 4;
}
- if ((operation == OPERATION_MINIFI_PROPERTIES || operation ==
OPERATION_WHOLE_FLOW_CONFIG_FILE) && encrypt_config.isReencrypting()) {
+ if ((operation == OPERATION_MINIFI_PROPERTIES || operation ==
OPERATION_WHOLE_FLOW_CONFIG_FILE) && encrypt_config.isReEncrypting()) {
std::cout << "WARNING: an .old key was provided, which is used for both "
<< OPERATION_MINIFI_PROPERTIES << " and " << OPERATION_WHOLE_FLOW_CONFIG_FILE
<< ".\n"
<< "If both are currently encrypted, make sure to run " << argv[0] <<
" to re-encrypt both before removing the .old key,\n"
<< "otherwise you won't be able to recover the encrypted data!\n";
diff --git a/encrypt-config/FlowConfigEncryptor.cpp
b/encrypt-config/FlowConfigEncryptor.cpp
index aa9983091..e9e788296 100644
--- a/encrypt-config/FlowConfigEncryptor.cpp
+++ b/encrypt-config/FlowConfigEncryptor.cpp
@@ -30,26 +30,27 @@
namespace minifi = org::apache::nifi::minifi;
namespace {
-enum class Type {
+enum class ComponentType {
Processor,
ControllerService
};
struct SensitiveProperty {
- Type type;
+ ComponentType component_type;
minifi::utils::Identifier component_id;
std::string component_name;
std::string property_name;
std::string property_display_name;
+ std::string property_value;
};
} // namespace
namespace magic_enum::customize {
template<>
-constexpr customize_t enum_name<Type>(Type type) noexcept {
+constexpr customize_t enum_name<ComponentType>(ComponentType type) noexcept {
switch (type) {
- case Type::Processor: return "Processor";
- case Type::ControllerService: return "Controller service";
+ case ComponentType::Processor: return "Processor";
+ case ComponentType::ControllerService: return "Controller service";
}
return invalid_tag;
}
@@ -61,16 +62,17 @@ std::vector<SensitiveProperty>
listSensitiveProperties(const minifi::core::Proce
std::vector<minifi::core::Processor *> processors;
process_group.getAllProcessors(processors);
- for (const auto *processor : processors) {
+ for (const auto* processor : processors) {
gsl_Expects(processor);
for (const auto& [_, property] : processor->getProperties()) {
if (property.isSensitive()) {
sensitive_properties.push_back(SensitiveProperty{
- .type = Type::Processor,
+ .component_type = ComponentType::Processor,
.component_id = processor->getUUID(),
.component_name = processor->getName(),
.property_name = property.getName(),
- .property_display_name = property.getDisplayName()});
+ .property_display_name = property.getDisplayName(),
+ .property_value = property.getValue().to_string()});
}
}
}
@@ -82,11 +84,12 @@ std::vector<SensitiveProperty>
listSensitiveProperties(const minifi::core::Proce
for (const auto& [_, property] : controller_service->getProperties()) {
if (property.isSensitive()) {
sensitive_properties.push_back(SensitiveProperty{
- .type = Type::ControllerService,
+ .component_type = ComponentType::ControllerService,
.component_id = controller_service->getUUID(),
.component_name = controller_service->getName(),
.property_name = property.getName(),
- .property_display_name = property.getDisplayName()});
+ .property_display_name = property.getDisplayName(),
+ .property_value = property.getValue().to_string()});
}
}
}
@@ -94,81 +97,116 @@ std::vector<SensitiveProperty>
listSensitiveProperties(const minifi::core::Proce
return sensitive_properties;
}
-template<typename Func>
-void encryptSensitiveValuesInFlowConfigImpl(
- const minifi::encrypt_config::EncryptionKeys& keys, const
std::filesystem::path& minifi_home, const std::filesystem::path&
flow_config_path, Func create_overrides) {
- const auto configure = std::make_shared<minifi::Configure>();
+std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
createOverridesInteractively(const std::vector<SensitiveProperty>&
sensitive_properties) {
+ std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
overrides;
+ std::cout << '\n';
+ for (const auto& sensitive_property : sensitive_properties) {
+ std::cout << magic_enum::enum_name(sensitive_property.component_type) << "
" << sensitive_property.component_name << " (" <<
sensitive_property.component_id.to_string() << ") "
+ << "has sensitive property " <<
sensitive_property.property_display_name << "\n enter a new value or press
Enter to keep the current value unchanged: ";
+ std::cout.flush();
+ std::string new_value;
+ std::getline(std::cin, new_value);
+ if (!new_value.empty()) {
+
overrides[sensitive_property.component_id].add(sensitive_property.property_name,
new_value);
+ }
+ }
+ return overrides;
+}
+
+std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
createOverridesForSingleProperty(
+ const std::vector<SensitiveProperty>& sensitive_properties, const
std::string& component_id, const std::string& property_name, const std::string&
property_value) {
+ const auto sensitive_property_it =
std::ranges::find_if(sensitive_properties, [&](const auto& sensitive_property) {
+ return sensitive_property.component_id.to_string().view() == component_id
&& (sensitive_property.property_name == property_name ||
sensitive_property.property_display_name == property_name);
+ });
+ if (sensitive_property_it == sensitive_properties.end()) {
+ std::cout << "No sensitive property found with this component ID and
property name.\n";
+ return {};
+ }
+ return {{sensitive_property_it->component_id,
minifi::core::flow::Overrides{}.add(sensitive_property_it->property_name,
property_value)}};
+}
+
+std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
createOverridesForReEncryption(const std::vector<SensitiveProperty>&
sensitive_properties) {
+ std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
overrides;
+ for (const auto& sensitive_property : sensitive_properties) {
+
overrides[sensitive_property.component_id].addOptional(sensitive_property.property_name,
sensitive_property.property_value);
+ }
+ return overrides;
+}
+
+} // namespace
+
+namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor {
+
+EncryptionRequest::EncryptionRequest(EncryptionType type) : type{type} {
+ gsl_Expects(type == EncryptionType::Interactive || type ==
EncryptionType::ReEncrypt);
+}
+
+EncryptionRequest::EncryptionRequest(std::string_view component_id,
std::string_view property_name, std::string_view property_value)
+ : type{EncryptionType::SingleProperty},
+ component_id{component_id},
+ property_name{property_name},
+ property_value{property_value} {}
+
+void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const
std::filesystem::path& minifi_home, const std::filesystem::path&
flow_config_path, const EncryptionRequest& request) {
+ const bool is_re_encrypting = keys.old_key.has_value();
+ if (is_re_encrypting && request.type != EncryptionType::ReEncrypt) {
+ throw std::runtime_error("Error: found .old key; please run --re-encrypt
and then remove the .old key");
+ }
+ if (!is_re_encrypting && request.type == EncryptionType::ReEncrypt) {
+ throw std::runtime_error("Error: cannot re-encrypt without an .old key!");
+ }
+
+ const auto configure = std::make_shared<Configure>();
configure->setHome(minifi_home);
configure->loadConfigureFile(DEFAULT_NIFI_PROPERTIES_FILE);
- bool encrypt_whole_flow_config_file =
(configure->get(minifi::Configure::nifi_flow_configuration_encrypt) |
minifi::utils::andThen(minifi::utils::string::toBool)).value_or(false);
- auto encryptor = encrypt_whole_flow_config_file ?
minifi::utils::crypto::EncryptionProvider::create(minifi_home) : std::nullopt;
- auto filesystem =
std::make_shared<minifi::utils::file::FileSystem>(encrypt_whole_flow_config_file,
encryptor);
+ bool encrypt_whole_flow_config_file =
(configure->get(Configure::nifi_flow_configuration_encrypt) |
utils::andThen(utils::string::toBool)).value_or(false);
+ auto whole_file_encryptor = encrypt_whole_flow_config_file ?
utils::crypto::EncryptionProvider::create(minifi_home) : std::nullopt;
+ auto filesystem =
std::make_shared<utils::file::FileSystem>(encrypt_whole_flow_config_file,
whole_file_encryptor);
+
+ auto sensitive_properties_decryptor = is_re_encrypting ?
+
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{*keys.old_key}}
:
+
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{keys.encryption_key}};
- minifi::core::extension::ExtensionManager::get().initialize(configure);
+ core::extension::ExtensionManager::get().initialize(configure);
- minifi::core::flow::AdaptiveConfiguration
adaptive_configuration{minifi::core::ConfigurationContext{
+ core::flow::AdaptiveConfiguration
adaptive_configuration{core::ConfigurationContext{
.flow_file_repo = nullptr,
.content_repo = nullptr,
.configuration = configure,
.path = flow_config_path,
.filesystem = filesystem,
- .sensitive_properties_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{keys.encryption_key}}
+ .sensitive_properties_encryptor = sensitive_properties_decryptor
}};
const auto flow_config_content = filesystem->read(flow_config_path);
if (!flow_config_content) {
- throw std::runtime_error(minifi::utils::string::join_pack("Could not read
the flow configuration file \"", flow_config_path.string(), "\""));
+ throw std::runtime_error(utils::string::join_pack("Could not read the flow
configuration file \"", flow_config_path.string(), "\""));
}
const auto process_group =
adaptive_configuration.getRootFromPayload(*flow_config_content);
gsl_Expects(process_group);
const auto sensitive_properties = listSensitiveProperties(*process_group);
- std::unordered_map<minifi::utils::Identifier,
std::unordered_map<std::string, std::string>> overrides =
create_overrides(sensitive_properties);
+ const auto overrides = [&]() -> std::unordered_map<utils::Identifier,
core::flow::Overrides> {
+ switch (request.type) {
+ case EncryptionType::Interactive: return
createOverridesInteractively(sensitive_properties);
+ case EncryptionType::SingleProperty: return
createOverridesForSingleProperty(sensitive_properties, request.component_id,
request.property_name, request.property_value);
+ case EncryptionType::ReEncrypt: return
createOverridesForReEncryption(sensitive_properties);
+ }
+ return {};
+ }();
if (overrides.empty()) {
+ std::cout << "Nothing to do, exiting.\n";
return;
}
+ if (is_re_encrypting) {
+
adaptive_configuration.setSensitivePropertiesEncryptor(utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{keys.encryption_key}});
+ }
+
std::string flow_config_str =
adaptive_configuration.serializeWithOverrides(*process_group, overrides);
adaptive_configuration.persist(flow_config_str);
}
-} // namespace
-
-namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor {
-
-void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const
std::filesystem::path& minifi_home, const std::filesystem::path&
flow_config_path) {
- encryptSensitiveValuesInFlowConfigImpl(keys, minifi_home, flow_config_path,
- [](const auto& sensitive_properties) {
- std::unordered_map<utils::Identifier, std::unordered_map<std::string,
std::string>> overrides;
- std::cout << '\n';
- for (const auto& sensitive_property : sensitive_properties) {
- std::cout << magic_enum::enum_name(sensitive_property.type) << " " <<
sensitive_property.component_name << " (" <<
sensitive_property.component_id.to_string() << ") "
- << "has sensitive property " <<
sensitive_property.property_display_name << "\n enter a new value or press
Enter to keep the current value unchanged: ";
- std::cout.flush();
- std::string new_value;
- std::getline(std::cin, new_value);
- if (!new_value.empty()) {
-
overrides[sensitive_property.component_id].emplace(sensitive_property.property_name,
new_value);
- }
- }
- return overrides;
- });
-}
-
-void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const
std::filesystem::path& minifi_home, const std::filesystem::path&
flow_config_path,
- const std::string& component_id, const std::string& property_name, const
std::string& property_value) {
- encryptSensitiveValuesInFlowConfigImpl(keys, minifi_home, flow_config_path,
- [&](const auto& sensitive_properties) ->
std::unordered_map<utils::Identifier, std::unordered_map<std::string,
std::string>> {
- const auto sensitive_property_it =
std::ranges::find_if(sensitive_properties, [&](const auto& sensitive_property) {
- return sensitive_property.component_id.to_string() == component_id &&
(sensitive_property.property_name == property_name ||
sensitive_property.property_display_name == property_name);
- });
- if (sensitive_property_it == sensitive_properties.end()) {
- std::cout << "No sensitive property found with this component ID and
property name.\n";
- return {};
- }
- return {{sensitive_property_it->component_id,
{{sensitive_property_it->property_name, property_value}}}};
- });
-}
} // namespace
org::apache::nifi::minifi::encrypt_config::flow_config_encryptor
diff --git a/encrypt-config/FlowConfigEncryptor.h
b/encrypt-config/FlowConfigEncryptor.h
index b4109581b..4c699c91b 100644
--- a/encrypt-config/FlowConfigEncryptor.h
+++ b/encrypt-config/FlowConfigEncryptor.h
@@ -24,8 +24,22 @@
namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor {
-void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const
std::filesystem::path& minifi_home, const std::filesystem::path&
flow_config_path);
-void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const
std::filesystem::path& minifi_home, const std::filesystem::path&
flow_config_path,
- const std::string& component_id, const std::string& property_name, const
std::string& property_value);
+enum class EncryptionType {
+ Interactive,
+ SingleProperty,
+ ReEncrypt
+};
+
+struct EncryptionRequest {
+ explicit EncryptionRequest(EncryptionType type);
+ EncryptionRequest(std::string_view component_id, std::string_view
property_name, std::string_view property_value);
+
+ EncryptionType type;
+ std::string component_id;
+ std::string property_name;
+ std::string property_value;
+};
+
+void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const
std::filesystem::path& minifi_home, const std::filesystem::path&
flow_config_path, const EncryptionRequest& request);
} // namespace
org::apache::nifi::minifi::encrypt_config::flow_config_encryptor
diff --git a/libminifi/include/core/flow/AdaptiveConfiguration.h
b/libminifi/include/core/flow/AdaptiveConfiguration.h
index bc1ae9fbe..fe041ff9c 100644
--- a/libminifi/include/core/flow/AdaptiveConfiguration.h
+++ b/libminifi/include/core/flow/AdaptiveConfiguration.h
@@ -35,7 +35,8 @@ class AdaptiveConfiguration : public StructuredConfiguration {
std::unique_ptr<core::ProcessGroup> getRootFromPayload(const std::string
&payload) override;
- std::string serializeWithOverrides(const core::ProcessGroup& process_group,
const std::unordered_map<utils::Identifier, std::unordered_map<std::string,
std::string>>& overrides) const;
+ void setSensitivePropertiesEncryptor(utils::crypto::EncryptionProvider
sensitive_properties_encryptor);
+ std::string serializeWithOverrides(const core::ProcessGroup& process_group,
const std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides)
const;
};
} // namespace org::apache::nifi::minifi::core::flow
diff --git a/libminifi/include/core/flow/FlowSerializer.h
b/libminifi/include/core/flow/FlowSerializer.h
index 02ec6c1cc..05f374fe8 100644
--- a/libminifi/include/core/flow/FlowSerializer.h
+++ b/libminifi/include/core/flow/FlowSerializer.h
@@ -26,6 +26,21 @@
namespace org::apache::nifi::minifi::core::flow {
+class Overrides {
+ public:
+ Overrides& add(std::string_view property_name, std::string_view
property_value);
+ Overrides& addOptional(std::string_view property_name, std::string_view
property_value);
+ [[nodiscard]] std::optional<std::string> get(std::string_view property_name)
const;
+ [[nodiscard]] std::vector<std::pair<std::string, std::string>> getRequired()
const;
+
+ private:
+ struct OverrideItem {
+ std::string value;
+ bool is_required;
+ };
+ std::unordered_map<std::string, OverrideItem> overrides_;
+};
+
class FlowSerializer {
public:
FlowSerializer() = default;
@@ -37,7 +52,7 @@ class FlowSerializer {
FlowSerializer& operator=(FlowSerializer&&) = delete;
[[nodiscard]] virtual std::string serialize(const core::ProcessGroup&
process_group, const FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider,
- const std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>& overrides) const = 0;
+ const std::unordered_map<utils::Identifier, Overrides>& overrides) const
= 0;
};
} // namespace org::apache::nifi::minifi::core::flow
diff --git a/libminifi/include/core/json/JsonFlowSerializer.h
b/libminifi/include/core/json/JsonFlowSerializer.h
index 1faa95c93..9f3f6377d 100644
--- a/libminifi/include/core/json/JsonFlowSerializer.h
+++ b/libminifi/include/core/json/JsonFlowSerializer.h
@@ -26,12 +26,12 @@ class JsonFlowSerializer : public
core::flow::FlowSerializer {
explicit JsonFlowSerializer(rapidjson::Document document) :
flow_definition_json_(std::move(document)) {}
[[nodiscard]] std::string serialize(const core::ProcessGroup& process_group,
const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider&
encryption_provider,
- const std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>& overrides) const override;
+ const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const override;
private:
void encryptSensitiveProperties(rapidjson::Value& property_jsons,
rapidjson::Document::AllocatorType& alloc,
const std::map<std::string, Property>& properties, const
utils::crypto::EncryptionProvider& encryption_provider,
- std::unordered_map<std::string, std::string> component_overrides) const;
+ const core::flow::Overrides& overrides) const;
rapidjson::Document flow_definition_json_;
std::shared_ptr<logging::Logger>
logger_{logging::LoggerFactory<JsonFlowSerializer>::getLogger()};
diff --git a/libminifi/include/core/yaml/YamlFlowSerializer.h
b/libminifi/include/core/yaml/YamlFlowSerializer.h
index 3483b1638..19a16e3d1 100644
--- a/libminifi/include/core/yaml/YamlFlowSerializer.h
+++ b/libminifi/include/core/yaml/YamlFlowSerializer.h
@@ -26,11 +26,11 @@ class YamlFlowSerializer : public
core::flow::FlowSerializer {
explicit YamlFlowSerializer(const YAML::Node& flow_definition_yaml) :
flow_definition_yaml_(flow_definition_yaml) {}
[[nodiscard]] std::string serialize(const core::ProcessGroup& process_group,
const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider&
encryption_provider,
- const std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>& overrides) const override;
+ const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const override;
private:
void encryptSensitiveProperties(YAML::Node property_yamls, const
std::map<std::string, Property>& properties, const
utils::crypto::EncryptionProvider& encryption_provider,
- std::unordered_map<std::string, std::string> component_overrides) const;
+ const core::flow::Overrides& overrides) const;
YAML::Node flow_definition_yaml_;
std::shared_ptr<logging::Logger>
logger_{logging::LoggerFactory<YamlFlowSerializer>::getLogger()};
diff --git a/libminifi/include/utils/crypto/EncryptionProvider.h
b/libminifi/include/utils/crypto/EncryptionProvider.h
index 15e14ccca..b72c38134 100644
--- a/libminifi/include/utils/crypto/EncryptionProvider.h
+++ b/libminifi/include/utils/crypto/EncryptionProvider.h
@@ -47,7 +47,7 @@ class EncryptionProvider {
}
private:
- const XSalsa20Cipher cipher_impl_;
+ XSalsa20Cipher cipher_impl_;
};
} // namespace org::apache::nifi::minifi::utils::crypto
diff --git a/libminifi/include/utils/crypto/EncryptionUtils.h
b/libminifi/include/utils/crypto/EncryptionUtils.h
index 2349ce2ee..66b95cd17 100644
--- a/libminifi/include/utils/crypto/EncryptionUtils.h
+++ b/libminifi/include/utils/crypto/EncryptionUtils.h
@@ -47,6 +47,10 @@ struct EncryptedData {
Bytes ciphertext_plus_mac;
};
+struct EncryptionError : public std::runtime_error {
+ using runtime_error::runtime_error;
+};
+
/**
* Encrypt the input (raw version).
*
diff --git a/libminifi/include/utils/crypto/ciphers/XSalsa20.h
b/libminifi/include/utils/crypto/ciphers/XSalsa20.h
index d5727671c..5ce17d14d 100644
--- a/libminifi/include/utils/crypto/ciphers/XSalsa20.h
+++ b/libminifi/include/utils/crypto/ciphers/XSalsa20.h
@@ -32,16 +32,16 @@ class XSalsa20Cipher {
return utils::crypto::generateKey();
}
- std::string encrypt(std::string_view data) const {
+ [[nodiscard]] std::string encrypt(std::string_view data) const {
return utils::crypto::encrypt(data, encryption_key_);
}
- std::string decrypt(std::string_view data) const {
+ [[nodiscard]] std::string decrypt(std::string_view data) const {
return utils::crypto::decrypt(data, encryption_key_);
}
private:
- const Bytes encryption_key_;
+ Bytes encryption_key_;
};
} // namespace org::apache::nifi::minifi::utils::crypto
diff --git a/libminifi/src/core/flow/AdaptiveConfiguration.cpp
b/libminifi/src/core/flow/AdaptiveConfiguration.cpp
index 6eac01fbb..85c590a38 100644
--- a/libminifi/src/core/flow/AdaptiveConfiguration.cpp
+++ b/libminifi/src/core/flow/AdaptiveConfiguration.cpp
@@ -71,8 +71,11 @@ std::unique_ptr<core::ProcessGroup>
AdaptiveConfiguration::getRootFromPayload(co
}
}
-std::string AdaptiveConfiguration::serializeWithOverrides(const
core::ProcessGroup& process_group,
- const std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>& overrides) const {
+void
AdaptiveConfiguration::setSensitivePropertiesEncryptor(utils::crypto::EncryptionProvider
sensitive_properties_encryptor) {
+ sensitive_properties_encryptor_ = std::move(sensitive_properties_encryptor);
+}
+
+std::string AdaptiveConfiguration::serializeWithOverrides(const
core::ProcessGroup& process_group, const std::unordered_map<utils::Identifier,
core::flow::Overrides>& overrides) const {
gsl_Expects(flow_serializer_);
return flow_serializer_->serialize(process_group, schema_,
sensitive_properties_encryptor_, overrides);
}
diff --git a/libminifi/src/core/flow/FlowSerializer.cpp
b/libminifi/src/core/flow/FlowSerializer.cpp
new file mode 100644
index 000000000..a55115ebc
--- /dev/null
+++ b/libminifi/src/core/flow/FlowSerializer.cpp
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "core/flow/FlowSerializer.h"
+#include "utils/OptionalUtils.h"
+
+namespace org::apache::nifi::minifi::core::flow {
+
+Overrides& Overrides::add(std::string_view property_name, std::string_view
property_value) {
+ overrides_.emplace(property_name, OverrideItem{.value =
std::string{property_value}, .is_required = true});
+ return *this;
+}
+
+Overrides& Overrides::addOptional(std::string_view property_name,
std::string_view property_value) {
+ overrides_.emplace(property_name, OverrideItem{.value =
std::string{property_value}, .is_required = false});
+ return *this;
+}
+
+std::optional<std::string> Overrides::get(std::string_view property_name)
const {
+ const auto it = overrides_.find(std::string{property_name});
+ if (it != overrides_.end()) {
+ return {it->second.value};
+ } else {
+ return std::nullopt;
+ }
+}
+
+std::vector<std::pair<std::string, std::string>> Overrides::getRequired()
const {
+ std::vector<std::pair<std::string, std::string>> required_overrides;
+ for (const auto& [name, override_item] : overrides_) {
+ if (override_item.is_required) {
+ required_overrides.emplace_back(name, override_item.value);
+ }
+ }
+ return required_overrides;
+}
+
+} // namespace org::apache::nifi::minifi::core::flow
diff --git a/libminifi/src/core/flow/StructuredConfiguration.cpp
b/libminifi/src/core/flow/StructuredConfiguration.cpp
index df79c9b6d..8bcabf368 100644
--- a/libminifi/src/core/flow/StructuredConfiguration.cpp
+++ b/libminifi/src/core/flow/StructuredConfiguration.cpp
@@ -673,6 +673,9 @@ PropertyValue
StructuredConfiguration::getValidatedProcessorPropertyForDefaultTy
coercedValue = property_value_node.getScalarAsString().value();
}
return coercedValue;
+ } catch (const utils::crypto::EncryptionError& e) {
+ logger_->log_error("Fetching property failed with a decryption error: {}",
e.what());
+ throw;
} catch (const std::exception& e) {
logger_->log_error("Fetching property failed with an exception of {}",
e.what());
logger_->log_error("Invalid conversion for field {}. Value {}",
property_from_processor.getName(), property_value_node.getDebugString());
diff --git a/libminifi/src/core/json/JsonFlowSerializer.cpp
b/libminifi/src/core/json/JsonFlowSerializer.cpp
index 82d6d961c..a62982a05 100644
--- a/libminifi/src/core/json/JsonFlowSerializer.cpp
+++ b/libminifi/src/core/json/JsonFlowSerializer.cpp
@@ -17,6 +17,8 @@
#include "core/json/JsonFlowSerializer.h"
+#include <unordered_set>
+
#include "rapidjson/prettywriter.h"
#include "utils/crypto/property_encryption/PropertyEncryptionUtils.h"
@@ -37,8 +39,11 @@ rapidjson::Value& getMember(rapidjson::Value& node, const
std::string& member_na
}
}
-void JsonFlowSerializer::encryptSensitiveProperties(rapidjson::Value
&property_jsons, rapidjson::Document::AllocatorType &alloc,
- const std::map<std::string, Property> &properties, const
utils::crypto::EncryptionProvider &encryption_provider,
std::unordered_map<std::string, std::string> component_overrides) const {
+void JsonFlowSerializer::encryptSensitiveProperties(rapidjson::Value&
property_jsons, rapidjson::Document::AllocatorType& alloc,
+ const std::map<std::string, Property>& properties, const
utils::crypto::EncryptionProvider& encryption_provider,
+ const core::flow::Overrides& overrides) const {
+ std::unordered_set<std::string> processed_property_names;
+
for (auto &property : property_jsons.GetObject()) {
const std::string name{property.name.GetString(),
property.name.GetStringLength()};
if (!properties.contains(name)) {
@@ -47,22 +52,24 @@ void
JsonFlowSerializer::encryptSensitiveProperties(rapidjson::Value &property_j
}
if (properties.at(name).isSensitive()) {
auto& value = property.value;
- const std::string_view value_sv = component_overrides.contains(name) ?
component_overrides.at(name) : std::string_view{value.GetString(),
value.GetStringLength()};
+ const auto override_value = overrides.get(name);
+ const std::string_view value_sv = override_value ? *override_value :
std::string_view{value.GetString(), value.GetStringLength()};
const std::string encrypted_value =
utils::crypto::property_encryption::encrypt(value_sv, encryption_provider);
value.SetString(encrypted_value.c_str(), encrypted_value.size(), alloc);
}
- component_overrides.erase(name);
+ processed_property_names.insert(name);
}
- for (const auto& [name, value] : component_overrides) {
+ for (const auto& [name, value] : overrides.getRequired()) {
gsl_Expects(properties.contains(name) &&
properties.at(name).isSensitive());
+ if (processed_property_names.contains(name)) { continue; }
const std::string encrypted_value =
utils::crypto::property_encryption::encrypt(value, encryption_provider);
property_jsons.AddMember(rapidjson::Value(name, alloc),
rapidjson::Value(encrypted_value, alloc), alloc);
}
}
-std::string JsonFlowSerializer::serialize(const core::ProcessGroup
&process_group, const core::flow::FlowSchema &schema, const
utils::crypto::EncryptionProvider &encryption_provider,
- const std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>& overrides) const {
+std::string JsonFlowSerializer::serialize(const core::ProcessGroup&
process_group, const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider,
+ const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const {
gsl_Expects(schema.root_group.size() == 1 && schema.identifier.size() == 1 &&
schema.processors.size() == 1 && schema.processor_properties.size() == 1
&&
schema.controller_services.size() == 1 &&
schema.controller_service_properties.size() == 1);
@@ -86,7 +93,7 @@ std::string JsonFlowSerializer::serialize(const
core::ProcessGroup &process_grou
logger_->log_warn("Processor {} not found in the flow definition",
processor_id->to_string());
continue;
}
- const auto& processor_overrides = overrides.contains(*processor_id) ?
overrides.at(*processor_id) : std::unordered_map<std::string, std::string>{};
+ const auto processor_overrides = overrides.contains(*processor_id) ?
overrides.at(*processor_id) : core::flow::Overrides{};
encryptSensitiveProperties(getMember(processor_json,
schema.processor_properties[0]), alloc, processor->getProperties(),
encryption_provider,
processor_overrides);
}
@@ -109,7 +116,7 @@ std::string JsonFlowSerializer::serialize(const
core::ProcessGroup &process_grou
logger_->log_warn("Controller service {} not found in the flow
definition", controller_service_id_str);
continue;
}
- const auto& controller_service_overrides =
overrides.contains(*controller_service_id) ?
overrides.at(*controller_service_id) : std::unordered_map<std::string,
std::string>{};
+ const auto controller_service_overrides =
overrides.contains(*controller_service_id) ?
overrides.at(*controller_service_id) : core::flow::Overrides{};
encryptSensitiveProperties(getMember(controller_service_json,
schema.controller_service_properties[0]), alloc,
controller_service->getProperties(), encryption_provider,
controller_service_overrides);
}
diff --git a/libminifi/src/core/yaml/YamlFlowSerializer.cpp
b/libminifi/src/core/yaml/YamlFlowSerializer.cpp
index 00bd87db3..3dc025a9e 100644
--- a/libminifi/src/core/yaml/YamlFlowSerializer.cpp
+++ b/libminifi/src/core/yaml/YamlFlowSerializer.cpp
@@ -17,12 +17,16 @@
#include "core/yaml/YamlFlowSerializer.h"
+#include <unordered_set>
+
#include "utils/crypto/property_encryption/PropertyEncryptionUtils.h"
namespace org::apache::nifi::minifi::core::yaml {
void YamlFlowSerializer::encryptSensitiveProperties(YAML::Node property_yamls,
const std::map<std::string, Property>& properties, const
utils::crypto::EncryptionProvider& encryption_provider,
- std::unordered_map<std::string, std::string> component_overrides) const {
+ const core::flow::Overrides& overrides) const {
+ std::unordered_set<std::string> processed_property_names;
+
for (auto kv : property_yamls) {
auto name = kv.first.as<std::string>();
if (!properties.contains(name)) {
@@ -32,25 +36,28 @@ void
YamlFlowSerializer::encryptSensitiveProperties(YAML::Node property_yamls, c
if (properties.at(name).isSensitive()) {
if (kv.second.IsSequence()) {
for (auto property_item : kv.second) {
- auto value = component_overrides.contains(name) ?
component_overrides.at(name) : property_item["value"].as<std::string>();
+ const auto override_value = overrides.get(name);
+ auto value = override_value ? *override_value :
property_item["value"].as<std::string>();
property_item["value"] =
utils::crypto::property_encryption::encrypt(value, encryption_provider);
}
} else {
- auto value = component_overrides.contains(name) ?
component_overrides.at(name) : kv.second.as<std::string>();
+ const auto override_value = overrides.get(name);
+ auto value = override_value ? *override_value :
kv.second.as<std::string>();
property_yamls[name] =
utils::crypto::property_encryption::encrypt(value, encryption_provider);
}
- component_overrides.erase(name);
+ processed_property_names.insert(name);
}
}
- for (const auto& [name, value] : component_overrides) {
+ for (const auto& [name, value] : overrides.getRequired()) {
gsl_Expects(properties.contains(name) &&
properties.at(name).isSensitive());
+ if (processed_property_names.contains(name)) { continue; }
property_yamls[name] = utils::crypto::property_encryption::encrypt(value,
encryption_provider);
}
}
std::string YamlFlowSerializer::serialize(const core::ProcessGroup&
process_group, const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider,
- const std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>& overrides) const {
+ const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const {
gsl_Expects(schema.identifier.size() == 1 &&
schema.processors.size() == 1 && schema.processor_properties.size() == 1
&&
schema.controller_services.size() == 1 &&
schema.controller_service_properties.size() == 1);
@@ -68,7 +75,7 @@ std::string YamlFlowSerializer::serialize(const
core::ProcessGroup& process_grou
logger_->log_warn("Processor {} not found in the flow definition",
processor_id->to_string());
continue;
}
- const auto& processor_overrides = overrides.contains(*processor_id) ?
overrides.at(*processor_id) : std::unordered_map<std::string, std::string>{};
+ const auto processor_overrides = overrides.contains(*processor_id) ?
overrides.at(*processor_id) : core::flow::Overrides{};
encryptSensitiveProperties(processor_yaml[schema.processor_properties[0]],
processor->getProperties(), encryption_provider, processor_overrides);
}
@@ -89,7 +96,7 @@ std::string YamlFlowSerializer::serialize(const
core::ProcessGroup& process_grou
logger_->log_warn("Controller service {} not found in the flow
definition", controller_service_id_str);
continue;
}
- const auto& controller_service_overrides =
overrides.contains(*controller_service_id) ?
overrides.at(*controller_service_id) : std::unordered_map<std::string,
std::string>{};
+ const auto controller_service_overrides =
overrides.contains(*controller_service_id) ?
overrides.at(*controller_service_id) : core::flow::Overrides{};
encryptSensitiveProperties(controller_service_yaml[schema.controller_service_properties[0]],
controller_service->getProperties(), encryption_provider,
controller_service_overrides);
}
diff --git a/libminifi/src/utils/crypto/EncryptionUtils.cpp
b/libminifi/src/utils/crypto/EncryptionUtils.cpp
index 4d4215318..6d1b8fd12 100644
--- a/libminifi/src/utils/crypto/EncryptionUtils.cpp
+++ b/libminifi/src/utils/crypto/EncryptionUtils.cpp
@@ -59,11 +59,11 @@ std::string EncryptionType::separator() { return "||"; }
Bytes encryptRaw(const Bytes& plaintext, const Bytes& key, const Bytes& nonce)
{
if (key.size() != EncryptionType::keyLength()) {
- throw std::invalid_argument{"Expected key of " +
std::to_string(EncryptionType::keyLength()) +
+ throw EncryptionError{"Expected key of " +
std::to_string(EncryptionType::keyLength()) +
" bytes, but got " + std::to_string(key.size()) + " bytes during
encryption"};
}
if (nonce.size() != EncryptionType::nonceLength()) {
- throw std::invalid_argument{"Expected nonce of " +
std::to_string(EncryptionType::nonceLength()) +
+ throw EncryptionError{"Expected nonce of " +
std::to_string(EncryptionType::nonceLength()) +
" bytes, but got " + std::to_string(nonce.size()) + " bytes during
encryption"};
}
@@ -84,22 +84,22 @@ std::string encrypt(std::string_view plaintext, const
Bytes& key) {
Bytes decryptRaw(const Bytes& input, const Bytes& key, const Bytes& nonce) {
if (key.size() != EncryptionType::keyLength()) {
- throw std::invalid_argument{"Expected key of " +
std::to_string(EncryptionType::keyLength()) +
+ throw EncryptionError{"Expected key of " +
std::to_string(EncryptionType::keyLength()) +
" bytes, but got " + std::to_string(key.size()) + " bytes during
decryption"};
}
if (nonce.size() != EncryptionType::nonceLength()) {
- throw std::invalid_argument{"Expected a nonce of " +
std::to_string(EncryptionType::nonceLength()) +
+ throw EncryptionError{"Expected a nonce of " +
std::to_string(EncryptionType::nonceLength()) +
" bytes, but got " + std::to_string(nonce.size()) + " bytes during
decryption"};
}
if (input.size() < EncryptionType::macLength()) {
- throw std::invalid_argument{"Input is too short: expected at least " +
std::to_string(EncryptionType::macLength()) +
+ throw EncryptionError{"Input is too short: expected at least " +
std::to_string(EncryptionType::macLength()) +
" bytes, but got " + std::to_string(input.size()) + " bytes during
decryption"};
}
Bytes plaintext(input.size() - EncryptionType::macLength());
if (crypto_secretbox_open_easy(reinterpret_cast<unsigned
char*>(plaintext.data()), reinterpret_cast<const unsigned char*>(input.data()),
input.size(),
reinterpret_cast<const unsigned char*>(nonce.data()),
reinterpret_cast<const unsigned char*>(key.data()))) {
- throw std::runtime_error{"Decryption failed; the input may be forged!"};
+ throw EncryptionError{"Decryption failed; the input may be forged!"};
}
return plaintext;
}
@@ -113,7 +113,7 @@ std::string decrypt(std::string_view input, const Bytes&
key) {
EncryptedData parseEncrypted(std::string_view input) {
std::vector<std::string> nonce_and_rest = utils::string::split(input,
EncryptionType::separator());
if (nonce_and_rest.size() != 2) {
- throw std::invalid_argument{"Incorrect input; expected '<nonce>" +
EncryptionType::separator() + "<ciphertext_plus_mac>'"};
+ throw EncryptionError{"Incorrect input; expected '<nonce>" +
EncryptionType::separator() + "<ciphertext_plus_mac>'"};
}
Bytes nonce = utils::string::from_base64(nonce_and_rest[0]);
diff --git a/libminifi/test/unit/JsonFlowSerializerTests.cpp
b/libminifi/test/unit/JsonFlowSerializerTests.cpp
index 49cd913aa..dd6f72aa4 100644
--- a/libminifi/test/unit/JsonFlowSerializerTests.cpp
+++ b/libminifi/test/unit/JsonFlowSerializerTests.cpp
@@ -527,6 +527,8 @@ constexpr std::string_view
config_json_with_nifi_schema_part_2 = R"(
const utils::crypto::Bytes secret_key =
utils::string::from_hex("75536923a75928a970077f9dae2c2b166a5413e020cb5190de4fb8edce1a38c7");
const utils::crypto::EncryptionProvider encryption_provider{secret_key};
+using OverridesMap = std::unordered_map<utils::Identifier,
core::flow::Overrides>;
+
TEST_CASE("JsonFlowSerializer can encrypt the sensitive properties") {
ConfigurationTestController test_controller;
core::flow::AdaptiveConfiguration
json_configuration{test_controller.getContext()};
@@ -543,24 +545,24 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive
properties") {
REQUIRE(res);
const auto flow_serializer = core::json::JsonFlowSerializer{std::move(doc)};
- using Overrides = std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>;
const auto processor_id =
utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value();
const auto controller_service_id =
utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value();
const auto [overrides, expected_results] = GENERATE_REF(
- std::make_tuple(Overrides{},
+ std::make_tuple(OverridesMap{},
std::array{"very_secure_password", "very_secure_passphrase"}),
- std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password",
"password123"}}}},
+ std::make_tuple(OverridesMap{{processor_id,
core::flow::Overrides{}.add("invokehttp-proxy-password", "password123")}},
std::array{"password123", "very_secure_passphrase"}),
- std::make_tuple(Overrides{{controller_service_id, {{"Passphrase", "speak
friend and enter"}}}},
+ std::make_tuple(OverridesMap{{controller_service_id,
core::flow::Overrides{}.add("Passphrase", "speak friend and enter")}},
std::array{"very_secure_password", "speak friend and enter"}),
- std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password",
"password123"}}}, {controller_service_id, {{"Passphrase", "speak friend and
enter"}}}},
+ std::make_tuple(OverridesMap{{processor_id,
core::flow::Overrides{}.add("invokehttp-proxy-password", "password123")},
+ {controller_service_id,
core::flow::Overrides{}.add("Passphrase", "speak friend and enter")}},
std::array{"password123", "speak friend and enter"}));
std::string config_json_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
{
- std::regex regex{R"_("invokehttp-proxy-password": "(enc\{.*\})",)_"};
+ std::regex regex{R"_("invokehttp-proxy-password": "(.*)",)_"};
std::smatch match_results;
CHECK(std::regex_search(config_json_encrypted, match_results, regex));
@@ -570,7 +572,7 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive
properties") {
}
{
- std::regex regex{R"_("Passphrase": "(enc\{.*\})",)_"};
+ std::regex regex{R"_("Passphrase": "(.*)",)_"};
std::smatch match_results;
CHECK(std::regex_search(config_json_encrypted, match_results, regex));
@@ -595,17 +597,108 @@ TEST_CASE("JsonFlowSerializer with an override can add a
new property to the flo
const auto flow_serializer = core::json::JsonFlowSerializer{std::move(doc)};
const auto second_controller_service_id =
utils::Identifier::parse("b418f4ff-e598-4ea2-921f-14f9dd864482").value();
- const auto overrides = std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>{{second_controller_service_id,
{{"Passphrase", "new passphrase"}}}};
- std::string config_json_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
+ SECTION("with required overrides") {
+ const OverridesMap overrides{{second_controller_service_id,
core::flow::Overrides{}.add("Passphrase", "new passphrase")}};
+
+ std::string config_json_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
+
+ std::regex regex{R"_("Passphrase": "(.*)")_"};
+ std::smatch match_results;
+
+ // skip the first match
+ REQUIRE(std::regex_search(config_json_encrypted.cbegin(),
config_json_encrypted.cend(), match_results, regex));
+
+ // verify the second match
+ REQUIRE(std::regex_search(match_results.suffix().first,
config_json_encrypted.cend(), match_results, regex));
+ REQUIRE(match_results.size() == 2);
+ std::string encrypted_value = match_results[1];
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == "new passphrase");
+ }
+
+ SECTION("with optional overrides: the override is only used if the property
is already in the flow config") {
+ const auto first_controller_service_id =
utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value();
+ const OverridesMap overrides{{first_controller_service_id,
core::flow::Overrides{}.addOptional("Passphrase", "first new passphrase")},
+ {second_controller_service_id,
core::flow::Overrides{}.addOptional("Passphrase", "second new passphrase")}};
+
+ std::string config_json_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
+
+ std::regex regex{R"_("Passphrase": "(.*)")_"};
+ std::smatch match_results;
+
+ // verify the first match
+ REQUIRE(std::regex_search(config_json_encrypted.cbegin(),
config_json_encrypted.cend(), match_results, regex));
+ REQUIRE(match_results.size() == 2);
+ std::string encrypted_value = match_results[1];
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == "first new passphrase");
+
+ // check that there is no second match
+ CHECK_FALSE(std::regex_search(match_results.suffix().first,
config_json_encrypted.cend(), match_results, regex));
+ }
+}
+
+TEST_CASE("The encrypted flow configuration can be decrypted with the correct
key") {
+ ConfigurationTestController test_controller;
+ auto configuration_context = test_controller.getContext();
+ configuration_context.sensitive_properties_encryptor = encryption_provider;
+ core::flow::AdaptiveConfiguration
json_configuration_before{configuration_context};
+
+ const auto schema = core::flow::FlowSchema::getNiFiFlowJson();
+ const auto config_json_with_nifi_schema =
utils::string::join_pack(config_json_with_nifi_schema_part_1,
config_json_with_nifi_schema_part_2);
+ const auto process_group_before =
json_configuration_before.getRootFromPayload(config_json_with_nifi_schema);
+ REQUIRE(process_group_before);
+
+ rapidjson::Document doc;
+ rapidjson::ParseResult res = doc.Parse(config_json_with_nifi_schema.data(),
config_json_with_nifi_schema.size());
+ REQUIRE(res);
+ const auto flow_serializer = core::json::JsonFlowSerializer{std::move(doc)};
+ std::string config_json_encrypted =
flow_serializer.serialize(*process_group_before, schema, encryption_provider,
{});
+
+ core::flow::AdaptiveConfiguration
json_configuration_after{configuration_context};
+ const auto process_group_after =
json_configuration_after.getRootFromPayload(config_json_encrypted);
+ REQUIRE(process_group_after);
+
+ const auto processor_id =
utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value();
+ const auto* processor_before =
process_group_before->findProcessorById(processor_id);
+ REQUIRE(processor_before);
+ const auto* processor_after =
process_group_after->findProcessorById(processor_id);
+ REQUIRE(processor_after);
+ CHECK(processor_before->getProperties().at("HTTP Method").getValue() ==
processor_after->getProperties().at("HTTP Method").getValue());
+
CHECK(processor_before->getProperties().at("invokehttp-proxy-password").getValue()
==
processor_after->getProperties().at("invokehttp-proxy-password").getValue());
+
+ const auto controller_service_id = "b9801278-7b5d-4314-aed6-713fd4b5f933";
+ const auto controller_service_node_before =
process_group_before->findControllerService(controller_service_id);
+ REQUIRE(controller_service_node_before);
+ const auto controller_service_before =
controller_service_node_before->getControllerServiceImplementation();
+ REQUIRE(controller_service_node_before);
+ const auto controller_service_node_after =
process_group_after->findControllerService(controller_service_id);
+ REQUIRE(controller_service_node_after);
+ const auto controller_service_after =
controller_service_node_before->getControllerServiceImplementation();
+ REQUIRE(controller_service_after);
+ CHECK(controller_service_before->getProperties().at("CA
Certificate").getValue() == controller_service_after->getProperties().at("CA
Certificate").getValue());
+ CHECK(controller_service_before->getProperties().at("Passphrase").getValue()
== controller_service_after->getProperties().at("Passphrase").getValue());
+}
+
+TEST_CASE("The encrypted flow configuration cannot be decrypted with an
incorrect key") {
+ ConfigurationTestController test_controller;
+ auto configuration_context = test_controller.getContext();
+ configuration_context.sensitive_properties_encryptor = encryption_provider;
+ core::flow::AdaptiveConfiguration
json_configuration_before{configuration_context};
+
+ const auto schema = core::flow::FlowSchema::getNiFiFlowJson();
+ const auto config_json_with_nifi_schema =
utils::string::join_pack(config_json_with_nifi_schema_part_1,
config_json_with_nifi_schema_part_2);
+ const auto process_group_before =
json_configuration_before.getRootFromPayload(config_json_with_nifi_schema);
+ REQUIRE(process_group_before);
+
+ rapidjson::Document doc;
+ rapidjson::ParseResult res = doc.Parse(config_json_with_nifi_schema.data(),
config_json_with_nifi_schema.size());
+ REQUIRE(res);
+ const auto flow_serializer = core::json::JsonFlowSerializer{std::move(doc)};
+ std::string config_json_encrypted =
flow_serializer.serialize(*process_group_before, schema, encryption_provider,
{});
- // find the second match
- std::regex regex{R"_("Passphrase": "(enc\{.*\})")_"};
- std::smatch match_results;
- REQUIRE(std::regex_search(config_json_encrypted.cbegin(),
config_json_encrypted.cend(), match_results, regex));
- REQUIRE(std::regex_search(match_results.suffix().first,
config_json_encrypted.cend(), match_results, regex));
- REQUIRE(match_results.size() == 2);
+ const utils::crypto::Bytes different_secret_key =
utils::string::from_hex("ea55b7d0edc22280c9547e4d89712b3fae74f96d82f240a004fb9fbd0640eec7");
+ configuration_context.sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{different_secret_key};
- std::string encrypted_value = match_results[1];
- CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == "new passphrase");
+ core::flow::AdaptiveConfiguration
json_configuration_after{configuration_context};
+
REQUIRE_THROWS_AS(json_configuration_after.getRootFromPayload(config_json_encrypted),
utils::crypto::EncryptionError);
}
diff --git a/libminifi/test/unit/YamlFlowSerializerTests.cpp
b/libminifi/test/unit/YamlFlowSerializerTests.cpp
index 25dff1514..14d806247 100644
--- a/libminifi/test/unit/YamlFlowSerializerTests.cpp
+++ b/libminifi/test/unit/YamlFlowSerializerTests.cpp
@@ -173,6 +173,8 @@ NiFi Properties Overrides: {}
const utils::crypto::Bytes secret_key =
utils::string::from_hex("cb76fe6fe4cbfdc3770c0cb0afc910f81ced4d436b11f691395fc2a9dbea27ca");
const utils::crypto::EncryptionProvider encryption_provider{secret_key};
+using OverridesMap = std::unordered_map<utils::Identifier,
core::flow::Overrides>;
+
TEST_CASE("YamlFlowSerializer can encrypt the sensitive properties") {
ConfigurationTestController test_controller;
core::YamlConfiguration yaml_configuration{test_controller.getContext()};
@@ -184,24 +186,24 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive
properties") {
YAML::Node root_yaml_node = YAML::Load(std::string{config_yaml});
const auto flow_serializer = core::yaml::YamlFlowSerializer{root_yaml_node};
- using Overrides = std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>;
const auto processor_id =
utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value();
const auto controller_service_id =
utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value();
const auto [overrides, expected_results] = GENERATE_REF(
- std::make_tuple(Overrides{},
+ std::make_tuple(OverridesMap{},
std::array{"very_secure_password", "very_secure_passphrase"}),
- std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password",
"password123"}}}},
+ std::make_tuple(OverridesMap{{processor_id,
core::flow::Overrides{}.add("invokehttp-proxy-password", "password123")}},
std::array{"password123", "very_secure_passphrase"}),
- std::make_tuple(Overrides{{controller_service_id, {{"Passphrase", "speak
friend and enter"}}}},
+ std::make_tuple(OverridesMap{{controller_service_id,
core::flow::Overrides{}.add("Passphrase", "speak friend and enter")}},
std::array{"very_secure_password", "speak friend and enter"}),
- std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password",
"password123"}}}, {controller_service_id, {{"Passphrase", "speak friend and
enter"}}}},
+ std::make_tuple(OverridesMap{{processor_id,
core::flow::Overrides{}.add("invokehttp-proxy-password", "password123")},
+ {controller_service_id,
core::flow::Overrides{}.add("Passphrase", "speak friend and enter")}},
std::array{"password123", "speak friend and enter"}));
std::string config_yaml_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
{
- std::regex regex{R"_(invokehttp-proxy-password: (enc\{.*\}))_"};
+ std::regex regex{R"_(invokehttp-proxy-password: (.*))_"};
std::smatch match_results;
CHECK(std::regex_search(config_yaml_encrypted, match_results, regex));
@@ -211,7 +213,7 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive
properties") {
}
{
- std::regex regex{R"_(Passphrase: (enc\{.*\}))_"};
+ std::regex regex{R"_(Passphrase: (.*))_"};
std::smatch match_results;
CHECK(std::regex_search(config_yaml_encrypted, match_results, regex));
@@ -233,17 +235,102 @@ TEST_CASE("YamlFlowSerializer with an override can add a
new property to the flo
const auto flow_serializer = core::yaml::YamlFlowSerializer{root_yaml_node};
const auto second_controller_service_id =
utils::Identifier::parse("b418f4ff-e598-4ea2-921f-14f9dd864482").value();
- const auto overrides = std::unordered_map<utils::Identifier,
std::unordered_map<std::string, std::string>>{{second_controller_service_id,
{{"Passphrase", "new passphrase"}}}};
- std::string config_yaml_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
+ SECTION("with required overrides") {
+ const OverridesMap overrides{{second_controller_service_id,
core::flow::Overrides{}.add("Passphrase", "new passphrase")}};
+
+ std::string config_yaml_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
+
+ std::regex regex{R"_(Passphrase: (.*))_"};
+ std::smatch match_results;
+
+ // skip the first match
+ REQUIRE(std::regex_search(config_yaml_encrypted.cbegin(),
config_yaml_encrypted.cend(), match_results, regex));
+
+ // verify the second match
+ REQUIRE(std::regex_search(match_results.suffix().first,
config_yaml_encrypted.cend(), match_results, regex));
+ REQUIRE(match_results.size() == 2);
+ std::string encrypted_value = match_results[1];
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == "new passphrase");
+ }
+
+ SECTION("with optional overrides: the override is only used if the property
is already in the flow config") {
+ const auto first_controller_service_id =
utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value();
+ const OverridesMap overrides{{first_controller_service_id,
core::flow::Overrides{}.addOptional("Passphrase", "first new passphrase")},
+ {second_controller_service_id,
core::flow::Overrides{}.addOptional("Passphrase", "second new passphrase")}};
+
+ std::string config_yaml_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
+
+ std::regex regex{R"_(Passphrase: (.*))_"};
+ std::smatch match_results;
+
+ // verify the first match
+ REQUIRE(std::regex_search(config_yaml_encrypted.cbegin(),
config_yaml_encrypted.cend(), match_results, regex));
+ REQUIRE(match_results.size() == 2);
+ std::string encrypted_value = match_results[1];
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == "first new passphrase");
+
+ // check that there is no second match
+ CHECK_FALSE(std::regex_search(match_results.suffix().first,
config_yaml_encrypted.cend(), match_results, regex));
+ }
+}
+
+TEST_CASE("The encrypted flow configuration can be decrypted with the correct
key") {
+ ConfigurationTestController test_controller;
+ auto configuration_context = test_controller.getContext();
+ configuration_context.sensitive_properties_encryptor = encryption_provider;
+
+ core::flow::AdaptiveConfiguration
yaml_configuration_before{configuration_context};
+ const auto process_group_before =
yaml_configuration_before.getRootFromPayload(std::string{config_yaml});
+ REQUIRE(process_group_before);
+
+ const auto schema = core::flow::FlowSchema::getDefault();
+ YAML::Node root_yaml_node = YAML::Load(std::string{config_yaml});
+ const auto flow_serializer_before =
core::yaml::YamlFlowSerializer{root_yaml_node};
+ std::string config_yaml_encrypted =
flow_serializer_before.serialize(*process_group_before, schema,
encryption_provider, {});
+
+ core::flow::AdaptiveConfiguration
yaml_configuration_after{configuration_context};
+ const auto process_group_after =
yaml_configuration_after.getRootFromPayload(config_yaml_encrypted);
+ REQUIRE(process_group_after);
+
+ const auto processor_id =
utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value();
+ const auto* processor_before =
process_group_before->findProcessorById(processor_id);
+ REQUIRE(processor_before);
+ const auto* processor_after =
process_group_after->findProcessorById(processor_id);
+ REQUIRE(processor_after);
+ CHECK(processor_before->getProperties().at("HTTP Method").getValue() ==
processor_after->getProperties().at("HTTP Method").getValue());
+
CHECK(processor_before->getProperties().at("invokehttp-proxy-password").getValue()
==
processor_after->getProperties().at("invokehttp-proxy-password").getValue());
+
+ const auto controller_service_id = "b9801278-7b5d-4314-aed6-713fd4b5f933";
+ const auto controller_service_node_before =
process_group_before->findControllerService(controller_service_id);
+ REQUIRE(controller_service_node_before);
+ const auto controller_service_before =
controller_service_node_before->getControllerServiceImplementation();
+ REQUIRE(controller_service_node_before);
+ const auto controller_service_node_after =
process_group_after->findControllerService(controller_service_id);
+ REQUIRE(controller_service_node_after);
+ const auto controller_service_after =
controller_service_node_before->getControllerServiceImplementation();
+ REQUIRE(controller_service_after);
+ CHECK(controller_service_before->getProperties().at("CA
Certificate").getValue() == controller_service_after->getProperties().at("CA
Certificate").getValue());
+ CHECK(controller_service_before->getProperties().at("Passphrase").getValue()
== controller_service_after->getProperties().at("Passphrase").getValue());
+}
+
+TEST_CASE("The encrypted flow configuration cannot be decrypted with an
incorrect key") {
+ ConfigurationTestController test_controller;
+ auto configuration_context = test_controller.getContext();
+ configuration_context.sensitive_properties_encryptor = encryption_provider;
+
+ core::flow::AdaptiveConfiguration
yaml_configuration_before{configuration_context};
+ const auto process_group_before =
yaml_configuration_before.getRootFromPayload(std::string{config_yaml});
+ REQUIRE(process_group_before);
+
+ const auto schema = core::flow::FlowSchema::getDefault();
+ YAML::Node root_yaml_node = YAML::Load(std::string{config_yaml});
+ const auto flow_serializer = core::yaml::YamlFlowSerializer{root_yaml_node};
+ std::string config_yaml_encrypted =
flow_serializer.serialize(*process_group_before, schema, encryption_provider,
{});
- // find the second match
- std::regex regex{R"_(Passphrase: (enc\{.*\}))_"};
- std::smatch match_results;
- REQUIRE(std::regex_search(config_yaml_encrypted.cbegin(),
config_yaml_encrypted.cend(), match_results, regex));
- REQUIRE(std::regex_search(match_results.suffix().first,
config_yaml_encrypted.cend(), match_results, regex));
- REQUIRE(match_results.size() == 2);
+ const utils::crypto::Bytes different_secret_key =
utils::string::from_hex("ea55b7d0edc22280c9547e4d89712b3fae74f96d82f240a004fb9fbd0640eec7");
+ configuration_context.sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{different_secret_key};
- std::string encrypted_value = match_results[1];
- CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == "new passphrase");
+ core::flow::AdaptiveConfiguration
yaml_configuration_after{configuration_context};
+
REQUIRE_THROWS_AS(yaml_configuration_after.getRootFromPayload(config_yaml_encrypted),
utils::crypto::EncryptionError);
}
