This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 530e4dc7b7ea70e319493399ce03b0ccba255840 Author: Ferenc Gerlits <fgerl...@gmail.com> AuthorDate: Thu Mar 14 14:06:09 2024 +0100 MINIFICPP-2283 Create tool to encrypt sensitive properties in config.yml Closes 1725 Signed-off-by: Marton Szasz <sza...@apache.org> --- cmake/ArgParse.cmake | 4 +- encrypt-config/EncryptConfig.cpp | 116 +++++++++----- encrypt-config/EncryptConfig.h | 38 ++--- encrypt-config/EncryptConfigMain.cpp | 63 +++++--- encrypt-config/FlowConfigEncryptor.cpp | 174 +++++++++++++++++++++ .../FlowConfigEncryptor.h | 24 +-- encrypt-config/tests/ConfigFileEncryptorTests.cpp | 2 +- libminifi/include/core/FlowConfiguration.h | 2 +- libminifi/include/core/ProcessGroup.h | 2 + .../core/controller/ControllerServiceNode.h | 1 + .../include/core/flow/AdaptiveConfiguration.h | 2 + libminifi/include/core/flow/FlowSerializer.h | 5 +- libminifi/include/core/json/JsonFlowSerializer.h | 6 +- libminifi/include/core/yaml/YamlFlowSerializer.h | 6 +- libminifi/src/core/ProcessGroup.cpp | 8 + .../src/core/controller/ControllerServiceNode.cpp | 18 +-- libminifi/src/core/flow/AdaptiveConfiguration.cpp | 6 + .../src/core/flow/StructuredConfiguration.cpp | 2 +- libminifi/src/core/json/JsonFlowSerializer.cpp | 35 +++-- libminifi/src/core/yaml/YamlFlowSerializer.cpp | 35 +++-- libminifi/test/unit/JsonFlowSerializerTests.cpp | 131 ++++++++++++++-- libminifi/test/unit/YamlFlowSerializerTests.cpp | 56 ++++++- 22 files changed, 576 insertions(+), 160 deletions(-) diff --git a/cmake/ArgParse.cmake b/cmake/ArgParse.cmake index 883f34986..29251ceb9 100644 --- a/cmake/ArgParse.cmake +++ b/cmake/ArgParse.cmake @@ -18,7 +18,7 @@ include(FetchContent) FetchContent_Declare( argparse - URL https://github.com/p-ranav/argparse/archive/refs/tags/v2.9.tar.gz - URL_HASH SHA256=cd563293580b9dc592254df35b49cf8a19b4870ff5f611c7584cf967d9e6031e + URL https://github.com/p-ranav/argparse/archive/refs/tags/v3.0.tar.gz + URL_HASH SHA256=ba7b465759bb01069d57302855eaf4d1f7d677f21ad7b0b00b92939645c30f47 ) FetchContent_MakeAvailable(argparse) diff --git a/encrypt-config/EncryptConfig.cpp b/encrypt-config/EncryptConfig.cpp index d2876b63a..a1a3c89c9 100644 --- a/encrypt-config/EncryptConfig.cpp +++ b/encrypt-config/EncryptConfig.cpp @@ -25,65 +25,78 @@ #include "ConfigFile.h" #include "ConfigFileEncryptor.h" +#include "utils/Enum.h" #include "utils/file/FileUtils.h" #include "Defaults.h" +#include "core/extension/ExtensionManager.h" +#include "FlowConfigEncryptor.h" namespace { +constexpr std::string_view ENCRYPTION_KEY_PROPERTY_NAME = "nifi.bootstrap.sensitive.key"; +constexpr std::string_view SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME = "nifi.bootstrap.sensitive.properties.key"; -constexpr const char* OLD_KEY_PROPERTY_NAME = "nifi.bootstrap.sensitive.key.old"; -constexpr const char* ENCRYPTION_KEY_PROPERTY_NAME = "nifi.bootstrap.sensitive.key"; - +std::string readFile(const std::filesystem::path& file_path) { + try { + std::ifstream file_stream{file_path, std::ios::binary}; + file_stream.exceptions(std::ios::failbit | std::ios::badbit); + return {std::istreambuf_iterator<char>(file_stream), {}}; + } catch (...) { + throw std::runtime_error("Error while reading file \"" + file_path.string() + "\""); + } +} } // namespace namespace org::apache::nifi::minifi::encrypt_config { EncryptConfig::EncryptConfig(const std::string& minifi_home) : minifi_home_(minifi_home) { if (sodium_init() < 0) { + // encryption/decryption depends on the libsodium library which needs to be initialized throw std::runtime_error{"Could not initialize the libsodium library!"}; } - // encryption/decryption depends on the libsodium library which needs to be initialized - keys_ = getEncryptionKeys(); + + std::filesystem::current_path(minifi_home_); } -EncryptConfig::EncryptionType EncryptConfig::encryptSensitiveProperties() const { - encryptSensitiveProperties(keys_); - if (keys_.old_key) { - return EncryptionType::RE_ENCRYPT; - } - return EncryptionType::ENCRYPT; +bool EncryptConfig::isReencrypting() const { + encrypt_config::ConfigFile bootstrap_file{std::ifstream{bootstrapFilePath()}}; + + std::string decryption_key_name = utils::string::join_pack(ENCRYPTION_KEY_PROPERTY_NAME, ".old"); + std::optional<std::string> decryption_key_hex = bootstrap_file.getValue(decryption_key_name); + + return (decryption_key_hex && !decryption_key_hex->empty()); } -void EncryptConfig::encryptFlowConfig() const { +std::filesystem::path EncryptConfig::flowConfigPath() const { encrypt_config::ConfigFile properties_file{std::ifstream{propertiesFilePath()}}; std::optional<std::filesystem::path> config_path{properties_file.getValue(Configure::nifi_flow_configuration_file)}; if (!config_path) { config_path = utils::file::PathUtils::resolve(minifi_home_, "conf/config.yml"); - std::cout << "Couldn't find path of configuration file, using default: \"" << *config_path << "\"\n"; + std::cout << "Couldn't find path of configuration file, using default: " << *config_path << '\n'; } else { config_path = utils::file::PathUtils::resolve(minifi_home_, *config_path); - std::cout << "Encrypting flow configuration file: \"" << *config_path << "\"\n"; - } - std::string config_content; - try { - std::ifstream config_file{*config_path, std::ios::binary}; - config_file.exceptions(std::ios::failbit | std::ios::badbit); - config_content = std::string{std::istreambuf_iterator<char>(config_file), {}}; - } catch (...) { - throw std::runtime_error("Error while reading flow configuration file \"" + config_path->string() + "\""); + std::cout << "Encrypting flow configuration file: " << *config_path << '\n'; } + return *config_path; +} + +void EncryptConfig::encryptWholeFlowConfigFile() const { + EncryptionKeys keys = getEncryptionKeys(ENCRYPTION_KEY_PROPERTY_NAME); + + std::filesystem::path config_path = flowConfigPath(); + std::string config_content = readFile(config_path); try { - utils::crypto::decrypt(config_content, keys_.encryption_key); + utils::crypto::decrypt(config_content, keys.encryption_key); std::cout << "Flow config file is already properly encrypted.\n"; return; } catch (const std::exception&) {} if (utils::crypto::isEncrypted(config_content)) { - if (!keys_.old_key) { + if (!keys.old_key) { throw std::runtime_error("Config file is encrypted, but no old encryption key is set."); } std::cout << "Trying to decrypt flow config file using the old key ...\n"; try { - config_content = utils::crypto::decrypt(config_content, *keys_.old_key); + config_content = utils::crypto::decrypt(config_content, *keys.old_key); } catch (const std::exception&) { throw std::runtime_error("Flow config is encrypted, but couldn't be decrypted."); } @@ -91,15 +104,15 @@ void EncryptConfig::encryptFlowConfig() const { std::cout << "Flow config file is not encrypted, using as-is.\n"; } - std::string encrypted_content = utils::crypto::encrypt(config_content, keys_.encryption_key); + std::string encrypted_content = utils::crypto::encrypt(config_content, keys.encryption_key); try { - std::ofstream encrypted_file{*config_path, std::ios::binary}; + std::ofstream encrypted_file{config_path, std::ios::binary}; encrypted_file.exceptions(std::ios::failbit | std::ios::badbit); encrypted_file << encrypted_content; } catch (...) { - throw std::runtime_error("Error while writing encrypted flow configuration file \"" + config_path->string() + "\""); + throw std::runtime_error("Error while writing encrypted flow configuration file \"" + config_path.string() + "\""); } - std::cout << "Successfully encrypted flow configuration file: \"" << *config_path << "\"\n"; + std::cout << "Successfully encrypted flow configuration file: " << config_path << '\n'; } std::filesystem::path EncryptConfig::bootstrapFilePath() const { @@ -110,27 +123,31 @@ std::filesystem::path EncryptConfig::propertiesFilePath() const { return minifi_home_ / DEFAULT_NIFI_PROPERTIES_FILE; } -EncryptionKeys EncryptConfig::getEncryptionKeys() const { +EncryptionKeys EncryptConfig::getEncryptionKeys(std::string_view property_name) const { encrypt_config::ConfigFile bootstrap_file{std::ifstream{bootstrapFilePath()}}; - std::optional<std::string> decryption_key_hex = bootstrap_file.getValue(OLD_KEY_PROPERTY_NAME); - std::optional<std::string> encryption_key_hex = bootstrap_file.getValue(ENCRYPTION_KEY_PROPERTY_NAME); + + std::string decryption_key_name = utils::string::join_pack(property_name, ".old"); + std::optional<std::string> decryption_key_hex = bootstrap_file.getValue(decryption_key_name); + + std::string encryption_key_name{property_name}; + std::optional<std::string> encryption_key_hex = bootstrap_file.getValue(encryption_key_name); EncryptionKeys keys; if (decryption_key_hex && !decryption_key_hex->empty()) { - std::string binary_key = hexDecodeAndValidateKey(*decryption_key_hex, OLD_KEY_PROPERTY_NAME); + std::string binary_key = hexDecodeAndValidateKey(*decryption_key_hex, decryption_key_name); std::cout << "Old encryption key found in " << bootstrapFilePath() << "\n"; keys.old_key = utils::crypto::stringToBytes(binary_key); } if (encryption_key_hex && !encryption_key_hex->empty()) { - std::string binary_key = hexDecodeAndValidateKey(*encryption_key_hex, ENCRYPTION_KEY_PROPERTY_NAME); - std::cout << "Using the existing encryption key found in " << bootstrapFilePath() << '\n'; + std::string binary_key = hexDecodeAndValidateKey(*encryption_key_hex, encryption_key_name); + std::cout << "Using the existing encryption key " << property_name << " found in " << bootstrapFilePath() << '\n'; keys.encryption_key = utils::crypto::stringToBytes(binary_key); } else { std::cout << "Generating a new encryption key...\n"; utils::crypto::Bytes encryption_key = utils::crypto::generateKey(); - writeEncryptionKeyToBootstrapFile(encryption_key); - std::cout << "Wrote the new encryption key to " << bootstrapFilePath() << '\n'; + writeEncryptionKeyToBootstrapFile(encryption_key_name, encryption_key); + std::cout << "Wrote the new encryption key " << property_name << " to " << bootstrapFilePath() << '\n'; keys.encryption_key = encryption_key; } return keys; @@ -150,20 +167,22 @@ std::string EncryptConfig::hexDecodeAndValidateKey(const std::string& key, const } } -void EncryptConfig::writeEncryptionKeyToBootstrapFile(const utils::crypto::Bytes& encryption_key) const { +void EncryptConfig::writeEncryptionKeyToBootstrapFile(const std::string& encryption_key_name, const utils::crypto::Bytes& encryption_key) const { std::string key_encoded = utils::string::to_hex(utils::crypto::bytesToString(encryption_key)); encrypt_config::ConfigFile bootstrap_file{std::ifstream{bootstrapFilePath()}}; - if (bootstrap_file.hasValue(ENCRYPTION_KEY_PROPERTY_NAME)) { - bootstrap_file.update(ENCRYPTION_KEY_PROPERTY_NAME, key_encoded); + if (bootstrap_file.hasValue(encryption_key_name)) { + bootstrap_file.update(encryption_key_name, key_encoded); } else { - bootstrap_file.append(ENCRYPTION_KEY_PROPERTY_NAME, key_encoded); + bootstrap_file.append(encryption_key_name, key_encoded); } bootstrap_file.writeTo(bootstrapFilePath()); } -void EncryptConfig::encryptSensitiveProperties(const EncryptionKeys& keys) const { +void EncryptConfig::encryptSensitiveValuesInMinifiProperties() const { + EncryptionKeys keys = getEncryptionKeys(ENCRYPTION_KEY_PROPERTY_NAME); + encrypt_config::ConfigFile properties_file{std::ifstream{propertiesFilePath()}}; if (properties_file.size() == 0) { throw std::runtime_error{"Properties file " + propertiesFilePath().string() + " not found!"}; @@ -180,4 +199,17 @@ void EncryptConfig::encryptSensitiveProperties(const EncryptionKeys& keys) const << (num_properties_encrypted == 1 ? "property" : "properties") << " in " << propertiesFilePath() << '\n'; } +void EncryptConfig::encryptSensitiveValuesInFlowConfig( + const std::optional<std::string>& component_id, const std::optional<std::string>& property_name, const std::optional<std::string>& property_value) const { + if (!component_id && !property_name && !property_value) { + EncryptionKeys keys = getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME); + flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys, minifi_home_, flowConfigPath()); + } else if (component_id && property_name && property_value) { + EncryptionKeys keys = getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME); + flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys, minifi_home_, flowConfigPath(), *component_id, *property_name, *property_value); + } else { + throw std::runtime_error("either all of --component-id, --property-name and --property-value should be given (for batch mode) or none of them (for interactive mode)"); + } +} + } // namespace org::apache::nifi::minifi::encrypt_config diff --git a/encrypt-config/EncryptConfig.h b/encrypt-config/EncryptConfig.h index 6cdd6e440..1f48f48fb 100644 --- a/encrypt-config/EncryptConfig.h +++ b/encrypt-config/EncryptConfig.h @@ -21,40 +21,28 @@ #include "Utils.h" -namespace org { -namespace apache { -namespace nifi { -namespace minifi { -namespace encrypt_config { +namespace org::apache::nifi::minifi::encrypt_config { class EncryptConfig { public: - enum class EncryptionType { - ENCRYPT, - RE_ENCRYPT - }; - explicit EncryptConfig(const std::string& minifi_home); - EncryptionType encryptSensitiveProperties() const; - void encryptFlowConfig() const; + void encryptSensitiveValuesInMinifiProperties() const; + void encryptSensitiveValuesInFlowConfig(const std::optional<std::string>& component_id, const std::optional<std::string>& property_name, const std::optional<std::string>& property_value) const; + void encryptWholeFlowConfigFile() const; - private: - std::filesystem::path bootstrapFilePath() const; - std::filesystem::path propertiesFilePath() const; + [[nodiscard]] bool isReencrypting() const; - EncryptionKeys getEncryptionKeys() const; - std::string hexDecodeAndValidateKey(const std::string& key, const std::string& key_name) const; - void writeEncryptionKeyToBootstrapFile(const utils::crypto::Bytes& encryption_key) const; + private: + [[nodiscard]] std::filesystem::path bootstrapFilePath() const; + [[nodiscard]] std::filesystem::path propertiesFilePath() const; + [[nodiscard]] std::filesystem::path flowConfigPath() const; - void encryptSensitiveProperties(const EncryptionKeys& keys) const; + [[nodiscard]] EncryptionKeys getEncryptionKeys(std::string_view property_name) const; + [[nodiscard]] std::string hexDecodeAndValidateKey(const std::string& key, const std::string& key_name) const; + void writeEncryptionKeyToBootstrapFile(const std::string& encryption_key_name, const utils::crypto::Bytes& encryption_key) const; const std::filesystem::path minifi_home_; - EncryptionKeys keys_; }; -} // namespace encrypt_config -} // namespace minifi -} // namespace nifi -} // namespace apache -} // namespace org +} // namespace org::apache::nifi::minifi::encrypt_config diff --git a/encrypt-config/EncryptConfigMain.cpp b/encrypt-config/EncryptConfigMain.cpp index 34760781b..1faa91007 100644 --- a/encrypt-config/EncryptConfigMain.cpp +++ b/encrypt-config/EncryptConfigMain.cpp @@ -21,37 +21,64 @@ #include "EncryptConfig.h" #include "argparse/argparse.hpp" #include "agent/agent_version.h" +#include "utils/StringUtils.h" -using org::apache::nifi::minifi::encrypt_config::EncryptConfig; +namespace minifi = org::apache::nifi::minifi; +using minifi::encrypt_config::EncryptConfig; + +constexpr std::string_view OPERATION_MINIFI_PROPERTIES = "minifi-properties"; +constexpr std::string_view OPERATION_FLOW_CONFIG = "flow-config"; +constexpr std::string_view OPERATION_WHOLE_FLOW_CONFIG_FILE = "whole-flow-config-file"; int main(int argc, char* argv[]) try { argparse::ArgumentParser argument_parser("Apache MiNiFi C++ Encrypt-Config", org::apache::nifi::minifi::AgentBuild::VERSION); + argument_parser.add_argument("operation") + .default_value("minifi-properties") + .help(minifi::utils::string::join_pack("what to encrypt: ", OPERATION_MINIFI_PROPERTIES, " | ", OPERATION_FLOW_CONFIG, " | ", OPERATION_WHOLE_FLOW_CONFIG_FILE)); argument_parser.add_argument("-m", "--minifi-home") - .required() - .metavar("MINIFI_HOME") - .help("Specifies the home directory used by the minifi agent"); - argument_parser.add_argument("-e", "--encrypt-flow-config") - .default_value(false) - .implicit_value(true) - .help("If set, the flow configuration file (as specified in minifi.properties) is also encrypted."); + .required() + .metavar("MINIFI_HOME") + .help("Specifies the home directory used by the minifi agent"); + argument_parser.add_argument("--component-id") + .metavar("ID") + .help(minifi::utils::string::join_pack("Processor or controller service id (", OPERATION_FLOW_CONFIG, " only)")); + argument_parser.add_argument("--property-name") + .metavar("NAME") + .help(minifi::utils::string::join_pack("The name of the sensitive property (", OPERATION_FLOW_CONFIG, " only)")); + argument_parser.add_argument("--property-value") + .metavar("VALUE") + .help(minifi::utils::string::join_pack("The new value of the sensitive property (", OPERATION_FLOW_CONFIG, " only)")); try { argument_parser.parse_args(argc, argv); } catch (const std::runtime_error& err) { - std::cerr << err.what() << std::endl; - std::cerr << argument_parser; - std::exit(1); + std::cerr << err.what() << "\n\n" << argument_parser; + return 1; } EncryptConfig encrypt_config{argument_parser.get("-m")}; - EncryptConfig::EncryptionType type = encrypt_config.encryptSensitiveProperties(); - if (argument_parser.get<bool>("--encrypt-flow-config")) { - encrypt_config.encryptFlowConfig(); - } else if (type == EncryptConfig::EncryptionType::RE_ENCRYPT) { - std::cout << "WARNING: you did not request the flow config to be updated, " - << "if it is currently encrypted and the old key is removed, " - << "you won't be able to recover the flow config.\n"; + std::string operation = argument_parser.get("operation"); + + if (operation == OPERATION_MINIFI_PROPERTIES) { + encrypt_config.encryptSensitiveValuesInMinifiProperties(); + } else if (operation == OPERATION_FLOW_CONFIG) { + auto component_id = argument_parser.present("--component-id"); + auto property_name = argument_parser.present("--property-name"); + auto property_value = argument_parser.present("--property-value"); + encrypt_config.encryptSensitiveValuesInFlowConfig(component_id, property_name, property_value); + } else if (operation == OPERATION_WHOLE_FLOW_CONFIG_FILE) { + encrypt_config.encryptWholeFlowConfigFile(); + } else { + std::cerr << "Unknown operation: " << operation << "\n\n" << argument_parser; + return 4; } + + if ((operation == OPERATION_MINIFI_PROPERTIES || operation == OPERATION_WHOLE_FLOW_CONFIG_FILE) && encrypt_config.isReencrypting()) { + std::cout << "WARNING: an .old key was provided, which is used for both " << OPERATION_MINIFI_PROPERTIES << " and " << OPERATION_WHOLE_FLOW_CONFIG_FILE << ".\n" + << "If both are currently encrypted, make sure to run " << argv[0] << " to re-encrypt both before removing the .old key,\n" + << "otherwise you won't be able to recover the encrypted data!\n"; + } + return 0; } catch (const std::exception& ex) { std::cerr << ex.what() << "\n(" << typeid(ex).name() << ")\n"; diff --git a/encrypt-config/FlowConfigEncryptor.cpp b/encrypt-config/FlowConfigEncryptor.cpp new file mode 100644 index 000000000..aa9983091 --- /dev/null +++ b/encrypt-config/FlowConfigEncryptor.cpp @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "FlowConfigEncryptor.h" + +#include "core/extension/ExtensionManager.h" +#include "core/FlowConfiguration.h" +#include "core/flow/AdaptiveConfiguration.h" +#include "core/ProcessGroup.h" +#include "core/RepositoryFactory.h" +#include "core/repository/VolatileContentRepository.h" +#include "Defaults.h" +#include "Utils.h" +#include "utils/file/FileSystem.h" +#include "utils/Id.h" + +namespace minifi = org::apache::nifi::minifi; + +namespace { +enum class Type { + Processor, + ControllerService +}; + +struct SensitiveProperty { + Type type; + minifi::utils::Identifier component_id; + std::string component_name; + std::string property_name; + std::string property_display_name; +}; +} // namespace + +namespace magic_enum::customize { +template<> +constexpr customize_t enum_name<Type>(Type type) noexcept { + switch (type) { + case Type::Processor: return "Processor"; + case Type::ControllerService: return "Controller service"; + } + return invalid_tag; +} +} // namespace magic_enum::customize + +namespace { +std::vector<SensitiveProperty> listSensitiveProperties(const minifi::core::ProcessGroup &process_group) { + std::vector<SensitiveProperty> sensitive_properties; + + std::vector<minifi::core::Processor *> processors; + process_group.getAllProcessors(processors); + for (const auto *processor : processors) { + gsl_Expects(processor); + for (const auto& [_, property] : processor->getProperties()) { + if (property.isSensitive()) { + sensitive_properties.push_back(SensitiveProperty{ + .type = Type::Processor, + .component_id = processor->getUUID(), + .component_name = processor->getName(), + .property_name = property.getName(), + .property_display_name = property.getDisplayName()}); + } + } + } + + for (const auto* controller_service_node : process_group.getAllControllerServices()) { + gsl_Expects(controller_service_node); + const auto* controller_service = controller_service_node->getControllerServiceImplementation(); + gsl_Expects(controller_service); + for (const auto& [_, property] : controller_service->getProperties()) { + if (property.isSensitive()) { + sensitive_properties.push_back(SensitiveProperty{ + .type = Type::ControllerService, + .component_id = controller_service->getUUID(), + .component_name = controller_service->getName(), + .property_name = property.getName(), + .property_display_name = property.getDisplayName()}); + } + } + } + + return sensitive_properties; +} + +template<typename Func> +void encryptSensitiveValuesInFlowConfigImpl( + const minifi::encrypt_config::EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path, Func create_overrides) { + const auto configure = std::make_shared<minifi::Configure>(); + configure->setHome(minifi_home); + configure->loadConfigureFile(DEFAULT_NIFI_PROPERTIES_FILE); + + bool encrypt_whole_flow_config_file = (configure->get(minifi::Configure::nifi_flow_configuration_encrypt) | minifi::utils::andThen(minifi::utils::string::toBool)).value_or(false); + auto encryptor = encrypt_whole_flow_config_file ? minifi::utils::crypto::EncryptionProvider::create(minifi_home) : std::nullopt; + auto filesystem = std::make_shared<minifi::utils::file::FileSystem>(encrypt_whole_flow_config_file, encryptor); + + minifi::core::extension::ExtensionManager::get().initialize(configure); + + minifi::core::flow::AdaptiveConfiguration adaptive_configuration{minifi::core::ConfigurationContext{ + .flow_file_repo = nullptr, + .content_repo = nullptr, + .configuration = configure, + .path = flow_config_path, + .filesystem = filesystem, + .sensitive_properties_encryptor = minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{keys.encryption_key}} + }}; + + const auto flow_config_content = filesystem->read(flow_config_path); + if (!flow_config_content) { + throw std::runtime_error(minifi::utils::string::join_pack("Could not read the flow configuration file \"", flow_config_path.string(), "\"")); + } + + const auto process_group = adaptive_configuration.getRootFromPayload(*flow_config_content); + gsl_Expects(process_group); + const auto sensitive_properties = listSensitiveProperties(*process_group); + + std::unordered_map<minifi::utils::Identifier, std::unordered_map<std::string, std::string>> overrides = create_overrides(sensitive_properties); + if (overrides.empty()) { + return; + } + + std::string flow_config_str = adaptive_configuration.serializeWithOverrides(*process_group, overrides); + adaptive_configuration.persist(flow_config_str); +} +} // namespace + +namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor { + +void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path) { + encryptSensitiveValuesInFlowConfigImpl(keys, minifi_home, flow_config_path, + [](const auto& sensitive_properties) { + std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>> overrides; + std::cout << '\n'; + for (const auto& sensitive_property : sensitive_properties) { + std::cout << magic_enum::enum_name(sensitive_property.type) << " " << sensitive_property.component_name << " (" << sensitive_property.component_id.to_string() << ") " + << "has sensitive property " << sensitive_property.property_display_name << "\n enter a new value or press Enter to keep the current value unchanged: "; + std::cout.flush(); + std::string new_value; + std::getline(std::cin, new_value); + if (!new_value.empty()) { + overrides[sensitive_property.component_id].emplace(sensitive_property.property_name, new_value); + } + } + return overrides; + }); +} + +void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path, + const std::string& component_id, const std::string& property_name, const std::string& property_value) { + encryptSensitiveValuesInFlowConfigImpl(keys, minifi_home, flow_config_path, + [&](const auto& sensitive_properties) -> std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>> { + const auto sensitive_property_it = std::ranges::find_if(sensitive_properties, [&](const auto& sensitive_property) { + return sensitive_property.component_id.to_string() == component_id && (sensitive_property.property_name == property_name || sensitive_property.property_display_name == property_name); + }); + if (sensitive_property_it == sensitive_properties.end()) { + std::cout << "No sensitive property found with this component ID and property name.\n"; + return {}; + } + return {{sensitive_property_it->component_id, {{sensitive_property_it->property_name, property_value}}}}; + }); +} + +} // namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor diff --git a/libminifi/include/core/flow/AdaptiveConfiguration.h b/encrypt-config/FlowConfigEncryptor.h similarity index 55% copy from libminifi/include/core/flow/AdaptiveConfiguration.h copy to encrypt-config/FlowConfigEncryptor.h index 101aaaa8c..b4109581b 100644 --- a/libminifi/include/core/flow/AdaptiveConfiguration.h +++ b/encrypt-config/FlowConfigEncryptor.h @@ -1,5 +1,4 @@ /** - * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,22 +17,15 @@ #pragma once #include <string> -#include <memory> -#include <vector> - -#include "StructuredConfiguration.h" - -namespace org::apache::nifi::minifi::core::flow { +#include <filesystem> -class AdaptiveConfiguration : public StructuredConfiguration { - public: - explicit AdaptiveConfiguration(ConfigurationContext ctx); +#include "Utils.h" +#include "core/flow/AdaptiveConfiguration.h" - std::vector<std::string> getSupportedFormats() const override { - return {"application/json", "text/yml"}; - } +namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor { - std::unique_ptr<core::ProcessGroup> getRootFromPayload(const std::string &payload) override; -}; +void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path); +void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path, + const std::string& component_id, const std::string& property_name, const std::string& property_value); -} // namespace org::apache::nifi::minifi::core::flow +} // namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor diff --git a/encrypt-config/tests/ConfigFileEncryptorTests.cpp b/encrypt-config/tests/ConfigFileEncryptorTests.cpp index 4a779238f..0df8878b6 100644 --- a/encrypt-config/tests/ConfigFileEncryptorTests.cpp +++ b/encrypt-config/tests/ConfigFileEncryptorTests.cpp @@ -80,7 +80,7 @@ TEST_CASE("ConfigFileEncryptor can encrypt the sensitive properties", "[encrypt- REQUIRE(test_file.size() == 110); REQUIRE(check_encryption(test_file, Configuration::nifi_rest_api_password, original_password.length())); - SECTION("calling encryptSensitiveProperties a second time does nothing") { + SECTION("calling encryptSensitiveValuesInMinifiProperties a second time does nothing") { ConfigFile test_file_copy = test_file; uint32_t num_properties_encrypted = encryptSensitivePropertiesInFile(test_file, KEY); diff --git a/libminifi/include/core/FlowConfiguration.h b/libminifi/include/core/FlowConfiguration.h index 0a93c1c73..9cc746180 100644 --- a/libminifi/include/core/FlowConfiguration.h +++ b/libminifi/include/core/FlowConfiguration.h @@ -107,6 +107,7 @@ class FlowConfiguration : public CoreComponent { } bool persist(const core::ProcessGroup& process_group); + bool persist(const std::string& serialized_flow); /** * Returns the configuration path string @@ -150,7 +151,6 @@ class FlowConfiguration : public CoreComponent { utils::ChecksumCalculator checksum_calculator_; private: - bool persist(const std::string& serialized_flow); virtual std::string serialize(const ProcessGroup&) { return ""; } std::shared_ptr<logging::Logger> logger_; diff --git a/libminifi/include/core/ProcessGroup.h b/libminifi/include/core/ProcessGroup.h index 23741f955..b43fe7790 100644 --- a/libminifi/include/core/ProcessGroup.h +++ b/libminifi/include/core/ProcessGroup.h @@ -205,6 +205,8 @@ class ProcessGroup : public CoreComponent { */ std::shared_ptr<core::controller::ControllerServiceNode> findControllerService(const std::string &nodeId) const; + std::vector<const core::controller::ControllerServiceNode*> getAllControllerServices() const; + // update property value void updatePropertyValue(const std::string& processorName, const std::string& propertyName, const std::string& propertyValue); diff --git a/libminifi/include/core/controller/ControllerServiceNode.h b/libminifi/include/core/controller/ControllerServiceNode.h index 2371d32b1..5b3c70663 100644 --- a/libminifi/include/core/controller/ControllerServiceNode.h +++ b/libminifi/include/core/controller/ControllerServiceNode.h @@ -79,6 +79,7 @@ class ControllerServiceNode : public CoreComponent, public ConfigurableComponent * @return the implementation of the Controller Service */ std::shared_ptr<ControllerService> &getControllerServiceImplementation(); + const ControllerService* getControllerServiceImplementation() const; std::vector<std::shared_ptr<ControllerServiceNode> > &getLinkedControllerServices(); std::vector<std::shared_ptr<ConfigurableComponent> > &getLinkedComponents(); diff --git a/libminifi/include/core/flow/AdaptiveConfiguration.h b/libminifi/include/core/flow/AdaptiveConfiguration.h index 101aaaa8c..bc1ae9fbe 100644 --- a/libminifi/include/core/flow/AdaptiveConfiguration.h +++ b/libminifi/include/core/flow/AdaptiveConfiguration.h @@ -34,6 +34,8 @@ class AdaptiveConfiguration : public StructuredConfiguration { } std::unique_ptr<core::ProcessGroup> getRootFromPayload(const std::string &payload) override; + + std::string serializeWithOverrides(const core::ProcessGroup& process_group, const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const; }; } // namespace org::apache::nifi::minifi::core::flow diff --git a/libminifi/include/core/flow/FlowSerializer.h b/libminifi/include/core/flow/FlowSerializer.h index c10b69ba3..02ec6c1cc 100644 --- a/libminifi/include/core/flow/FlowSerializer.h +++ b/libminifi/include/core/flow/FlowSerializer.h @@ -17,10 +17,12 @@ #pragma once #include <string> +#include <unordered_map> #include "core/flow/FlowSchema.h" #include "core/ProcessGroup.h" #include "utils/crypto/EncryptionProvider.h" +#include "utils/Id.h" namespace org::apache::nifi::minifi::core::flow { @@ -34,7 +36,8 @@ class FlowSerializer { FlowSerializer(FlowSerializer&&) = delete; FlowSerializer& operator=(FlowSerializer&&) = delete; - [[nodiscard]] virtual std::string serialize(const core::ProcessGroup& process_group, const FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider) const = 0; + [[nodiscard]] virtual std::string serialize(const core::ProcessGroup& process_group, const FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider, + const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const = 0; }; } // namespace org::apache::nifi::minifi::core::flow diff --git a/libminifi/include/core/json/JsonFlowSerializer.h b/libminifi/include/core/json/JsonFlowSerializer.h index 726c78a31..1faa95c93 100644 --- a/libminifi/include/core/json/JsonFlowSerializer.h +++ b/libminifi/include/core/json/JsonFlowSerializer.h @@ -25,11 +25,13 @@ class JsonFlowSerializer : public core::flow::FlowSerializer { public: explicit JsonFlowSerializer(rapidjson::Document document) : flow_definition_json_(std::move(document)) {} - [[nodiscard]] std::string serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider) const override; + [[nodiscard]] std::string serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider, + const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const override; private: void encryptSensitiveProperties(rapidjson::Value& property_jsons, rapidjson::Document::AllocatorType& alloc, - const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider) const; + const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider, + std::unordered_map<std::string, std::string> component_overrides) const; rapidjson::Document flow_definition_json_; std::shared_ptr<logging::Logger> logger_{logging::LoggerFactory<JsonFlowSerializer>::getLogger()}; diff --git a/libminifi/include/core/yaml/YamlFlowSerializer.h b/libminifi/include/core/yaml/YamlFlowSerializer.h index 6fcb90124..3483b1638 100644 --- a/libminifi/include/core/yaml/YamlFlowSerializer.h +++ b/libminifi/include/core/yaml/YamlFlowSerializer.h @@ -25,10 +25,12 @@ class YamlFlowSerializer : public core::flow::FlowSerializer { public: explicit YamlFlowSerializer(const YAML::Node& flow_definition_yaml) : flow_definition_yaml_(flow_definition_yaml) {} - [[nodiscard]] std::string serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider) const override; + [[nodiscard]] std::string serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider, + const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const override; private: - void encryptSensitiveProperties(YAML::Node property_yamls, const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider) const; + void encryptSensitiveProperties(YAML::Node property_yamls, const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider, + std::unordered_map<std::string, std::string> component_overrides) const; YAML::Node flow_definition_yaml_; std::shared_ptr<logging::Logger> logger_{logging::LoggerFactory<YamlFlowSerializer>::getLogger()}; diff --git a/libminifi/src/core/ProcessGroup.cpp b/libminifi/src/core/ProcessGroup.cpp index e0918b974..26e5142e1 100644 --- a/libminifi/src/core/ProcessGroup.cpp +++ b/libminifi/src/core/ProcessGroup.cpp @@ -270,6 +270,14 @@ std::shared_ptr<core::controller::ControllerServiceNode> ProcessGroup::findContr return controller_service_map_.getControllerServiceNode(nodeId); } +std::vector<const core::controller::ControllerServiceNode*> ProcessGroup::getAllControllerServices() const { + std::vector<const core::controller::ControllerServiceNode*> controller_service_nodes; + for (const auto& node : controller_service_map_.getAllControllerServices()) { + controller_service_nodes.push_back(node.get()); + } + return controller_service_nodes; +} + void ProcessGroup::getAllProcessors(std::vector<Processor*>& processor_vec) const { std::lock_guard<std::recursive_mutex> lock(mutex_); diff --git a/libminifi/src/core/controller/ControllerServiceNode.cpp b/libminifi/src/core/controller/ControllerServiceNode.cpp index 309757417..f600e280f 100644 --- a/libminifi/src/core/controller/ControllerServiceNode.cpp +++ b/libminifi/src/core/controller/ControllerServiceNode.cpp @@ -20,17 +20,16 @@ #include <memory> #include <vector> -namespace org { -namespace apache { -namespace nifi { -namespace minifi { -namespace core { -namespace controller { +namespace org::apache::nifi::minifi::core::controller { std::shared_ptr<ControllerService> &ControllerServiceNode::getControllerServiceImplementation() { return controller_service_; } +const ControllerService* ControllerServiceNode::getControllerServiceImplementation() const { + return controller_service_.get(); +} + std::vector<std::shared_ptr<ControllerServiceNode> > &ControllerServiceNode::getLinkedControllerServices() { return linked_controller_services_; } @@ -39,9 +38,4 @@ std::vector<std::shared_ptr<ConfigurableComponent> > &ControllerServiceNode::get return linked_components_; } -} /* namespace controller */ -} /* namespace core */ -} /* namespace minifi */ -} /* namespace nifi */ -} /* namespace apache */ -} /* namespace org */ +} // namespace org::apache::nifi::minifi::core::controller diff --git a/libminifi/src/core/flow/AdaptiveConfiguration.cpp b/libminifi/src/core/flow/AdaptiveConfiguration.cpp index 0a75e42ae..6eac01fbb 100644 --- a/libminifi/src/core/flow/AdaptiveConfiguration.cpp +++ b/libminifi/src/core/flow/AdaptiveConfiguration.cpp @@ -71,4 +71,10 @@ std::unique_ptr<core::ProcessGroup> AdaptiveConfiguration::getRootFromPayload(co } } +std::string AdaptiveConfiguration::serializeWithOverrides(const core::ProcessGroup& process_group, + const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const { + gsl_Expects(flow_serializer_); + return flow_serializer_->serialize(process_group, schema_, sensitive_properties_encryptor_, overrides); +} + } // namespace org::apache::nifi::minifi::core::flow diff --git a/libminifi/src/core/flow/StructuredConfiguration.cpp b/libminifi/src/core/flow/StructuredConfiguration.cpp index 461c950bc..df79c9b6d 100644 --- a/libminifi/src/core/flow/StructuredConfiguration.cpp +++ b/libminifi/src/core/flow/StructuredConfiguration.cpp @@ -914,7 +914,7 @@ void StructuredConfiguration::addNewId(const std::string& uuid) { std::string StructuredConfiguration::serialize(const core::ProcessGroup& process_group) { gsl_Expects(flow_serializer_); - return flow_serializer_->serialize(process_group, schema_, sensitive_properties_encryptor_); + return flow_serializer_->serialize(process_group, schema_, sensitive_properties_encryptor_, {}); } } // namespace org::apache::nifi::minifi::core::flow diff --git a/libminifi/src/core/json/JsonFlowSerializer.cpp b/libminifi/src/core/json/JsonFlowSerializer.cpp index a68605d66..82d6d961c 100644 --- a/libminifi/src/core/json/JsonFlowSerializer.cpp +++ b/libminifi/src/core/json/JsonFlowSerializer.cpp @@ -38,7 +38,7 @@ rapidjson::Value& getMember(rapidjson::Value& node, const std::string& member_na } void JsonFlowSerializer::encryptSensitiveProperties(rapidjson::Value &property_jsons, rapidjson::Document::AllocatorType &alloc, - const std::map<std::string, Property> &properties, const utils::crypto::EncryptionProvider &encryption_provider) const { + const std::map<std::string, Property> &properties, const utils::crypto::EncryptionProvider &encryption_provider, std::unordered_map<std::string, std::string> component_overrides) const { for (auto &property : property_jsons.GetObject()) { const std::string name{property.name.GetString(), property.name.GetStringLength()}; if (!properties.contains(name)) { @@ -47,14 +47,22 @@ void JsonFlowSerializer::encryptSensitiveProperties(rapidjson::Value &property_j } if (properties.at(name).isSensitive()) { auto& value = property.value; - const std::string_view value_sv{value.GetString(), value.GetStringLength()}; + const std::string_view value_sv = component_overrides.contains(name) ? component_overrides.at(name) : std::string_view{value.GetString(), value.GetStringLength()}; const std::string encrypted_value = utils::crypto::property_encryption::encrypt(value_sv, encryption_provider); value.SetString(encrypted_value.c_str(), encrypted_value.size(), alloc); } + component_overrides.erase(name); + } + + for (const auto& [name, value] : component_overrides) { + gsl_Expects(properties.contains(name) && properties.at(name).isSensitive()); + const std::string encrypted_value = utils::crypto::property_encryption::encrypt(value, encryption_provider); + property_jsons.AddMember(rapidjson::Value(name, alloc), rapidjson::Value(encrypted_value, alloc), alloc); } } -std::string JsonFlowSerializer::serialize(const core::ProcessGroup &process_group, const core::flow::FlowSchema &schema, const utils::crypto::EncryptionProvider &encryption_provider) const { +std::string JsonFlowSerializer::serialize(const core::ProcessGroup &process_group, const core::flow::FlowSchema &schema, const utils::crypto::EncryptionProvider &encryption_provider, + const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const { gsl_Expects(schema.root_group.size() == 1 && schema.identifier.size() == 1 && schema.processors.size() == 1 && schema.processor_properties.size() == 1 && schema.controller_services.size() == 1 && schema.controller_service_properties.size() == 1); @@ -78,23 +86,32 @@ std::string JsonFlowSerializer::serialize(const core::ProcessGroup &process_grou logger_->log_warn("Processor {} not found in the flow definition", processor_id->to_string()); continue; } - encryptSensitiveProperties(getMember(processor_json, schema.processor_properties[0]), alloc, processor->getProperties(), encryption_provider); + const auto& processor_overrides = overrides.contains(*processor_id) ? overrides.at(*processor_id) : std::unordered_map<std::string, std::string>{}; + encryptSensitiveProperties(getMember(processor_json, schema.processor_properties[0]), alloc, processor->getProperties(), encryption_provider, + processor_overrides); } auto controller_services = getMember(root_group, schema.controller_services[0]).GetArray(); for (auto &controller_service_json : controller_services) { - const std::string controller_service_id{getMember(controller_service_json, schema.identifier[0]).GetString(), getMember(controller_service_json, schema.identifier[0]).GetStringLength()}; - const auto controller_service_node = process_group.findControllerService(controller_service_id); + const std::string controller_service_id_str{getMember(controller_service_json, schema.identifier[0]).GetString(), getMember(controller_service_json, schema.identifier[0]).GetStringLength()}; + const auto controller_service_id = utils::Identifier::parse(controller_service_id_str); + if (!controller_service_id) { + logger_->log_warn("Invalid controller service ID found in the flow definition: {}", controller_service_id_str); + continue; + } + const auto controller_service_node = process_group.findControllerService(controller_service_id_str); if (!controller_service_node) { - logger_->log_warn("Controller service node {} not found in the flow definition", controller_service_id); + logger_->log_warn("Controller service node {} not found in the flow definition", controller_service_id_str); continue; } const auto controller_service = controller_service_node->getControllerServiceImplementation(); if (!controller_service) { - logger_->log_warn("Controller service {} not found in the flow definition", controller_service_id); + logger_->log_warn("Controller service {} not found in the flow definition", controller_service_id_str); continue; } - encryptSensitiveProperties(getMember(controller_service_json, schema.controller_service_properties[0]), alloc, controller_service->getProperties(), encryption_provider); + const auto& controller_service_overrides = overrides.contains(*controller_service_id) ? overrides.at(*controller_service_id) : std::unordered_map<std::string, std::string>{}; + encryptSensitiveProperties(getMember(controller_service_json, schema.controller_service_properties[0]), alloc, controller_service->getProperties(), encryption_provider, + controller_service_overrides); } rapidjson::StringBuffer buffer; diff --git a/libminifi/src/core/yaml/YamlFlowSerializer.cpp b/libminifi/src/core/yaml/YamlFlowSerializer.cpp index b61f67ae3..00bd87db3 100644 --- a/libminifi/src/core/yaml/YamlFlowSerializer.cpp +++ b/libminifi/src/core/yaml/YamlFlowSerializer.cpp @@ -21,7 +21,8 @@ namespace org::apache::nifi::minifi::core::yaml { -void YamlFlowSerializer::encryptSensitiveProperties(YAML::Node property_yamls, const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider) const { +void YamlFlowSerializer::encryptSensitiveProperties(YAML::Node property_yamls, const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider, + std::unordered_map<std::string, std::string> component_overrides) const { for (auto kv : property_yamls) { auto name = kv.first.as<std::string>(); if (!properties.contains(name)) { @@ -31,18 +32,25 @@ void YamlFlowSerializer::encryptSensitiveProperties(YAML::Node property_yamls, c if (properties.at(name).isSensitive()) { if (kv.second.IsSequence()) { for (auto property_item : kv.second) { - auto value = property_item["value"].as<std::string>(); + auto value = component_overrides.contains(name) ? component_overrides.at(name) : property_item["value"].as<std::string>(); property_item["value"] = utils::crypto::property_encryption::encrypt(value, encryption_provider); } } else { - auto value = kv.second.as<std::string>(); + auto value = component_overrides.contains(name) ? component_overrides.at(name) : kv.second.as<std::string>(); property_yamls[name] = utils::crypto::property_encryption::encrypt(value, encryption_provider); } + component_overrides.erase(name); } } + + for (const auto& [name, value] : component_overrides) { + gsl_Expects(properties.contains(name) && properties.at(name).isSensitive()); + property_yamls[name] = utils::crypto::property_encryption::encrypt(value, encryption_provider); + } } -std::string YamlFlowSerializer::serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider) const { +std::string YamlFlowSerializer::serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider, + const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const { gsl_Expects(schema.identifier.size() == 1 && schema.processors.size() == 1 && schema.processor_properties.size() == 1 && schema.controller_services.size() == 1 && schema.controller_service_properties.size() == 1); @@ -60,22 +68,29 @@ std::string YamlFlowSerializer::serialize(const core::ProcessGroup& process_grou logger_->log_warn("Processor {} not found in the flow definition", processor_id->to_string()); continue; } - encryptSensitiveProperties(processor_yaml[schema.processor_properties[0]], processor->getProperties(), encryption_provider); + const auto& processor_overrides = overrides.contains(*processor_id) ? overrides.at(*processor_id) : std::unordered_map<std::string, std::string>{}; + encryptSensitiveProperties(processor_yaml[schema.processor_properties[0]], processor->getProperties(), encryption_provider, processor_overrides); } for (auto controller_service_yaml : flow_definition_yaml[schema.controller_services[0]]) { - const auto controller_service_id = controller_service_yaml[schema.identifier[0]].Scalar(); - const auto controller_service_node = process_group.findControllerService(controller_service_id); + const auto controller_service_id_str = controller_service_yaml[schema.identifier[0]].Scalar(); + const auto controller_service_id = utils::Identifier::parse(controller_service_id_str); + if (!controller_service_id) { + logger_->log_warn("Invalid controller service ID found in the flow definition: {}", controller_service_id_str); + continue; + } + const auto controller_service_node = process_group.findControllerService(controller_service_id_str); if (!controller_service_node) { - logger_->log_warn("Controller service node {} not found in the flow definition", controller_service_id); + logger_->log_warn("Controller service node {} not found in the flow definition", controller_service_id_str); continue; } const auto controller_service = controller_service_node->getControllerServiceImplementation(); if (!controller_service) { - logger_->log_warn("Controller service {} not found in the flow definition", controller_service_id); + logger_->log_warn("Controller service {} not found in the flow definition", controller_service_id_str); continue; } - encryptSensitiveProperties(controller_service_yaml[schema.controller_service_properties[0]], controller_service->getProperties(), encryption_provider); + const auto& controller_service_overrides = overrides.contains(*controller_service_id) ? overrides.at(*controller_service_id) : std::unordered_map<std::string, std::string>{}; + encryptSensitiveProperties(controller_service_yaml[schema.controller_service_properties[0]], controller_service->getProperties(), encryption_provider, controller_service_overrides); } return YAML::Dump(flow_definition_yaml) + '\n'; diff --git a/libminifi/test/unit/JsonFlowSerializerTests.cpp b/libminifi/test/unit/JsonFlowSerializerTests.cpp index 3412376a7..49cd913aa 100644 --- a/libminifi/test/unit/JsonFlowSerializerTests.cpp +++ b/libminifi/test/unit/JsonFlowSerializerTests.cpp @@ -19,6 +19,7 @@ #include "../Catch.h" #include "../ConfigurationTestController.h" +#include "catch2/generators/catch_generators.hpp" #include "core/flow/FlowSchema.h" #include "core/json/JsonFlowSerializer.h" #include "core/json/JsonNode.h" @@ -104,7 +105,8 @@ constexpr std::string_view config_json_with_default_schema = R"({ } )"; -constexpr std::string_view config_json_with_nifi_schema = R"({ +// in two parts because Visual Studio doesn't like long string constants +constexpr std::string_view config_json_with_nifi_schema_part_1 = R"({ "encodingVersion": { "majorVersion": 2, "minorVersion": 0 @@ -390,7 +392,9 @@ constexpr std::string_view config_json_with_nifi_schema = R"({ } ], "labels": [], - "funnels": [], + "funnels": [],)"; + +constexpr std::string_view config_json_with_nifi_schema_part_2 = R"( "controllerServices": [ { "identifier": "b9801278-7b5d-4314-aed6-713fd4b5f933", @@ -452,6 +456,66 @@ constexpr std::string_view config_json_with_nifi_schema = R"({ ], "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "8b4d66dc-9085-4722-b35b-3492f363baa3" + }, + { + "identifier": "b418f4ff-e598-4ea2-921f-14f9dd864482", + "instanceIdentifier": "dbb76c00-97ad-4b3a-bf0c-d5d1b88e79d3", + "name": "Second SSLContextService", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.minifi.controllers.SSLContextService", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-system", + "version": "1.23.06" + }, + "properties": { + "Private Key": "second_ssl_service_certs/agent-key.pem", + "Client Certificate": "second_ssl_service_certs/agent-cert.pem", + "CA Certificate": "second_ssl_service_certs/ca-cert.pem", + "Use System Cert Store": "false" + }, + "propertyDescriptors": { + "Private Key": { + "name": "Private Key", + "identifiesControllerService": false, + "sensitive": false + }, + "Client Certificate": { + "name": "Client Certificate", + "identifiesControllerService": false, + "sensitive": false + }, + "Passphrase": { + "name": "Passphrase", + "identifiesControllerService": false, + "sensitive": true + }, + "CA Certificate": { + "name": "CA Certificate", + "identifiesControllerService": false, + "sensitive": false + }, + "Use System Cert Store": { + "name": "Use System Cert Store", + "identifiesControllerService": false, + "sensitive": false + } + }, + "controllerServiceApis": [ + { + "type": "org.apache.nifi.minifi.controllers.SSLContextService", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-system", + "version": "1.23.06" + } + } + ], + "componentType": "CONTROLLER_SERVICE", + "groupIdentifier": "910ec043-2372-4edb-9ef4-8ce720c50685" } ], "variables": {}, @@ -467,16 +531,9 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive properties") { ConfigurationTestController test_controller; core::flow::AdaptiveConfiguration json_configuration{test_controller.getContext()}; - core::flow::FlowSchema schema; - std::string flow_definition; - SECTION("Default schema") { - schema = core::flow::FlowSchema::getDefault(); - flow_definition = std::string{config_json_with_default_schema}; - } - SECTION("NiFi schema") { - schema = core::flow::FlowSchema::getNiFiFlowJson(); - flow_definition = std::string{config_json_with_nifi_schema}; - } + const auto [schema, flow_definition] = GENERATE( + std::make_tuple(core::flow::FlowSchema::getDefault(), std::string{config_json_with_default_schema}), + std::make_tuple(core::flow::FlowSchema::getNiFiFlowJson(), utils::string::join_pack(config_json_with_nifi_schema_part_1, config_json_with_nifi_schema_part_2))); const auto process_group = json_configuration.getRootFromPayload(flow_definition); REQUIRE(process_group); @@ -486,7 +543,21 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive properties") { REQUIRE(res); const auto flow_serializer = core::json::JsonFlowSerializer{std::move(doc)}; - std::string config_json_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider); + using Overrides = std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>; + const auto processor_id = utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value(); + const auto controller_service_id = utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value(); + + const auto [overrides, expected_results] = GENERATE_REF( + std::make_tuple(Overrides{}, + std::array{"very_secure_password", "very_secure_passphrase"}), + std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password", "password123"}}}}, + std::array{"password123", "very_secure_passphrase"}), + std::make_tuple(Overrides{{controller_service_id, {{"Passphrase", "speak friend and enter"}}}}, + std::array{"very_secure_password", "speak friend and enter"}), + std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password", "password123"}}}, {controller_service_id, {{"Passphrase", "speak friend and enter"}}}}, + std::array{"password123", "speak friend and enter"})); + + std::string config_json_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider, overrides); { std::regex regex{R"_("invokehttp-proxy-password": "(enc\{.*\})",)_"}; @@ -495,7 +566,7 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive properties") { REQUIRE(match_results.size() == 2); std::string encrypted_value = match_results[1]; - CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "very_secure_password"); + CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == expected_results[0]); } { @@ -505,6 +576,36 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive properties") { REQUIRE(match_results.size() == 2); std::string encrypted_value = match_results[1]; - CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "very_secure_passphrase"); + CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == expected_results[1]); } } + +TEST_CASE("JsonFlowSerializer with an override can add a new property to the flow config file") { + ConfigurationTestController test_controller; + core::flow::AdaptiveConfiguration json_configuration{test_controller.getContext()}; + + const auto schema = core::flow::FlowSchema::getNiFiFlowJson(); + const auto config_json_with_nifi_schema = utils::string::join_pack(config_json_with_nifi_schema_part_1, config_json_with_nifi_schema_part_2); + const auto process_group = json_configuration.getRootFromPayload(config_json_with_nifi_schema); + REQUIRE(process_group); + + rapidjson::Document doc; + rapidjson::ParseResult res = doc.Parse(config_json_with_nifi_schema.data(), config_json_with_nifi_schema.size()); + REQUIRE(res); + const auto flow_serializer = core::json::JsonFlowSerializer{std::move(doc)}; + + const auto second_controller_service_id = utils::Identifier::parse("b418f4ff-e598-4ea2-921f-14f9dd864482").value(); + const auto overrides = std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>{{second_controller_service_id, {{"Passphrase", "new passphrase"}}}}; + + std::string config_json_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider, overrides); + + // find the second match + std::regex regex{R"_("Passphrase": "(enc\{.*\})")_"}; + std::smatch match_results; + REQUIRE(std::regex_search(config_json_encrypted.cbegin(), config_json_encrypted.cend(), match_results, regex)); + REQUIRE(std::regex_search(match_results.suffix().first, config_json_encrypted.cend(), match_results, regex)); + REQUIRE(match_results.size() == 2); + + std::string encrypted_value = match_results[1]; + CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "new passphrase"); +} diff --git a/libminifi/test/unit/YamlFlowSerializerTests.cpp b/libminifi/test/unit/YamlFlowSerializerTests.cpp index a35162ccd..25dff1514 100644 --- a/libminifi/test/unit/YamlFlowSerializerTests.cpp +++ b/libminifi/test/unit/YamlFlowSerializerTests.cpp @@ -19,6 +19,7 @@ #include "../Catch.h" #include "../ConfigurationTestController.h" +#include "catch2/generators/catch_generators.hpp" #include "core/flow/FlowSchema.h" #include "core/yaml/YamlFlowSerializer.h" #include "core/yaml/YamlNode.h" @@ -142,6 +143,14 @@ Controller Services: Passphrase: very_secure_passphrase Private Key: certs/agent-key.pem Use System Cert Store: false + - id: b418f4ff-e598-4ea2-921f-14f9dd864482 + name: Second SSLContextService + type: org.apache.nifi.minifi.controllers.SSLContextService + Properties: + CA Certificate: second_ssl_service_certs/ca-cert.pem + Client Certificate: second_ssl_service_certs/agent-cert.pem + Private Key: second_ssl_service_certs/agent-key.pem + Use System Cert Store: false Process Groups: [] Input Ports: [] Output Ports: [] @@ -175,7 +184,21 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive properties") { YAML::Node root_yaml_node = YAML::Load(std::string{config_yaml}); const auto flow_serializer = core::yaml::YamlFlowSerializer{root_yaml_node}; - std::string config_yaml_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider); + using Overrides = std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>; + const auto processor_id = utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value(); + const auto controller_service_id = utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value(); + + const auto [overrides, expected_results] = GENERATE_REF( + std::make_tuple(Overrides{}, + std::array{"very_secure_password", "very_secure_passphrase"}), + std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password", "password123"}}}}, + std::array{"password123", "very_secure_passphrase"}), + std::make_tuple(Overrides{{controller_service_id, {{"Passphrase", "speak friend and enter"}}}}, + std::array{"very_secure_password", "speak friend and enter"}), + std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password", "password123"}}}, {controller_service_id, {{"Passphrase", "speak friend and enter"}}}}, + std::array{"password123", "speak friend and enter"})); + + std::string config_yaml_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider, overrides); { std::regex regex{R"_(invokehttp-proxy-password: (enc\{.*\}))_"}; @@ -184,7 +207,7 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive properties") { REQUIRE(match_results.size() == 2); std::string encrypted_value = match_results[1]; - CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "very_secure_password"); + CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == expected_results[0]); } { @@ -194,6 +217,33 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive properties") { REQUIRE(match_results.size() == 2); std::string encrypted_value = match_results[1]; - CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "very_secure_passphrase"); + CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == expected_results[1]); } } + +TEST_CASE("YamlFlowSerializer with an override can add a new property to the flow config file") { + ConfigurationTestController test_controller; + core::YamlConfiguration yaml_configuration{test_controller.getContext()}; + const auto process_group = yaml_configuration.getRootFromPayload(std::string{config_yaml}); + REQUIRE(process_group); + + const auto schema = core::flow::FlowSchema::getDefault(); + + YAML::Node root_yaml_node = YAML::Load(std::string{config_yaml}); + const auto flow_serializer = core::yaml::YamlFlowSerializer{root_yaml_node}; + + const auto second_controller_service_id = utils::Identifier::parse("b418f4ff-e598-4ea2-921f-14f9dd864482").value(); + const auto overrides = std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>{{second_controller_service_id, {{"Passphrase", "new passphrase"}}}}; + + std::string config_yaml_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider, overrides); + + // find the second match + std::regex regex{R"_(Passphrase: (enc\{.*\}))_"}; + std::smatch match_results; + REQUIRE(std::regex_search(config_yaml_encrypted.cbegin(), config_yaml_encrypted.cend(), match_results, regex)); + REQUIRE(std::regex_search(match_results.suffix().first, config_yaml_encrypted.cend(), match_results, regex)); + REQUIRE(match_results.size() == 2); + + std::string encrypted_value = match_results[1]; + CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "new passphrase"); +}