This is an automated email from the ASF dual-hosted git repository.
lordgamez pushed a commit to branch MINIFICPP-2368
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit af644edca61434df993e8133d4b615b28cadb863
Author: Gabor Gyimesi <gamezb...@gmail.com>
AuthorDate: Wed May 22 12:29:21 2024 +0200
MINIFICPP-2368 Add sensitive parameter support
---
CONFIGURE.md | 4 +-
encrypt-config/EncryptConfigMain.cpp | 8 +-
encrypt-config/FlowConfigEncryptor.cpp | 96 +++++++++------
extensions/coap/tests/CoapIntegrationBase.h | 2 +-
extensions/rocksdb-repos/tests/RepoTests.cpp | 2 +-
.../tests/integration/VerifyInvokeHTTP.h | 2 +-
.../tests/unit/FlowJsonTests.cpp | 133 +++++++++++++++++++++
.../tests/unit/YamlConfigurationTests.cpp | 116 ++++++++++++++++++
.../tests/unit/YamlProcessGroupParserTests.cpp | 2 +-
libminifi/include/core/FlowConfiguration.h | 8 +-
libminifi/include/core/ParameterContext.h | 4 +
libminifi/include/core/ParameterTokenParser.h | 33 ++++-
.../include/core/flow/AdaptiveConfiguration.h | 2 +-
libminifi/include/core/flow/FlowSchema.h | 1 +
libminifi/include/core/json/JsonFlowSerializer.h | 6 +
libminifi/include/core/yaml/YamlFlowSerializer.h | 6 +
libminifi/src/core/FlowConfiguration.cpp | 2 +-
libminifi/src/core/ParameterTokenParser.cpp | 23 +++-
libminifi/src/core/flow/AdaptiveConfiguration.cpp | 6 +-
libminifi/src/core/flow/FlowSchema.cpp | 6 +-
.../src/core/flow/StructuredConfiguration.cpp | 26 ++--
libminifi/src/core/json/JsonFlowSerializer.cpp | 64 ++++++++--
libminifi/src/core/yaml/YamlFlowSerializer.cpp | 51 +++++++-
libminifi/test/integration/C2PauseResumeTest.cpp | 4 +-
.../ControllerServiceIntegrationTests.cpp | 4 +-
.../keyvalue-tests/PersistentStateStorageTest.cpp | 2 +-
.../keyvalue-tests/VolatileMapStateStorageTest.cpp | 2 +-
.../test/libtest/integration/IntegrationBase.cpp | 4 +-
.../libtest/unit/ConfigurationTestController.h | 6 +-
.../test/libtest/unit/TestControllerWithFlow.cpp | 2 +-
.../test/persistence-tests/PersistenceTests.cpp | 4 +-
libminifi/test/unit/JsonFlowSerializerTests.cpp | 61 ++++++++--
libminifi/test/unit/ParameterTokenParserTest.cpp | 80 ++++++++-----
libminifi/test/unit/YamlFlowSerializerTests.cpp | 39 ++++--
minifi_main/MiNiFiMain.cpp | 2 +-
35 files changed, 665 insertions(+), 148 deletions(-)
diff --git a/CONFIGURE.md b/CONFIGURE.md
index adcec293a..f34a93bcc 100644
--- a/CONFIGURE.md
+++ b/CONFIGURE.md
@@ -138,7 +138,7 @@ Processor properties in flow configurations can be
parameterized using parameter
- The parameters can be used in the flow configuration by using the
`#{parameterName}` syntax for processor properties.
- Only alpha-numeric characters (a-z, A-Z, 0-9), hyphens ( - ), underscores (
_ ), periods ( . ), and spaces are allowed in parameter name.
- `#` character can be used to escape the parameter syntax. E.g. if the
`parameterName` parameter's value is `xxx` then `#{parameterName}` will be
replaced with `xxx`, `##{parameterName}` will be replaced with
`#{parameterName}`, and `#####{parameterName}` will be replaced with `##xxx`.
- - Parameters can only be used in values of non-sensitive processor properties.
+ - Sensitive parameters can only be assigned to sensitive properties and
non-sensitive parameters can only be assigned to non-sensitive properties.
An example for using parameters in a JSON configuration file:
@@ -153,6 +153,7 @@ An example for using parameters in a JSON configuration
file:
{
"name": "tail_base_dir",
"description": "Base dir of tailed files",
+ "sensitive": false,
"value": "/tmp/tail/file/path"
}
]
@@ -200,6 +201,7 @@ An example for using parameters in a YAML configuration
file:
Parameters:
- name: tail_base_dir
description: 'Base dir of tailed files'
+ sensitive: false
value: /tmp/tail/file/path
Processors:
- name: Tail test_file1.log
diff --git a/encrypt-config/EncryptConfigMain.cpp
b/encrypt-config/EncryptConfigMain.cpp
index a18e8d075..e00ad6a16 100644
--- a/encrypt-config/EncryptConfigMain.cpp
+++ b/encrypt-config/EncryptConfigMain.cpp
@@ -42,12 +42,12 @@ int main(int argc, char* argv[]) try {
argument_parser.add_argument("--component-id")
.metavar("ID")
.help(minifi::utils::string::join_pack("Processor or controller service
id (", OPERATION_FLOW_CONFIG, " only)"));
- argument_parser.add_argument("--property-name")
+ argument_parser.add_argument("--property-name", "--parameter-name",
"--item-name")
.metavar("NAME")
- .help(minifi::utils::string::join_pack("The name of the sensitive
property (", OPERATION_FLOW_CONFIG, " only)"));
- argument_parser.add_argument("--property-value")
+ .help(minifi::utils::string::join_pack("The name of the sensitive
property or parameter (", OPERATION_FLOW_CONFIG, " only)"));
+ argument_parser.add_argument("--property-value", "--parameter-value",
"--item-value")
.metavar("VALUE")
- .help(minifi::utils::string::join_pack("The new value of the sensitive
property (", OPERATION_FLOW_CONFIG, " only)"));
+ .help(minifi::utils::string::join_pack("The new value of the sensitive
property or parameter (", OPERATION_FLOW_CONFIG, " only)"));
argument_parser.add_argument("--re-encrypt")
.flag()
.help(minifi::utils::string::join_pack("Decrypt all properties with the
old key and re-encrypt them with a new key (", OPERATION_FLOW_CONFIG, "
only)"));
diff --git a/encrypt-config/FlowConfigEncryptor.cpp
b/encrypt-config/FlowConfigEncryptor.cpp
index 1e35f79d1..2261d148b 100644
--- a/encrypt-config/FlowConfigEncryptor.cpp
+++ b/encrypt-config/FlowConfigEncryptor.cpp
@@ -32,17 +32,19 @@ namespace minifi = org::apache::nifi::minifi;
namespace {
enum class ComponentType {
Processor,
- ControllerService
+ ControllerService,
+ ParameterContext
};
-struct SensitiveProperty {
+struct SensitiveItem {
ComponentType component_type;
minifi::utils::Identifier component_id;
std::string component_name;
- std::string property_name;
- std::string property_display_name;
- std::string property_value;
+ std::string item_name;
+ std::string item_display_name;
+ std::string item_value;
};
+
} // namespace
namespace magic_enum::customize {
@@ -51,14 +53,30 @@ constexpr customize_t
enum_name<ComponentType>(ComponentType type) noexcept {
switch (type) {
case ComponentType::Processor: return "Processor";
case ComponentType::ControllerService: return "Controller service";
+ case ComponentType::ParameterContext: return "Parameter context";
}
return invalid_tag;
}
} // namespace magic_enum::customize
namespace {
-std::vector<SensitiveProperty> listSensitiveProperties(const
minifi::core::ProcessGroup &process_group) {
- std::vector<SensitiveProperty> sensitive_properties;
+std::vector<SensitiveItem> listSensitiveItems(const minifi::core::ProcessGroup
&process_group,
+ const std::unordered_map<std::string,
std::unique_ptr<minifi::core::ParameterContext>>& parameter_contexts) {
+ std::vector<SensitiveItem> sensitive_items;
+
+ for (const auto& [parameter_context_name, parameter_context] :
parameter_contexts) {
+ for (const auto& [parameter_name, parameter] :
parameter_context->getParameters()) {
+ if (parameter.sensitive) {
+ sensitive_items.push_back(SensitiveItem{
+ .component_type = ComponentType::ParameterContext,
+ .component_id = parameter_context->getUUID(),
+ .component_name = parameter_context_name,
+ .item_name = parameter_name,
+ .item_display_name = parameter_name,
+ .item_value = parameter.value});
+ }
+ }
+ }
std::vector<minifi::core::Processor *> processors;
process_group.getAllProcessors(processors);
@@ -66,13 +84,13 @@ std::vector<SensitiveProperty>
listSensitiveProperties(const minifi::core::Proce
gsl_Expects(processor);
for (const auto& [_, property] : processor->getProperties()) {
if (property.isSensitive()) {
- sensitive_properties.push_back(SensitiveProperty{
+ sensitive_items.push_back(SensitiveItem{
.component_type = ComponentType::Processor,
.component_id = processor->getUUID(),
.component_name = processor->getName(),
- .property_name = property.getName(),
- .property_display_name = property.getDisplayName(),
- .property_value = property.getValue().to_string()});
+ .item_name = property.getName(),
+ .item_display_name = property.getDisplayName(),
+ .item_value = property.getValue().to_string()});
}
}
}
@@ -82,58 +100,59 @@ std::vector<SensitiveProperty>
listSensitiveProperties(const minifi::core::Proce
gsl_Expects(controller_service_node);
const auto* controller_service =
controller_service_node->getControllerServiceImplementation();
gsl_Expects(controller_service);
+ auto props = controller_service->getProperties();
if (processed_controller_services.contains(controller_service->getUUID()))
{
continue;
}
processed_controller_services.insert(controller_service->getUUID());
- for (const auto& [_, property] : controller_service->getProperties()) {
+ for (const auto& [_, property] : props) {
if (property.isSensitive()) {
- sensitive_properties.push_back(SensitiveProperty{
+ sensitive_items.push_back(SensitiveItem{
.component_type = ComponentType::ControllerService,
.component_id = controller_service->getUUID(),
.component_name = controller_service->getName(),
- .property_name = property.getName(),
- .property_display_name = property.getDisplayName(),
- .property_value = property.getValue().to_string()});
+ .item_name = property.getName(),
+ .item_display_name = property.getDisplayName(),
+ .item_value = property.getValue().to_string()});
}
}
}
- return sensitive_properties;
+ return sensitive_items;
}
-std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
createOverridesInteractively(const std::vector<SensitiveProperty>&
sensitive_properties) {
+std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
createOverridesInteractively(const std::vector<SensitiveItem>& sensitive_items)
{
std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
overrides;
std::cout << '\n';
- for (const auto& sensitive_property : sensitive_properties) {
- std::cout << magic_enum::enum_name(sensitive_property.component_type) << "
" << sensitive_property.component_name << " (" <<
sensitive_property.component_id.to_string() << ") "
- << "has sensitive property " <<
sensitive_property.property_display_name << "\n enter a new value or press
Enter to keep the current value unchanged: ";
+ for (const auto& sensitive_item : sensitive_items) {
+ std::cout << magic_enum::enum_name(sensitive_item.component_type) << " "
<< sensitive_item.component_name << " (" <<
sensitive_item.component_id.to_string() << ") "
+ << "has sensitive item " << sensitive_item.item_display_name <<
"\n enter a new value or press Enter to keep the current value unchanged: ";
std::cout.flush();
std::string new_value;
std::getline(std::cin, new_value);
if (!new_value.empty()) {
-
overrides[sensitive_property.component_id].add(sensitive_property.property_name,
new_value);
+ overrides[sensitive_item.component_id].add(sensitive_item.item_name,
new_value);
}
}
return overrides;
}
-std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
createOverridesForSingleProperty(
- const std::vector<SensitiveProperty>& sensitive_properties, const
std::string& component_id, const std::string& property_name, const std::string&
property_value) {
- const auto sensitive_property_it =
std::ranges::find_if(sensitive_properties, [&](const auto& sensitive_property) {
- return sensitive_property.component_id.to_string().view() == component_id
&& (sensitive_property.property_name == property_name ||
sensitive_property.property_display_name == property_name);
+std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
createOverridesForSingleItem(
+ const std::vector<SensitiveItem>& sensitive_items, const std::string&
component_id, const std::string& item_name, const std::string& item_value) {
+ const auto sensitive_item_it = std::ranges::find_if(sensitive_items,
[&](const auto& sensitive_item) {
+ return sensitive_item.component_id.to_string().view() == component_id &&
(sensitive_item.item_name == item_name || sensitive_item.item_display_name ==
item_name);
});
- if (sensitive_property_it == sensitive_properties.end()) {
- std::cout << "No sensitive property found with this component ID and
property name.\n";
+ if (sensitive_item_it == sensitive_items.end()) {
+ std::cout << "No sensitive item found with this component ID and item
name.\n";
return {};
}
- return {{sensitive_property_it->component_id,
minifi::core::flow::Overrides{}.add(sensitive_property_it->property_name,
property_value)}};
+ return {{sensitive_item_it->component_id,
minifi::core::flow::Overrides{}.add(sensitive_item_it->item_name, item_value)}};
}
-std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
createOverridesForReEncryption(const std::vector<SensitiveProperty>&
sensitive_properties) {
+std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
createOverridesForReEncryption(const std::vector<SensitiveItem>&
sensitive_items) {
std::unordered_map<minifi::utils::Identifier, minifi::core::flow::Overrides>
overrides;
- for (const auto& sensitive_property : sensitive_properties) {
-
overrides[sensitive_property.component_id].addOptional(sensitive_property.property_name,
sensitive_property.property_value);
+ for (const auto& sensitive_item : sensitive_items) {
+
overrides[sensitive_item.component_id].addOptional(sensitive_item.item_name,
sensitive_item.item_value);
}
return overrides;
}
@@ -169,7 +188,7 @@ void encryptSensitiveValuesInFlowConfig(const
EncryptionKeys& keys, const std::f
auto whole_file_encryptor = encrypt_whole_flow_config_file ?
utils::crypto::EncryptionProvider::create(minifi_home) : std::nullopt;
auto filesystem =
std::make_shared<utils::file::FileSystem>(encrypt_whole_flow_config_file,
whole_file_encryptor);
- auto sensitive_properties_decryptor = is_re_encrypting ?
+ auto sensitive_values_decryptor = is_re_encrypting ?
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{*keys.old_key}}
:
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{keys.encryption_key}};
@@ -181,7 +200,7 @@ void encryptSensitiveValuesInFlowConfig(const
EncryptionKeys& keys, const std::f
.configuration = configure,
.path = flow_config_path,
.filesystem = filesystem,
- .sensitive_properties_encryptor = sensitive_properties_decryptor
+ .sensitive_values_encryptor = sensitive_values_decryptor
}};
const auto flow_config_content = filesystem->read(flow_config_path);
@@ -191,16 +210,17 @@ void encryptSensitiveValuesInFlowConfig(const
EncryptionKeys& keys, const std::f
const auto process_group =
adaptive_configuration.getRootFromPayload(*flow_config_content);
gsl_Expects(process_group);
- const auto sensitive_properties = listSensitiveProperties(*process_group);
+ const auto sensitive_items = listSensitiveItems(*process_group,
adaptive_configuration.getParameterContexts());
const auto overrides = [&]() -> std::unordered_map<utils::Identifier,
core::flow::Overrides> {
switch (request.type) {
- case EncryptionType::Interactive: return
createOverridesInteractively(sensitive_properties);
- case EncryptionType::SingleProperty: return
createOverridesForSingleProperty(sensitive_properties, request.component_id,
request.property_name, request.property_value);
- case EncryptionType::ReEncrypt: return
createOverridesForReEncryption(sensitive_properties);
+ case EncryptionType::Interactive: return
createOverridesInteractively(sensitive_items);
+ case EncryptionType::SingleProperty: return
createOverridesForSingleItem(sensitive_items, request.component_id,
request.property_name, request.property_value);
+ case EncryptionType::ReEncrypt: return
createOverridesForReEncryption(sensitive_items);
}
return {};
}();
+
if (overrides.empty()) {
std::cout << "Nothing to do, exiting.\n";
return;
diff --git a/extensions/coap/tests/CoapIntegrationBase.h
b/extensions/coap/tests/CoapIntegrationBase.h
index 6974143ca..1aa695502 100644
--- a/extensions/coap/tests/CoapIntegrationBase.h
+++ b/extensions/coap/tests/CoapIntegrationBase.h
@@ -72,7 +72,7 @@ class CoapIntegrationBase : public
minifi::test::IntegrationBase {
.configuration = configuration,
.path = test_file_location,
.filesystem = std::make_shared<utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
};
auto yaml_ptr =
std::make_shared<core::YamlConfiguration>(configuration_context);
core::YamlConfiguration yaml_config(configuration_context);
diff --git a/extensions/rocksdb-repos/tests/RepoTests.cpp
b/extensions/rocksdb-repos/tests/RepoTests.cpp
index 70e31c86f..fbcf32e4e 100644
--- a/extensions/rocksdb-repos/tests/RepoTests.cpp
+++ b/extensions/rocksdb-repos/tests/RepoTests.cpp
@@ -303,7 +303,7 @@ TEST_CASE("Test FlowFile Restore", "[TestFFR6]") {
.configuration = config,
.path = "",
.filesystem = std::make_shared<utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
});
auto flowController = std::make_shared<minifi::FlowController>(prov_repo,
ff_repository, config, std::move(flowConfig), content_repo);
diff --git
a/extensions/standard-processors/tests/integration/VerifyInvokeHTTP.h
b/extensions/standard-processors/tests/integration/VerifyInvokeHTTP.h
index 650ef2143..0419c9627 100644
--- a/extensions/standard-processors/tests/integration/VerifyInvokeHTTP.h
+++ b/extensions/standard-processors/tests/integration/VerifyInvokeHTTP.h
@@ -101,7 +101,7 @@ class VerifyInvokeHTTP : public HTTPIntegrationBase {
.configuration = configuration,
.path = flow_yml_path,
.filesystem = std::make_shared<minifi::utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}}
+ .sensitive_values_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}}
};
auto yaml_ptr =
std::make_unique<core::YamlConfiguration>(configuration_context);
flowController_ = std::make_unique<minifi::FlowController>(test_repo,
test_flow_repo, configuration, std::move(yaml_ptr), content_repo);
diff --git a/extensions/standard-processors/tests/unit/FlowJsonTests.cpp
b/extensions/standard-processors/tests/unit/FlowJsonTests.cpp
index ede8dccb3..d088bad28 100644
--- a/extensions/standard-processors/tests/unit/FlowJsonTests.cpp
+++ b/extensions/standard-processors/tests/unit/FlowJsonTests.cpp
@@ -28,6 +28,7 @@
#include "unit/ConfigurationTestController.h"
#include "Funnel.h"
#include "core/Resource.h"
+#include "utils/crypto/property_encryption/PropertyEncryptionUtils.h"
using namespace std::literals::chrono_literals;
@@ -50,11 +51,13 @@ TEST_CASE("NiFi flow json format is correctly parsed") {
{
"name": "file_size",
"description": "",
+ "sensitive": false,
"value": "10 B"
},
{
"name": "batch_size",
"description": "",
+ "sensitive": false,
"value": "12"
}
]
@@ -201,6 +204,7 @@ TEST_CASE("Parameters from different parameter contexts
should not be replaced")
{
"name": "file_size",
"description": "",
+ "sensitive": false,
"value": "10 B"
}
]
@@ -213,6 +217,7 @@ TEST_CASE("Parameters from different parameter contexts
should not be replaced")
{
"name": "batch_size",
"description": "",
+ "sensitive": false,
"value": "12"
}
]
@@ -260,6 +265,7 @@ TEST_CASE("Cannot use the same parameter context name
twice") {
{
"name": "file_size",
"description": "",
+ "sensitive": false,
"value": "10 B"
}
]
@@ -272,6 +278,7 @@ TEST_CASE("Cannot use the same parameter context name
twice") {
{
"name": "batch_size",
"description": "",
+ "sensitive": false,
"value": "12"
}
]
@@ -304,11 +311,13 @@ TEST_CASE("Cannot use the same parameter name within a
parameter context twice")
{
"name": "file_size",
"description": "",
+ "sensitive": false,
"value": "10 B"
},
{
"name": "file_size",
"description": "",
+ "sensitive": false,
"value": "12 B"
}
]
@@ -366,6 +375,7 @@ TEST_CASE("Cannot use non-sensitive parameter in sensitive
property") {
{
"name": "my_value",
"description": "",
+ "sensitive": false,
"value": "value1"
}
]
@@ -408,6 +418,7 @@ TEST_CASE("Cannot use non-sensitive parameter in sensitive
property value sequen
{
"name": "my_value",
"description": "",
+ "sensitive": false,
"value": "value1"
}
]
@@ -453,6 +464,7 @@ TEST_CASE("Parameters can be used in nested process
groups") {
{
"name": "batch_size",
"description": "",
+ "sensitive": false,
"value": "12"
}
]
@@ -465,6 +477,7 @@ TEST_CASE("Parameters can be used in nested process
groups") {
{
"name": "file_size",
"description": "",
+ "sensitive": false,
"value": "10 B"
}
]
@@ -546,6 +559,7 @@ TEST_CASE("Subprocessgroups cannot inherit parameters from
parent processgroup")
{
"name": "batch_size",
"description": "",
+ "sensitive": false,
"value": "12"
}
]
@@ -558,6 +572,7 @@ TEST_CASE("Subprocessgroups cannot inherit parameters from
parent processgroup")
{
"name": "file_size",
"description": "",
+ "sensitive": false,
"value": "10 B"
}
]
@@ -684,11 +699,13 @@ TEST_CASE("Property value sequences can use parameters") {
{
"name": "first_value",
"description": "",
+ "sensitive": false,
"value": "value1"
},
{
"name": "second_value",
"description": "",
+ "sensitive": false,
"value": "value2"
}
]
@@ -743,11 +760,13 @@ TEST_CASE("Dynamic properties can use parameters") {
{
"name": "first_value",
"description": "",
+ "sensitive": false,
"value": "value1"
},
{
"name": "second_value",
"description": "",
+ "sensitive": false,
"value": "value2"
}
]
@@ -789,4 +808,118 @@ TEST_CASE("Dynamic properties can use parameters") {
CHECK(value == "value1");
}
+TEST_CASE("Test sensitive parameters in sensitive properties") {
+ ConfigurationTestController test_controller;
+ auto context = test_controller.getContext();
+ auto encrypted_parameter_value =
minifi::utils::crypto::property_encryption::encrypt("value1",
*context.sensitive_values_encryptor);
+ auto encrypted_sensitive_property_value =
minifi::utils::crypto::property_encryption::encrypt("#{my_value}",
*context.sensitive_values_encryptor);
+ core::flow::AdaptiveConfiguration config(context);
+
+ static const std::string CONFIG_JSON =
+ fmt::format(R"(
+{{
+ "parameterContexts": [
+ {{
+ "identifier": "721e10b7-8e00-3188-9a27-476cca376978",
+ "name": "my-context",
+ "description": "my parameter context",
+ "parameters": [
+ {{
+ "name": "my_value",
+ "description": "",
+ "sensitive": true,
+ "value": "{}"
+ }}
+ ]
+ }}
+ ],
+ "rootGroup": {{
+ "name": "MiNiFi Flow",
+ "processors": [{{
+ "identifier": "00000000-0000-0000-0000-000000000001",
+ "name": "MyProcessor",
+ "type": "org.apache.nifi.processors.DummyFlowJsonProcessor",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "schedulingPeriod": "3 sec",
+ "properties": {{
+ "Sensitive Property": "{}"
+ }}
+ }}],
+ "parameterContextName": "my-context"
+ }}
+}})", encrypted_parameter_value, encrypted_sensitive_property_value);
+
+ std::unique_ptr<core::ProcessGroup> flow =
config.getRootFromPayload(CONFIG_JSON);
+ REQUIRE(flow);
+
+ auto* proc = flow->findProcessorByName("MyProcessor");
+ REQUIRE(proc);
+ REQUIRE(proc->getProperty("Sensitive Property") == "value1");
+}
+
+TEST_CASE("Test sensitive parameters in sensitive property value sequence") {
+ ConfigurationTestController test_controller;
+ auto context = test_controller.getContext();
+ auto encrypted_parameter_value_1 =
minifi::utils::crypto::property_encryption::encrypt("value1",
*context.sensitive_values_encryptor);
+ auto encrypted_parameter_value_2 =
minifi::utils::crypto::property_encryption::encrypt("value2",
*context.sensitive_values_encryptor);
+ auto encrypted_sensitive_property_value_1 =
minifi::utils::crypto::property_encryption::encrypt("#{first_value}",
*context.sensitive_values_encryptor);
+ auto encrypted_sensitive_property_value_2 =
minifi::utils::crypto::property_encryption::encrypt("#{second_value}",
*context.sensitive_values_encryptor);
+ core::flow::AdaptiveConfiguration config(context);
+
+ static const std::string CONFIG_JSON =
+ fmt::format(R"(
+{{
+ "parameterContexts": [
+ {{
+ "identifier": "721e10b7-8e00-3188-9a27-476cca376978",
+ "name": "my-context",
+ "description": "my parameter context",
+ "parameters": [
+ {{
+ "name": "first_value",
+ "description": "",
+ "sensitive": true,
+ "value": "{}"
+ }},
+ {{
+ "name": "second_value",
+ "description": "",
+ "sensitive": true,
+ "value": "{}"
+ }}
+ ]
+ }}
+ ],
+ "rootGroup": {{
+ "name": "MiNiFi Flow",
+ "processors": [{{
+ "identifier": "00000000-0000-0000-0000-000000000001",
+ "name": "MyProcessor",
+ "type": "org.apache.nifi.processors.DummyFlowJsonProcessor",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "schedulingPeriod": "3 sec",
+ "properties": {{
+ "Sensitive Property": [
+ {{"value": "{}"}},
+ {{"value": "{}"}}
+ ]
+ }}
+ }}],
+ "parameterContextName": "my-context"
+ }}
+}})", encrypted_parameter_value_1, encrypted_parameter_value_2,
encrypted_sensitive_property_value_1, encrypted_sensitive_property_value_2);
+
+ std::unique_ptr<core::ProcessGroup> flow =
config.getRootFromPayload(CONFIG_JSON);
+ REQUIRE(flow);
+
+ auto* proc = flow->findProcessorByName("MyProcessor");
+ REQUIRE(proc);
+ core::Property property("Sensitive Property", "");
+ proc->getProperty("Sensitive Property", property);
+ auto values = property.getValues();
+ REQUIRE(values.size() == 2);
+ CHECK(values[0] == "value1");
+ CHECK(values[1] == "value2");
+}
+
} // namespace org::apache::nifi::minifi::test
diff --git
a/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp
b/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp
index 4a519af86..caac798f8 100644
--- a/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp
+++ b/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp
@@ -30,6 +30,7 @@
#include "unit/ConfigurationTestController.h"
#include "unit/TestUtils.h"
#include "core/Resource.h"
+#include "utils/crypto/property_encryption/PropertyEncryptionUtils.h"
using namespace std::literals::chrono_literals;
@@ -1092,9 +1093,11 @@ Parameter Contexts:
Parameters:
- name: lookup.frequency
description: ''
+ sensitive: false
value: 12 min
- name: batch_size
description: ''
+ sensitive: false
value: 12
Processors:
- id: b0c04f28-0158-1000-0000-000000000000
@@ -1144,6 +1147,7 @@ Parameter Contexts:
Parameters:
- name: lookup.frequency
description: ''
+ sensitive: false
value: 12 min
- id: 123e10b7-8e00-3188-9a27-476cca376978
name: other-context
@@ -1151,6 +1155,7 @@ Parameter Contexts:
Parameters:
- name: batch_size
description: ''
+ sensitive: false
value: 1
Processors:
- id: b0c04f28-0158-1000-0000-000000000000
@@ -1195,6 +1200,7 @@ Parameter Contexts:
Parameters:
- name: lookup.frequency
description: ''
+ sensitive: false
value: 12 min
- id: 123e10b7-8e00-3188-9a27-476cca376978
name: my-context
@@ -1202,6 +1208,7 @@ Parameter Contexts:
Parameters:
- name: batch_size
description: ''
+ sensitive: false
value: 1
Processors: []
Controller Services: []
@@ -1233,9 +1240,11 @@ Parameter Contexts:
Parameters:
- name: lookup.frequency
description: ''
+ sensitive: false
value: 12 min
- name: lookup.frequency
description: ''
+ sensitive: false
value: 1 min
Processors: []
Controller Services: []
@@ -1292,6 +1301,7 @@ Parameter Contexts:
Parameters:
- name: my_value
description: ''
+ sensitive: false
value: value1
Processors:
- id: b0c04f28-0158-1000-0000-000000000000
@@ -1333,6 +1343,7 @@ Parameter Contexts:
Parameters:
- name: my_value
description: ''
+ sensitive: false
value: value1
Processors:
- id: b0c04f28-0158-1000-0000-000000000000
@@ -1376,6 +1387,7 @@ Parameter Contexts:
Parameters:
- name: lookup.frequency
description: ''
+ sensitive: false
value: 12 min
- id: 123e10b7-8e00-3188-9a27-476cca376456
name: sub-context
@@ -1383,6 +1395,7 @@ Parameter Contexts:
Parameters:
- name: batch_size
description: ''
+ sensitive: false
value: 12
Processors:
- id: b0c04f28-0158-1000-0000-000000000000
@@ -1452,6 +1465,7 @@ Parameter Contexts:
Parameters:
- name: lookup.frequency
description: ''
+ sensitive: false
value: 12 min
- id: 123e10b7-8e00-3188-9a27-476cca376456
name: sub-context
@@ -1459,6 +1473,7 @@ Parameter Contexts:
Parameters:
- name: batch_size
description: ''
+ sensitive: false
value: 12
Processors:
- id: b0c04f28-0158-1000-0000-000000000000
@@ -1569,9 +1584,11 @@ Parameter Contexts:
Parameters:
- name: first_value
description: ''
+ sensitive: false
value: value1
- name: second_value
description: ''
+ sensitive: false
value: value2
Processors:
- id: b0c04f28-0158-1000-0000-000000000000
@@ -1616,9 +1633,11 @@ Parameter Contexts:
Parameters:
- name: first_value
description: ''
+ sensitive: false
value: value1
- name: second_value
description: ''
+ sensitive: false
value: value2
Processors:
- id: b0c04f28-0158-1000-0000-000000000000
@@ -1652,4 +1671,101 @@ Parameter Context Name: my-context
CHECK(value == "value1");
}
+TEST_CASE("Test sensitive parameters in sensitive properties",
"[YamlConfiguration]") {
+ ConfigurationTestController test_controller;
+ auto context = test_controller.getContext();
+ auto encrypted_parameter_value =
minifi::utils::crypto::property_encryption::encrypt("value1",
*context.sensitive_values_encryptor);
+ auto encrypted_sensitive_property_value =
minifi::utils::crypto::property_encryption::encrypt("#{my_value}",
*context.sensitive_values_encryptor);
+ core::YamlConfiguration yaml_config(context);
+
+ static const std::string TEST_CONFIG_YAML =
+ fmt::format(R"(
+MiNiFi Config Version: 3
+Flow Controller:
+ name: flowconfig
+Parameter Contexts:
+ - id: 721e10b7-8e00-3188-9a27-476cca376978
+ name: my-context
+ description: my parameter context
+ Parameters:
+ - name: my_value
+ description: ''
+ sensitive: true
+ value: {}
+Processors:
+- id: b0c04f28-0158-1000-0000-000000000000
+ name: DummyFlowYamlProcessor
+ class: org.apache.nifi.processors.DummyFlowYamlProcessor
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 1 sec
+ auto-terminated relationships list: [success]
+ Properties:
+ Simple Property: simple
+ Sensitive Property: {}
+Parameter Context Name: my-context
+ )", encrypted_parameter_value, encrypted_sensitive_property_value);
+
+ std::unique_ptr<core::ProcessGroup> flow =
yaml_config.getRootFromPayload(TEST_CONFIG_YAML);
+ REQUIRE(flow);
+ auto* proc = flow->findProcessorByName("DummyFlowYamlProcessor");
+ REQUIRE(proc);
+ REQUIRE(proc->getProperty("Simple Property") == "simple");
+ REQUIRE(proc->getProperty("Sensitive Property") == "value1");
+}
+
+TEST_CASE("Test sensitive parameters in sensitive property value sequence",
"[YamlConfiguration]") {
+ ConfigurationTestController test_controller;
+ auto context = test_controller.getContext();
+ auto encrypted_parameter_value_1 =
minifi::utils::crypto::property_encryption::encrypt("value1",
*context.sensitive_values_encryptor);
+ auto encrypted_parameter_value_2 =
minifi::utils::crypto::property_encryption::encrypt("value2",
*context.sensitive_values_encryptor);
+ auto encrypted_sensitive_property_value_1 =
minifi::utils::crypto::property_encryption::encrypt("#{my_value_1}",
*context.sensitive_values_encryptor);
+ auto encrypted_sensitive_property_value_2 =
minifi::utils::crypto::property_encryption::encrypt("#{my_value_2}",
*context.sensitive_values_encryptor);
+ core::YamlConfiguration yaml_config(context);
+
+ static const std::string TEST_CONFIG_YAML =
+ fmt::format(R"(
+MiNiFi Config Version: 3
+Flow Controller:
+ name: flowconfig
+Parameter Contexts:
+ - id: 721e10b7-8e00-3188-9a27-476cca376978
+ name: my-context
+ description: my parameter context
+ Parameters:
+ - name: my_value_1
+ description: ''
+ sensitive: true
+ value: {}
+ - name: my_value_2
+ description: ''
+ sensitive: true
+ value: {}
+Processors:
+- id: b0c04f28-0158-1000-0000-000000000000
+ name: DummyFlowYamlProcessor
+ class: org.apache.nifi.processors.DummyFlowYamlProcessor
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 1 sec
+ auto-terminated relationships list: [success]
+ Properties:
+ Simple Property: simple
+ Sensitive Property:
+ - value: {}
+ - value: {}
+Parameter Context Name: my-context
+ )", encrypted_parameter_value_1, encrypted_parameter_value_2,
encrypted_sensitive_property_value_1, encrypted_sensitive_property_value_2);
+
+ std::unique_ptr<core::ProcessGroup> flow =
yaml_config.getRootFromPayload(TEST_CONFIG_YAML);
+ REQUIRE(flow);
+ auto* proc = flow->findProcessorByName("DummyFlowYamlProcessor");
+ core::Property property("Sensitive Property", "");
+ proc->getProperty("Sensitive Property", property);
+ auto values = property.getValues();
+ REQUIRE(values.size() == 2);
+ CHECK(values[0] == "value1");
+ CHECK(values[1] == "value2");
+}
+
} // namespace org::apache::nifi::minifi::test
diff --git
a/extensions/standard-processors/tests/unit/YamlProcessGroupParserTests.cpp
b/extensions/standard-processors/tests/unit/YamlProcessGroupParserTests.cpp
index fb5d6df75..0147d30f7 100644
--- a/extensions/standard-processors/tests/unit/YamlProcessGroupParserTests.cpp
+++ b/extensions/standard-processors/tests/unit/YamlProcessGroupParserTests.cpp
@@ -27,7 +27,7 @@ static core::YamlConfiguration
config{core::ConfigurationContext{
.configuration = std::make_shared<minifi::Configure>(),
.path = "",
.filesystem = std::make_shared<utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
}};
TEST_CASE("Root process group is correctly parsed",
"[YamlProcessGroupParser]") {
diff --git a/libminifi/include/core/FlowConfiguration.h
b/libminifi/include/core/FlowConfiguration.h
index 0f4638665..57fb5b5a6 100644
--- a/libminifi/include/core/FlowConfiguration.h
+++ b/libminifi/include/core/FlowConfiguration.h
@@ -59,7 +59,7 @@ struct ConfigurationContext {
std::shared_ptr<Configure> configuration;
std::optional<std::filesystem::path> path{std::nullopt};
std::shared_ptr<utils::file::FileSystem>
filesystem{std::make_shared<utils::file::FileSystem>()};
- std::optional<utils::crypto::EncryptionProvider>
sensitive_properties_encryptor{std::nullopt};
+ std::optional<utils::crypto::EncryptionProvider>
sensitive_values_encryptor{std::nullopt};
};
enum class FlowSerializationType { Json, NifiJson, Yaml };
@@ -134,6 +134,10 @@ class FlowConfiguration : public CoreComponent {
utils::ChecksumCalculator& getChecksumCalculator() { return
checksum_calculator_; }
+ const std::unordered_map<std::string, std::unique_ptr<ParameterContext>>&
getParameterContexts() const {
+ return parameter_contexts_;
+ }
+
protected:
std::unordered_map<std::string, std::unique_ptr<ParameterContext>>
parameter_contexts_;
std::optional<std::filesystem::path> config_path_;
@@ -143,7 +147,7 @@ class FlowConfiguration : public CoreComponent {
std::shared_ptr<core::controller::StandardControllerServiceProvider>
service_provider_;
std::shared_ptr<state::response::FlowVersion> flow_version_;
std::shared_ptr<utils::file::FileSystem> filesystem_;
- utils::crypto::EncryptionProvider sensitive_properties_encryptor_;
+ utils::crypto::EncryptionProvider sensitive_values_encryptor_;
utils::ChecksumCalculator checksum_calculator_;
private:
diff --git a/libminifi/include/core/ParameterContext.h
b/libminifi/include/core/ParameterContext.h
index debf788be..fa0bb0938 100644
--- a/libminifi/include/core/ParameterContext.h
+++ b/libminifi/include/core/ParameterContext.h
@@ -34,6 +34,7 @@ class ParameterException : public std::runtime_error {
struct Parameter {
std::string name;
std::string description;
+ bool sensitive = false;
std::string value;
};
@@ -51,6 +52,9 @@ class ParameterContext : public CoreComponent {
void addParameter(const Parameter ¶meter);
std::optional<Parameter> getParameter(const std::string &name) const;
+ const std::unordered_map<std::string, Parameter>& getParameters() const {
+ return parameters_;
+ }
private:
std::string description_;
diff --git a/libminifi/include/core/ParameterTokenParser.h
b/libminifi/include/core/ParameterTokenParser.h
index b5ef47391..d4c3ab871 100644
--- a/libminifi/include/core/ParameterTokenParser.h
+++ b/libminifi/include/core/ParameterTokenParser.h
@@ -25,6 +25,7 @@
#include "ParameterContext.h"
#include "Exception.h"
+#include "utils/crypto/EncryptionProvider.h"
namespace org::apache::nifi::minifi::core {
@@ -114,15 +115,21 @@ class ParameterTokenParser {
InToken
};
- explicit ParameterTokenParser(std::string input) : input_(std::move(input)) {
+ explicit ParameterTokenParser(std::string input)
+ : input_(std::move(input)) {
parse();
}
+ virtual ~ParameterTokenParser() = default;
+
const std::vector<std::unique_ptr<ParameterToken>>& getTokens() const {
return tokens_;
}
- std::string replaceParameters(const std::optional<ParameterContext>&
parameter_context, bool is_sensitive) const;
+ std::string replaceParameters(const std::optional<ParameterContext>&
parameter_context) const;
+
+ protected:
+ virtual std::string getRawParameterValue(const Parameter& parameter) const =
0;
private:
void parse();
@@ -131,5 +138,27 @@ class ParameterTokenParser {
std::vector<std::unique_ptr<ParameterToken>> tokens_;
};
+class NonSensitiveParameterTokenParser : public ParameterTokenParser {
+ public:
+ using ParameterTokenParser::ParameterTokenParser;
+
+ protected:
+ std::string getRawParameterValue(const Parameter& parameter) const override;
+};
+
+class SensitiveParameterTokenParser : public ParameterTokenParser {
+ public:
+ SensitiveParameterTokenParser(std::string input,
utils::crypto::EncryptionProvider& sensitive_values_encryptor)
+ : ParameterTokenParser(std::move(input)),
+ sensitive_values_encryptor_(sensitive_values_encryptor) {
+ }
+
+ protected:
+ std::string getRawParameterValue(const Parameter& parameter) const override;
+
+ private:
+ utils::crypto::EncryptionProvider& sensitive_values_encryptor_;
+};
+
} // namespace org::apache::nifi::minifi::core
diff --git a/libminifi/include/core/flow/AdaptiveConfiguration.h
b/libminifi/include/core/flow/AdaptiveConfiguration.h
index 6487f5ef0..83238a653 100644
--- a/libminifi/include/core/flow/AdaptiveConfiguration.h
+++ b/libminifi/include/core/flow/AdaptiveConfiguration.h
@@ -35,7 +35,7 @@ class AdaptiveConfiguration : public StructuredConfiguration {
std::unique_ptr<core::ProcessGroup> getRootFromPayload(const std::string
&payload) override;
- void setSensitivePropertiesEncryptor(utils::crypto::EncryptionProvider
sensitive_properties_encryptor);
+ void setSensitivePropertiesEncryptor(utils::crypto::EncryptionProvider
sensitive_values_encryptor);
std::string serializeWithOverrides(const core::ProcessGroup& process_group,
const std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides)
const;
};
diff --git a/libminifi/include/core/flow/FlowSchema.h
b/libminifi/include/core/flow/FlowSchema.h
index 74b6993e0..6cbad427a 100644
--- a/libminifi/include/core/flow/FlowSchema.h
+++ b/libminifi/include/core/flow/FlowSchema.h
@@ -85,6 +85,7 @@ struct FlowSchema {
Keys description;
Keys value;
Keys parameter_context_name;
+ Keys sensitive;
static FlowSchema getDefault();
static FlowSchema getNiFiFlowJson();
diff --git a/libminifi/include/core/json/JsonFlowSerializer.h
b/libminifi/include/core/json/JsonFlowSerializer.h
index 9f3f6377d..b5042ed54 100644
--- a/libminifi/include/core/json/JsonFlowSerializer.h
+++ b/libminifi/include/core/json/JsonFlowSerializer.h
@@ -29,6 +29,12 @@ class JsonFlowSerializer : public core::flow::FlowSerializer
{
const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const override;
private:
+ void encryptSensitiveParameters(rapidjson::Value& flow_definition_json,
rapidjson::Document::AllocatorType& alloc, const core::flow::FlowSchema& schema,
+ const utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const;
+ void encryptSensitiveProcessorProperties(rapidjson::Value& root_group,
rapidjson::Document::AllocatorType& alloc, const core::ProcessGroup&
process_group,
+ const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const;
+ void encryptSensitiveControllerServiceProperties(rapidjson::Value&
root_group, rapidjson::Document::AllocatorType& alloc, const
core::ProcessGroup& process_group,
+ const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const;
void encryptSensitiveProperties(rapidjson::Value& property_jsons,
rapidjson::Document::AllocatorType& alloc,
const std::map<std::string, Property>& properties, const
utils::crypto::EncryptionProvider& encryption_provider,
const core::flow::Overrides& overrides) const;
diff --git a/libminifi/include/core/yaml/YamlFlowSerializer.h
b/libminifi/include/core/yaml/YamlFlowSerializer.h
index 19a16e3d1..240d112c5 100644
--- a/libminifi/include/core/yaml/YamlFlowSerializer.h
+++ b/libminifi/include/core/yaml/YamlFlowSerializer.h
@@ -29,6 +29,12 @@ class YamlFlowSerializer : public core::flow::FlowSerializer
{
const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const override;
private:
+ void encryptSensitiveParameters(YAML::Node& flow_definition_yaml, const
core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider&
encryption_provider,
+ const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const;
+ void encryptSensitiveProcessorProperties(YAML::Node& flow_definition_yaml,
const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema,
+ const utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const;
+ void encryptSensitiveControllerServiceProperties(YAML::Node&
flow_definition_yaml, const core::ProcessGroup& process_group, const
core::flow::FlowSchema& schema,
+ const utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const;
void encryptSensitiveProperties(YAML::Node property_yamls, const
std::map<std::string, Property>& properties, const
utils::crypto::EncryptionProvider& encryption_provider,
const core::flow::Overrides& overrides) const;
diff --git a/libminifi/src/core/FlowConfiguration.cpp
b/libminifi/src/core/FlowConfiguration.cpp
index f74acc494..fd46a064a 100644
--- a/libminifi/src/core/FlowConfiguration.cpp
+++ b/libminifi/src/core/FlowConfiguration.cpp
@@ -35,7 +35,7 @@ FlowConfiguration::FlowConfiguration(ConfigurationContext ctx)
configuration_(std::move(ctx.configuration)),
service_provider_(std::make_shared<core::controller::StandardControllerServiceProvider>(std::make_unique<core::controller::ControllerServiceNodeMap>(),
configuration_)),
filesystem_(std::move(ctx.filesystem)),
-
sensitive_properties_encryptor_(std::move(ctx.sensitive_properties_encryptor.value())),
+
sensitive_values_encryptor_(std::move(ctx.sensitive_values_encryptor.value())),
logger_(logging::LoggerFactory<FlowConfiguration>::getLogger()) {
std::string flowUrl;
std::string bucket_id = "default";
diff --git a/libminifi/src/core/ParameterTokenParser.cpp
b/libminifi/src/core/ParameterTokenParser.cpp
index 01fb191fb..d407807a3 100644
--- a/libminifi/src/core/ParameterTokenParser.cpp
+++ b/libminifi/src/core/ParameterTokenParser.cpp
@@ -19,6 +19,7 @@
#include <stdexcept>
#include "utils/RegexUtils.h"
+#include "utils/crypto/property_encryption/PropertyEncryptionUtils.h"
namespace org::apache::nifi::minifi::core {
@@ -68,7 +69,7 @@ void ParameterTokenParser::parse() {
}
}
-std::string ParameterTokenParser::replaceParameters(const
std::optional<ParameterContext>& parameter_context, bool is_sensitive) const {
+std::string ParameterTokenParser::replaceParameters(const
std::optional<ParameterContext>& parameter_context) const {
if (tokens_.empty()) {
return input_;
}
@@ -85,21 +86,31 @@ std::string ParameterTokenParser::replaceParameters(const
std::optional<Paramete
if (!parameter_context) {
throw ParameterException("Property references a parameter in its value,
but no parameter context was provided.");
}
-
gsl_Assert(token->getName().has_value());
auto parameter = parameter_context->getParameter(token->getName().value());
if (!parameter.has_value()) {
throw ParameterException("Parameter '" + token->getName().value() + "'
not found");
}
- if (is_sensitive) {
- throw ParameterException("Non-sensitive parameter '" + parameter->name +
"' cannot be referenced in a sensitive property");
- }
- result.append(std::string(token->getAdditionalHashmarks(), '#') +
parameter->value);
+ result.append(std::string(token->getAdditionalHashmarks(), '#') +
getRawParameterValue(*parameter));
last_end = token->getStart() + token->getSize();
}
result.append(input_.substr(last_end));
return result;
}
+std::string NonSensitiveParameterTokenParser::getRawParameterValue(const
Parameter& parameter) const {
+ if (parameter.sensitive) {
+ throw ParameterException("Sensitive parameter '" + parameter.name + "'
cannot be referenced in a non-sensitive property");
+ }
+ return parameter.value;
+}
+
+std::string SensitiveParameterTokenParser::getRawParameterValue(const
Parameter& parameter) const {
+ if (!parameter.sensitive) {
+ throw ParameterException("Non-sensitive parameter '" + parameter.name + "'
cannot be referenced in a sensitive property");
+ }
+ return utils::crypto::property_encryption::decrypt(parameter.value,
sensitive_values_encryptor_);
+}
+
} // namespace org::apache::nifi::minifi::core
diff --git a/libminifi/src/core/flow/AdaptiveConfiguration.cpp
b/libminifi/src/core/flow/AdaptiveConfiguration.cpp
index 85c590a38..3431e9223 100644
--- a/libminifi/src/core/flow/AdaptiveConfiguration.cpp
+++ b/libminifi/src/core/flow/AdaptiveConfiguration.cpp
@@ -71,13 +71,13 @@ std::unique_ptr<core::ProcessGroup>
AdaptiveConfiguration::getRootFromPayload(co
}
}
-void
AdaptiveConfiguration::setSensitivePropertiesEncryptor(utils::crypto::EncryptionProvider
sensitive_properties_encryptor) {
- sensitive_properties_encryptor_ = std::move(sensitive_properties_encryptor);
+void
AdaptiveConfiguration::setSensitivePropertiesEncryptor(utils::crypto::EncryptionProvider
sensitive_values_encryptor) {
+ sensitive_values_encryptor_ = std::move(sensitive_values_encryptor);
}
std::string AdaptiveConfiguration::serializeWithOverrides(const
core::ProcessGroup& process_group, const std::unordered_map<utils::Identifier,
core::flow::Overrides>& overrides) const {
gsl_Expects(flow_serializer_);
- return flow_serializer_->serialize(process_group, schema_,
sensitive_properties_encryptor_, overrides);
+ return flow_serializer_->serialize(process_group, schema_,
sensitive_values_encryptor_, overrides);
}
} // namespace org::apache::nifi::minifi::core::flow
diff --git a/libminifi/src/core/flow/FlowSchema.cpp
b/libminifi/src/core/flow/FlowSchema.cpp
index d2c3f4523..9d378800e 100644
--- a/libminifi/src/core/flow/FlowSchema.cpp
+++ b/libminifi/src/core/flow/FlowSchema.cpp
@@ -81,7 +81,8 @@ FlowSchema FlowSchema::getDefault() {
.parameters = {"Parameters"},
.description = {"description"},
.value = {"value"},
- .parameter_context_name = {"Parameter Context Name"}
+ .parameter_context_name = {"Parameter Context Name"},
+ .sensitive = {"sensitive"}
};
}
@@ -147,7 +148,8 @@ FlowSchema FlowSchema::getNiFiFlowJson() {
.parameters = {"parameters"},
.description = {"description"},
.value = {"value"},
- .parameter_context_name = {"parameterContextName"}
+ .parameter_context_name = {"parameterContextName"},
+ .sensitive = {"sensitive"}
};
}
diff --git a/libminifi/src/core/flow/StructuredConfiguration.cpp
b/libminifi/src/core/flow/StructuredConfiguration.cpp
index 760300308..719fb4917 100644
--- a/libminifi/src/core/flow/StructuredConfiguration.cpp
+++ b/libminifi/src/core/flow/StructuredConfiguration.cpp
@@ -165,10 +165,15 @@ void
StructuredConfiguration::parseParameterContexts(const Node& parameter_conte
for (const auto& parameter_node :
parameter_context_node[schema_.parameters]) {
checkRequiredField(parameter_node, schema_.name);
checkRequiredField(parameter_node, schema_.value);
+ checkRequiredField(parameter_node, schema_.sensitive);
auto parameter_name = parameter_node[schema_.name].getString().value();
auto parameter_value = parameter_node[schema_.value].getString().value();
+ auto sensitive = parameter_node[schema_.sensitive].getBool().value();
auto parameter_description = getOptionalField(parameter_node,
schema_.description, "");
- parameter_context->addParameter(Parameter{parameter_name,
parameter_description, parameter_value});
+ if (sensitive) {
+ parameter_value =
utils::crypto::property_encryption::decrypt(parameter_value,
sensitive_values_encryptor_);
+ }
+ parameter_context->addParameter(Parameter{parameter_name,
parameter_description, sensitive, parameter_value});
}
parameter_contexts_[name] = std::move(parameter_context);
}
@@ -670,13 +675,16 @@ void
StructuredConfiguration::parsePropertyValueSequence(const std::string& prop
if (nodeVal) {
Node propertiesNode = nodeVal["value"];
auto rawValueString = propertiesNode.getString().value();
+ std::unique_ptr<core::ParameterTokenParser> token_parser;
if (myProp.isSensitive()) {
- rawValueString =
utils::crypto::property_encryption::decrypt(rawValueString,
sensitive_properties_encryptor_);
+ rawValueString =
utils::crypto::property_encryption::decrypt(rawValueString,
sensitive_values_encryptor_);
+ token_parser =
std::make_unique<core::SensitiveParameterTokenParser>(rawValueString,
sensitive_values_encryptor_);
+ } else {
+ token_parser =
std::make_unique<core::NonSensitiveParameterTokenParser>(rawValueString);
}
try {
- core::ParameterTokenParser token_parser(rawValueString);
- rawValueString = token_parser.replaceParameters(parameter_context,
myProp.isSensitive());
+ rawValueString = token_parser->replaceParameters(parameter_context);
} catch (const ParameterException& e) {
logger_->log_error("Error while substituting parameters in property
'{}': {}", property_name, e.what());
throw;
@@ -720,13 +728,15 @@ PropertyValue
StructuredConfiguration::getValidatedProcessorPropertyForDefaultTy
coercedValue = property_value_node.getBool().value();
} else {
std::string property_value_string;
+ std::unique_ptr<core::ParameterTokenParser> token_parser;
if (property_from_processor.isSensitive()) {
- property_value_string =
utils::crypto::property_encryption::decrypt(property_value_node.getScalarAsString().value(),
sensitive_properties_encryptor_);
+ property_value_string =
utils::crypto::property_encryption::decrypt(property_value_node.getScalarAsString().value(),
sensitive_values_encryptor_);
+ token_parser =
std::make_unique<core::SensitiveParameterTokenParser>(property_value_string,
sensitive_values_encryptor_);
} else {
property_value_string =
property_value_node.getScalarAsString().value();
+ token_parser =
std::make_unique<core::NonSensitiveParameterTokenParser>(property_value_string);
}
- core::ParameterTokenParser token_parser(property_value_string);
- property_value_string =
token_parser.replaceParameters(parameter_context,
property_from_processor.isSensitive());
+ property_value_string =
token_parser->replaceParameters(parameter_context);
coercedValue = property_value_string;
}
return coercedValue;
@@ -993,7 +1003,7 @@ void StructuredConfiguration::addNewId(const std::string&
uuid) {
std::string StructuredConfiguration::serialize(const core::ProcessGroup&
process_group) {
gsl_Expects(flow_serializer_);
- return flow_serializer_->serialize(process_group, schema_,
sensitive_properties_encryptor_, {});
+ return flow_serializer_->serialize(process_group, schema_,
sensitive_values_encryptor_, {});
}
} // namespace org::apache::nifi::minifi::core::flow
diff --git a/libminifi/src/core/json/JsonFlowSerializer.cpp
b/libminifi/src/core/json/JsonFlowSerializer.cpp
index 58761ea89..cfbbb8334 100644
--- a/libminifi/src/core/json/JsonFlowSerializer.cpp
+++ b/libminifi/src/core/json/JsonFlowSerializer.cpp
@@ -68,18 +68,42 @@ void
JsonFlowSerializer::encryptSensitiveProperties(rapidjson::Value& property_j
}
}
-std::string JsonFlowSerializer::serialize(const core::ProcessGroup&
process_group, const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider,
- const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const {
- gsl_Expects(schema.root_group.size() == 1 && schema.identifier.size() == 1 &&
- schema.processors.size() == 1 && schema.processor_properties.size() == 1
&&
- schema.controller_services.size() == 1 &&
schema.controller_service_properties.size() == 1);
+void JsonFlowSerializer::encryptSensitiveParameters(rapidjson::Value&
flow_definition_json, rapidjson::Document::AllocatorType& alloc, const
core::flow::FlowSchema& schema,
+ const utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const {
+ if (!flow_definition_json.HasMember(schema.parameter_contexts[0])) {
+ return;
+ }
- rapidjson::Document doc;
- auto alloc = doc.GetAllocator();
- rapidjson::Value flow_definition_json;
- flow_definition_json.CopyFrom(flow_definition_json_, alloc);
- auto& root_group = getMember(flow_definition_json, schema.root_group[0]);
+ auto parameter_contexts = getMember(flow_definition_json,
schema.parameter_contexts[0]).GetArray();
+ for (auto& parameter_context : parameter_contexts) {
+ for (auto& parameter : getMember(parameter_context,
schema.parameters[0]).GetArray()) {
+ bool is_sensitive = getMember(parameter, schema.sensitive[0]).GetBool();
+ if (!is_sensitive) {
+ continue;
+ }
+ const std::string parameter_context_id_str{getMember(parameter_context,
schema.identifier[0]).GetString(), getMember(parameter_context,
schema.identifier[0]).GetStringLength()};
+ const auto parameter_context_id =
utils::Identifier::parse(parameter_context_id_str);
+ if (!parameter_context_id) {
+ logger_->log_warn("Invalid parameter context ID found in the flow
definition: {}", parameter_context_id_str);
+ continue;
+ }
+
+ std::string parameter_value{getMember(parameter,
schema.value[0]).GetString(), getMember(parameter,
schema.value[0]).GetStringLength()};
+ if (overrides.contains(*parameter_context_id)) {
+ const auto& override_values = overrides.at(*parameter_context_id);
+ const auto parameter_name = getMember(parameter,
schema.name[0]).GetString();
+ if (auto parameter_override_value =
override_values.get(parameter_name)) {
+ parameter_value = *parameter_override_value;
+ }
+ }
+ parameter_value =
utils::crypto::property_encryption::encrypt(parameter_value,
encryption_provider);
+ getMember(parameter, schema.value[0]).SetString(parameter_value.c_str(),
parameter_value.size(), alloc);
+ }
+ }
+}
+void JsonFlowSerializer::encryptSensitiveProcessorProperties(rapidjson::Value&
root_group, rapidjson::Document::AllocatorType& alloc, const
core::ProcessGroup& process_group,
+ const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const {
auto processors = getMember(root_group, schema.processors[0]).GetArray();
for (auto &processor_json : processors) {
const std::string processor_id_str{getMember(processor_json,
schema.identifier[0]).GetString(), getMember(processor_json,
schema.identifier[0]).GetStringLength()};
@@ -97,7 +121,10 @@ std::string JsonFlowSerializer::serialize(const
core::ProcessGroup& process_grou
encryptSensitiveProperties(getMember(processor_json,
schema.processor_properties[0]), alloc, processor->getProperties(),
encryption_provider,
processor_overrides);
}
+}
+void
JsonFlowSerializer::encryptSensitiveControllerServiceProperties(rapidjson::Value&
root_group, rapidjson::Document::AllocatorType& alloc, const
core::ProcessGroup& process_group,
+ const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const {
auto controller_services = getMember(root_group,
schema.controller_services[0]).GetArray();
for (auto &controller_service_json : controller_services) {
const std::string
controller_service_id_str{getMember(controller_service_json,
schema.identifier[0]).GetString(), getMember(controller_service_json,
schema.identifier[0]).GetStringLength()};
@@ -120,6 +147,23 @@ std::string JsonFlowSerializer::serialize(const
core::ProcessGroup& process_grou
encryptSensitiveProperties(getMember(controller_service_json,
schema.controller_service_properties[0]), alloc,
controller_service->getProperties(), encryption_provider,
controller_service_overrides);
}
+}
+
+std::string JsonFlowSerializer::serialize(const core::ProcessGroup&
process_group, const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider,
+ const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const {
+ gsl_Expects(schema.root_group.size() == 1 && schema.identifier.size() == 1 &&
+ schema.processors.size() == 1 && schema.processor_properties.size() == 1
&&
+ schema.controller_services.size() == 1 &&
schema.controller_service_properties.size() == 1);
+
+ rapidjson::Document doc;
+ auto alloc = doc.GetAllocator();
+ rapidjson::Value flow_definition_json;
+ flow_definition_json.CopyFrom(flow_definition_json_, alloc);
+ auto& root_group = getMember(flow_definition_json, schema.root_group[0]);
+
+ encryptSensitiveParameters(flow_definition_json, alloc, schema,
encryption_provider, overrides);
+ encryptSensitiveProcessorProperties(root_group, alloc, process_group,
schema, encryption_provider, overrides);
+ encryptSensitiveControllerServiceProperties(root_group, alloc,
process_group, schema, encryption_provider, overrides);
rapidjson::StringBuffer buffer;
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer{buffer};
diff --git a/libminifi/src/core/yaml/YamlFlowSerializer.cpp
b/libminifi/src/core/yaml/YamlFlowSerializer.cpp
index 59b48b876..7ac4ceebd 100644
--- a/libminifi/src/core/yaml/YamlFlowSerializer.cpp
+++ b/libminifi/src/core/yaml/YamlFlowSerializer.cpp
@@ -56,14 +56,37 @@ void
YamlFlowSerializer::encryptSensitiveProperties(YAML::Node property_yamls, c
}
}
-std::string YamlFlowSerializer::serialize(const core::ProcessGroup&
process_group, const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider,
+void YamlFlowSerializer::encryptSensitiveParameters(YAML::Node&
flow_definition_yaml, const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider,
const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const {
- gsl_Expects(schema.identifier.size() == 1 &&
- schema.processors.size() == 1 && schema.processor_properties.size() == 1
&&
- schema.controller_services.size() == 1 &&
schema.controller_service_properties.size() == 1);
-
- auto flow_definition_yaml = YAML::Clone(flow_definition_yaml_);
+ for (auto parameter_context :
flow_definition_yaml[schema.parameter_contexts[0]]) {
+ for (auto parameter : parameter_context[schema.parameters[0]]) {
+ bool is_sensitive = false;
+ std::istringstream is(parameter[schema.sensitive[0]].Scalar());
+ is >> std::boolalpha >> is_sensitive;
+ if (!is_sensitive) {
+ continue;
+ }
+ const auto parameter_context_id_str =
parameter_context[schema.identifier[0]].Scalar();
+ const auto parameter_context_id =
utils::Identifier::parse(parameter_context_id_str);
+ if (!parameter_context_id) {
+ logger_->log_warn("Invalid parameter context ID found in the flow
definition: {}", parameter_context_id_str);
+ continue;
+ }
+ auto parameter_value = parameter[schema.value[0]].Scalar();
+ if (overrides.contains(*parameter_context_id)) {
+ const auto& override_values = overrides.at(*parameter_context_id);
+ const auto parameter_name = parameter[schema.name[0]].Scalar();
+ if (auto parameter_override_value =
override_values.get(parameter_name)) {
+ parameter_value = *parameter_override_value;
+ }
+ }
+ parameter[schema.value[0]] =
utils::crypto::property_encryption::encrypt(parameter_value,
encryption_provider);
+ }
+ }
+}
+void YamlFlowSerializer::encryptSensitiveProcessorProperties(YAML::Node&
flow_definition_yaml, const core::ProcessGroup& process_group, const
core::flow::FlowSchema& schema,
+ const utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const {
for (auto processor_yaml : flow_definition_yaml[schema.processors[0]]) {
const auto processor_id =
utils::Identifier::parse(processor_yaml[schema.identifier[0]].Scalar());
if (!processor_id) {
@@ -78,7 +101,10 @@ std::string YamlFlowSerializer::serialize(const
core::ProcessGroup& process_grou
const auto processor_overrides = overrides.contains(*processor_id) ?
overrides.at(*processor_id) : core::flow::Overrides{};
encryptSensitiveProperties(processor_yaml[schema.processor_properties[0]],
processor->getProperties(), encryption_provider, processor_overrides);
}
+}
+void
YamlFlowSerializer::encryptSensitiveControllerServiceProperties(YAML::Node&
flow_definition_yaml, const core::ProcessGroup& process_group, const
core::flow::FlowSchema& schema,
+ const utils::crypto::EncryptionProvider& encryption_provider, const
std::unordered_map<utils::Identifier, core::flow::Overrides>& overrides) const {
for (auto controller_service_yaml :
flow_definition_yaml[schema.controller_services[0]]) {
const auto controller_service_id_str =
controller_service_yaml[schema.identifier[0]].Scalar();
const auto controller_service_id =
utils::Identifier::parse(controller_service_id_str);
@@ -99,6 +125,19 @@ std::string YamlFlowSerializer::serialize(const
core::ProcessGroup& process_grou
const auto controller_service_overrides =
overrides.contains(*controller_service_id) ?
overrides.at(*controller_service_id) : core::flow::Overrides{};
encryptSensitiveProperties(controller_service_yaml[schema.controller_service_properties[0]],
controller_service->getProperties(), encryption_provider,
controller_service_overrides);
}
+}
+
+std::string YamlFlowSerializer::serialize(const core::ProcessGroup&
process_group, const core::flow::FlowSchema& schema, const
utils::crypto::EncryptionProvider& encryption_provider,
+ const std::unordered_map<utils::Identifier, core::flow::Overrides>&
overrides) const {
+ gsl_Expects(schema.identifier.size() == 1 &&
+ schema.processors.size() == 1 && schema.processor_properties.size() == 1
&&
+ schema.controller_services.size() == 1 &&
schema.controller_service_properties.size() == 1);
+
+ auto flow_definition_yaml = YAML::Clone(flow_definition_yaml_);
+
+ encryptSensitiveParameters(flow_definition_yaml, schema,
encryption_provider, overrides);
+ encryptSensitiveProcessorProperties(flow_definition_yaml, process_group,
schema, encryption_provider, overrides);
+ encryptSensitiveControllerServiceProperties(flow_definition_yaml,
process_group, schema, encryption_provider, overrides);
return YAML::Dump(flow_definition_yaml) + '\n';
}
diff --git a/libminifi/test/integration/C2PauseResumeTest.cpp
b/libminifi/test/integration/C2PauseResumeTest.cpp
index 7e819403f..e79c74dcd 100644
--- a/libminifi/test/integration/C2PauseResumeTest.cpp
+++ b/libminifi/test/integration/C2PauseResumeTest.cpp
@@ -134,7 +134,7 @@ TEST_CASE("C2PauseResumeTest", "[c2test]") {
.configuration = configuration,
.path = test_file_path,
.filesystem = std::make_shared<minifi::utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{minifi::utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{minifi::utils::crypto::XSalsa20Cipher::generateKey()}}
});
std::vector<std::shared_ptr<core::RepositoryMetricsSource>>
repo_metric_sources{test_repo, test_flow_repo, content_repo};
auto metrics_publisher_store =
std::make_unique<minifi::state::MetricsPublisherStore>(configuration,
repo_metric_sources, yaml_ptr);
@@ -147,7 +147,7 @@ TEST_CASE("C2PauseResumeTest", "[c2test]") {
.configuration = configuration,
.path = test_file_path,
.filesystem = std::make_shared<minifi::utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{minifi::utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{minifi::utils::crypto::XSalsa20Cipher::generateKey()}}
});
auto root = yaml_config.getRoot();
const auto proc = root->findProcessorByName("invoke");
diff --git a/libminifi/test/integration/ControllerServiceIntegrationTests.cpp
b/libminifi/test/integration/ControllerServiceIntegrationTests.cpp
index 3073fc46e..40f3a7eaf 100644
--- a/libminifi/test/integration/ControllerServiceIntegrationTests.cpp
+++ b/libminifi/test/integration/ControllerServiceIntegrationTests.cpp
@@ -73,7 +73,7 @@ TEST_CASE("ControllerServiceIntegrationTests",
"[controller]") {
.configuration = configuration,
.path = test_file_path,
.filesystem = std::make_shared<minifi::utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}}
+ .sensitive_values_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}}
});
const auto controller = std::make_shared<minifi::FlowController>(test_repo,
test_flow_repo, configuration, std::move(yaml_ptr), content_repo);
@@ -85,7 +85,7 @@ TEST_CASE("ControllerServiceIntegrationTests",
"[controller]") {
.configuration = configuration,
.path = test_file_path,
.filesystem = std::make_shared<minifi::utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}}
+ .sensitive_values_encryptor =
minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}}
});
auto pg = yaml_config.getRoot();
diff --git a/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp
b/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp
index 2a4bbe308..ddd425eb1 100644
--- a/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp
+++ b/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp
@@ -104,7 +104,7 @@ class PersistentStateStorageTestsFixture {
.configuration = configuration,
.path = config_yaml,
.filesystem = std::make_shared<utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
});
process_group = yaml_config->getRoot();
auto* persistable_key_value_store_service_node =
process_group->findControllerService("testcontroller");
diff --git a/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp
b/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp
index a641dcb89..43fb814e7 100644
--- a/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp
+++ b/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp
@@ -100,7 +100,7 @@ class VolatileMapStateStorageTestFixture {
.configuration = configuration,
.path = config_yaml,
.filesystem = std::make_shared<utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
});
std::unique_ptr<core::ProcessGroup> process_group;
diff --git a/libminifi/test/libtest/integration/IntegrationBase.cpp
b/libminifi/test/libtest/integration/IntegrationBase.cpp
index 75d357706..36efc0cee 100644
--- a/libminifi/test/libtest/integration/IntegrationBase.cpp
+++ b/libminifi/test/libtest/integration/IntegrationBase.cpp
@@ -78,7 +78,7 @@ void IntegrationBase::run(const
std::optional<std::filesystem::path>& test_file_
filesystem = std::make_shared<minifi::utils::file::FileSystem>();
}
- std::optional<minifi::utils::crypto::EncryptionProvider>
sensitive_properties_encryptor = [&]() {
+ std::optional<minifi::utils::crypto::EncryptionProvider>
sensitive_values_encryptor = [&]() {
if (home_path) {
return
minifi::utils::crypto::EncryptionProvider::createSensitivePropertiesEncryptor(*home_path);
} else {
@@ -97,7 +97,7 @@ void IntegrationBase::run(const
std::optional<std::filesystem::path>& test_file_
.configuration = configuration,
.path = test_file_location,
.filesystem = filesystem,
- .sensitive_properties_encryptor = sensitive_properties_encryptor
+ .sensitive_values_encryptor = sensitive_values_encryptor
}, nifi_configuration_class_name);
auto controller_service_provider =
flow_config->getControllerServiceProvider();
diff --git a/libminifi/test/libtest/unit/ConfigurationTestController.h
b/libminifi/test/libtest/unit/ConfigurationTestController.h
index d726cc200..67217b2c7 100644
--- a/libminifi/test/libtest/unit/ConfigurationTestController.h
+++ b/libminifi/test/libtest/unit/ConfigurationTestController.h
@@ -45,12 +45,14 @@ class ConfigurationTestController : public TestController {
.content_repo = content_repo_,
.configuration = configuration_,
.path = "",
- .filesystem = std::make_shared<utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
+ .filesystem = filesystem_,
+ .sensitive_values_encryptor = sensitive_values_encryptor_
};
}
std::shared_ptr<core::Repository> flow_file_repo_;
std::shared_ptr<minifi::Configure> configuration_;
std::shared_ptr<core::ContentRepository> content_repo_;
+ std::shared_ptr<utils::file::FileSystem>
filesystem_{std::make_shared<utils::file::FileSystem>()};
+ utils::crypto::EncryptionProvider sensitive_values_encryptor_ =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}};
};
diff --git a/libminifi/test/libtest/unit/TestControllerWithFlow.cpp
b/libminifi/test/libtest/unit/TestControllerWithFlow.cpp
index 10b68187e..d0fa38e30 100644
--- a/libminifi/test/libtest/unit/TestControllerWithFlow.cpp
+++ b/libminifi/test/libtest/unit/TestControllerWithFlow.cpp
@@ -63,7 +63,7 @@ void TestControllerWithFlow::setupFlow() {
.configuration = configuration_,
.path = yaml_path_.string(),
.filesystem = std::make_shared<utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
});
auto root = flow->getRoot();
root_ = root.get();
diff --git a/libminifi/test/persistence-tests/PersistenceTests.cpp
b/libminifi/test/persistence-tests/PersistenceTests.cpp
index 50ab878b5..7560982ac 100644
--- a/libminifi/test/persistence-tests/PersistenceTests.cpp
+++ b/libminifi/test/persistence-tests/PersistenceTests.cpp
@@ -188,7 +188,7 @@ TEST_CASE("Processors Can Store FlowFiles", "[TestP1]") {
.configuration = config,
.path = "",
.filesystem = std::make_shared<utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
});
auto flowController = std::make_shared<minifi::FlowController>(prov_repo,
ff_repository, config, std::move(flowConfig), content_repo);
@@ -315,7 +315,7 @@ TEST_CASE("Persisted flowFiles are updated on
modification", "[TestP1]") {
.configuration = config,
.path = "",
.filesystem = std::make_shared<utils::file::FileSystem>(),
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
+ .sensitive_values_encryptor =
utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}}
});
auto flowController = std::make_shared<minifi::FlowController>(prov_repo,
ff_repository, config, std::move(flowConfig), content_repo);
diff --git a/libminifi/test/unit/JsonFlowSerializerTests.cpp
b/libminifi/test/unit/JsonFlowSerializerTests.cpp
index 917253874..6cd1eaaa1 100644
--- a/libminifi/test/unit/JsonFlowSerializerTests.cpp
+++ b/libminifi/test/unit/JsonFlowSerializerTests.cpp
@@ -35,6 +35,21 @@ constexpr std::string_view config_json_with_default_schema =
R"({
"Flow Controller": {
"name": "MiNiFi Flow"
},
+ "Parameter Contexts": [
+ {
+ "id": "721e10b7-8e00-3188-9a27-476cca376978",
+ "name": "my-context",
+ "description": "my parameter context",
+ "Parameters": [
+ {
+ "name": "secret_parameter",
+ "description": "",
+ "sensitive": true,
+ "value": "param_value_1"
+ }
+ ]
+ }
+],
"Processors": [
{
"name": "Generate random data",
@@ -113,7 +128,21 @@ constexpr std::string_view
config_json_with_nifi_schema_part_1 = R"({
},
"maxTimerDrivenThreadCount": 1,
"maxEventDrivenThreadCount": 1,
- "parameterContexts": [],
+ "parameterContexts": [
+ {
+ "identifier": "721e10b7-8e00-3188-9a27-476cca376978",
+ "name": "my-context",
+ "description": "my parameter context",
+ "parameters": [
+ {
+ "name": "secret_parameter",
+ "description": "",
+ "sensitive": true,
+ "value": "param_value_1"
+ }
+ ]
+ }
+ ],
"rootGroup": {
"identifier": "8b4d66dc-9085-4722-b35b-3492f363baa3",
"instanceIdentifier": "0fab8422-3fbe-45e0-bd3a-324f5cb0592b",
@@ -547,17 +576,20 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive
properties") {
const auto processor_id =
utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value();
const auto controller_service_id =
utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value();
+ const auto parameter_id =
utils::Identifier::parse("721e10b7-8e00-3188-9a27-476cca376978").value();
const auto [overrides, expected_results] = GENERATE_REF(
std::make_tuple(OverridesMap{},
- std::array{"very_secure_password", "very_secure_passphrase"}),
+ std::array{"very_secure_password", "very_secure_passphrase",
"param_value_1"}),
std::make_tuple(OverridesMap{{processor_id,
core::flow::Overrides{}.add("invokehttp-proxy-password", "password123")}},
- std::array{"password123", "very_secure_passphrase"}),
+ std::array{"password123", "very_secure_passphrase",
"param_value_1"}),
std::make_tuple(OverridesMap{{controller_service_id,
core::flow::Overrides{}.add("Passphrase", "speak friend and enter")}},
- std::array{"very_secure_password", "speak friend and enter"}),
+ std::array{"very_secure_password", "speak friend and enter",
"param_value_1"}),
std::make_tuple(OverridesMap{{processor_id,
core::flow::Overrides{}.add("invokehttp-proxy-password", "password123")},
{controller_service_id,
core::flow::Overrides{}.add("Passphrase", "speak friend and enter")}},
- std::array{"password123", "speak friend and enter"}));
+ std::array{"password123", "speak friend and enter",
"param_value_1"}),
+ std::make_tuple(OverridesMap{{parameter_id,
core::flow::Overrides{}.add("secret_parameter", "param_value_2")}},
+ std::array{"very_secure_password", "very_secure_passphrase",
"param_value_2"}));
std::string config_json_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
@@ -580,6 +612,16 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive
properties") {
std::string encrypted_value = match_results[1];
CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == expected_results[1]);
}
+
+ {
+ std::regex regex{R"_("value": "(.*)")_"};
+ std::smatch match_results;
+ CHECK(std::regex_search(config_json_encrypted, match_results, regex));
+
+ REQUIRE(match_results.size() == 2);
+ std::string encrypted_value = match_results[1];
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == expected_results[2]);
+ }
}
TEST_CASE("JsonFlowSerializer with an override can add a new property to the
flow config file") {
@@ -640,7 +682,7 @@ TEST_CASE("JsonFlowSerializer with an override can add a
new property to the flo
TEST_CASE("The encrypted flow configuration can be decrypted with the correct
key") {
ConfigurationTestController test_controller;
auto configuration_context = test_controller.getContext();
- configuration_context.sensitive_properties_encryptor = encryption_provider;
+ configuration_context.sensitive_values_encryptor = encryption_provider;
core::flow::AdaptiveConfiguration
json_configuration_before{configuration_context};
const auto schema = core::flow::FlowSchema::getNiFiFlowJson();
@@ -677,12 +719,15 @@ TEST_CASE("The encrypted flow configuration can be
decrypted with the correct ke
REQUIRE(controller_service_after);
CHECK(controller_service_before->getProperties().at("CA
Certificate").getValue() == controller_service_after->getProperties().at("CA
Certificate").getValue());
CHECK(controller_service_before->getProperties().at("Passphrase").getValue()
== controller_service_after->getProperties().at("Passphrase").getValue());
+
+ const auto& param_contexts = json_configuration_after.getParameterContexts();
+
CHECK(param_contexts.at("my-context")->getParameter("secret_parameter")->value
== "param_value_1");
}
TEST_CASE("The encrypted flow configuration cannot be decrypted with an
incorrect key") {
ConfigurationTestController test_controller;
auto configuration_context = test_controller.getContext();
- configuration_context.sensitive_properties_encryptor = encryption_provider;
+ configuration_context.sensitive_values_encryptor = encryption_provider;
core::flow::AdaptiveConfiguration
json_configuration_before{configuration_context};
const auto schema = core::flow::FlowSchema::getNiFiFlowJson();
@@ -697,7 +742,7 @@ TEST_CASE("The encrypted flow configuration cannot be
decrypted with an incorrec
std::string config_json_encrypted =
flow_serializer.serialize(*process_group_before, schema, encryption_provider,
{});
const utils::crypto::Bytes different_secret_key =
utils::string::from_hex("ea55b7d0edc22280c9547e4d89712b3fae74f96d82f240a004fb9fbd0640eec7");
- configuration_context.sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{different_secret_key};
+ configuration_context.sensitive_values_encryptor =
utils::crypto::EncryptionProvider{different_secret_key};
core::flow::AdaptiveConfiguration
json_configuration_after{configuration_context};
REQUIRE_THROWS_AS(json_configuration_after.getRootFromPayload(config_json_encrypted),
utils::crypto::EncryptionError);
diff --git a/libminifi/test/unit/ParameterTokenParserTest.cpp
b/libminifi/test/unit/ParameterTokenParserTest.cpp
index d820f214b..0691ddde8 100644
--- a/libminifi/test/unit/ParameterTokenParserTest.cpp
+++ b/libminifi/test/unit/ParameterTokenParserTest.cpp
@@ -18,16 +18,17 @@
#include "unit/TestBase.h"
#include "unit/Catch.h"
#include "core/ParameterTokenParser.h"
+#include "utils/crypto/property_encryption/PropertyEncryptionUtils.h"
namespace org::apache::nifi::minifi::test {
TEST_CASE("Empty string has zero parameters") {
- core::ParameterTokenParser parser("");
+ core::NonSensitiveParameterTokenParser parser("");
REQUIRE(parser.getTokens().empty());
}
TEST_CASE("Parse a single token") {
- core::ParameterTokenParser parser("#{token.1}");
+ core::NonSensitiveParameterTokenParser parser("#{token.1}");
auto& tokens = parser.getTokens();
REQUIRE(tokens.size() == 1);
CHECK(tokens.at(0)->getName().value() == "token.1");
@@ -36,7 +37,7 @@ TEST_CASE("Parse a single token") {
}
TEST_CASE("Parse multiple tokens") {
- core::ParameterTokenParser parser("#{token1} #{token-2}");
+ core::NonSensitiveParameterTokenParser parser("#{token1} #{token-2}");
auto& tokens = parser.getTokens();
REQUIRE(tokens.size() == 2);
CHECK(tokens.at(0)->getName().value() == "token1");
@@ -48,7 +49,7 @@ TEST_CASE("Parse multiple tokens") {
}
TEST_CASE("Parse the same token multiple times") {
- core::ParameterTokenParser parser("#{token1} #{token-2} #{token1}");
+ core::NonSensitiveParameterTokenParser parser("#{token1} #{token-2}
#{token1}");
auto& tokens = parser.getTokens();
REQUIRE(tokens.size() == 3);
CHECK(tokens.at(0)->getName().value() == "token1");
@@ -63,7 +64,7 @@ TEST_CASE("Parse the same token multiple times") {
}
TEST_CASE("Tokens can be escaped") {
- core::ParameterTokenParser parser("## ##{token1} #{token-2} ###{token_3}# ##
##not_a_token");
+ core::NonSensitiveParameterTokenParser parser("## ##{token1} #{token-2}
###{token_3}# ## ##not_a_token");
auto& tokens = parser.getTokens();
REQUIRE(tokens.size() == 3);
CHECK(tokens.at(0)->getValue().value() == "#{token1}");
@@ -78,7 +79,7 @@ TEST_CASE("Tokens can be escaped") {
}
TEST_CASE("Unfinished token is not a token") {
- core::ParameterTokenParser parser("this is #{_token_ 1} and #{token-2 not
finished");
+ core::NonSensitiveParameterTokenParser parser("this is #{_token_ 1} and
#{token-2 not finished");
auto& tokens = parser.getTokens();
REQUIRE(tokens.size() == 1);
CHECK(tokens.at(0)->getName().value() == "_token_ 1");
@@ -90,54 +91,71 @@ TEST_CASE("Test invalid token names") {
auto create_error_message = [](const std::string& invalid_name){
return "Invalid token name: '" + invalid_name + "'. Only alpha-numeric
characters (a-z, A-Z, 0-9), hyphens ( - ), underscores ( _ ), periods ( . ),
and spaces are allowed in token name.";
};
- CHECK_THROWS_WITH(core::ParameterTokenParser("#{}"),
create_error_message(""));
- CHECK_THROWS_WITH(core::ParameterTokenParser("#{#}"),
create_error_message("#"));
- CHECK_THROWS_WITH(core::ParameterTokenParser("#{[]}"),
create_error_message("[]"));
- CHECK_THROWS_WITH(core::ParameterTokenParser("#{a{}"),
create_error_message("a{"));
- CHECK_THROWS_WITH(core::ParameterTokenParser("#{$$}"),
create_error_message("$$"));
+ CHECK_THROWS_WITH(core::NonSensitiveParameterTokenParser("#{}"),
create_error_message(""));
+ CHECK_THROWS_WITH(core::NonSensitiveParameterTokenParser("#{#}"),
create_error_message("#"));
+ CHECK_THROWS_WITH(core::NonSensitiveParameterTokenParser("#{[]}"),
create_error_message("[]"));
+ CHECK_THROWS_WITH(core::NonSensitiveParameterTokenParser("#{a{}"),
create_error_message("a{"));
+ CHECK_THROWS_WITH(core::NonSensitiveParameterTokenParser("#{$$}"),
create_error_message("$$"));
}
TEST_CASE("Test token replacement") {
- core::ParameterTokenParser parser("## What is #{what}, baby don't hurt
#{who}, don't hurt #{who}, no more ##");
+ core::NonSensitiveParameterTokenParser parser("## What is #{what}, baby
don't hurt #{who}, don't hurt #{who}, no more ##");
core::ParameterContext context("test_context");
- context.addParameter(core::Parameter{"what", "", "love"});
- context.addParameter(core::Parameter{"who", "", "me"});
- REQUIRE(parser.replaceParameters(context, false) == "## What is love, baby
don't hurt me, don't hurt me, no more ##");
+ context.addParameter(core::Parameter{"what", "", false, "love"});
+ context.addParameter(core::Parameter{"who", "", false, "me"});
+ REQUIRE(parser.replaceParameters(context) == "## What is love, baby don't
hurt me, don't hurt me, no more ##");
}
TEST_CASE("Test replacement with escaped tokens") {
- core::ParameterTokenParser parser("### What is #####{what}, baby don't hurt
###{who}, don't hurt ###{who}, no ####{more} ##{");
+ core::NonSensitiveParameterTokenParser parser("### What is #####{what}, baby
don't hurt ###{who}, don't hurt ###{who}, no ####{more} ##{");
REQUIRE(parser.getTokens().size() == 4);
core::ParameterContext context("test_context");
- context.addParameter(core::Parameter{"what", "", "love"});
- context.addParameter(core::Parameter{"who", "", "me"});
- REQUIRE(parser.replaceParameters(context, false) == "### What is ##love,
baby don't hurt #me, don't hurt #me, no ##{more} ##{");
+ context.addParameter(core::Parameter{"what", "", false, "love"});
+ context.addParameter(core::Parameter{"who", "", false, "me"});
+ REQUIRE(parser.replaceParameters(context) == "### What is ##love, baby don't
hurt #me, don't hurt #me, no ##{more} ##{");
}
TEST_CASE("Test replacement with missing token in context") {
- core::ParameterTokenParser parser("What is #{what}, baby don't hurt #{who},
don't hurt #{who}, no more");
+ core::NonSensitiveParameterTokenParser parser("What is #{what}, baby don't
hurt #{who}, don't hurt #{who}, no more");
core::ParameterContext context("test_context");
- context.addParameter(core::Parameter{"what", "", "love"});
- REQUIRE_THROWS_WITH(parser.replaceParameters(context, false), "Parameter
'who' not found");
+ context.addParameter(core::Parameter{"what", "", false, "love"});
+ REQUIRE_THROWS_WITH(parser.replaceParameters(context), "Parameter 'who' not
found");
}
TEST_CASE("Sensitive property parameter replacement is not supported") {
- core::ParameterTokenParser parser("What is #{what}, baby don't hurt #{who},
don't hurt #{who}, no more");
+ utils::crypto::Bytes secret_key =
utils::string::from_hex("cb76fe6fe4cbfdc3770c0cb0afc910f81ced4d436b11f691395fc2a9dbea27ca");
+ utils::crypto::EncryptionProvider encryption_provider{secret_key};
+ core::SensitiveParameterTokenParser parser("What is #{what}, baby don't hurt
#{who}, don't hurt #{who}, no more", encryption_provider);
core::ParameterContext context("test_context");
- context.addParameter(core::Parameter{"what", "", "love"});
- context.addParameter(core::Parameter{"who", "", "me"});
- REQUIRE_THROWS_WITH(parser.replaceParameters(context, true), "Non-sensitive
parameter 'what' cannot be referenced in a sensitive property");
+ context.addParameter(core::Parameter{"what", "", false, "love"});
+ context.addParameter(core::Parameter{"who", "", false, "me"});
+ REQUIRE_THROWS_WITH(parser.replaceParameters(context), "Non-sensitive
parameter 'what' cannot be referenced in a sensitive property");
}
TEST_CASE("Parameter context is not provided when parameter is referenced") {
- core::ParameterTokenParser parser("What is #{what}, baby don't hurt #{who},
don't hurt #{who}, no more");
- REQUIRE_THROWS_WITH(parser.replaceParameters(std::nullopt, false), "Property
references a parameter in its value, but no parameter context was provided.");
+ core::NonSensitiveParameterTokenParser parser("What is #{what}, baby don't
hurt #{who}, don't hurt #{who}, no more");
+ REQUIRE_THROWS_WITH(parser.replaceParameters(std::nullopt), "Property
references a parameter in its value, but no parameter context was provided.");
}
TEST_CASE("Replace only escaped tokens") {
- core::ParameterTokenParser parser("No ##{parameters} are ####{present}");
- REQUIRE(parser.replaceParameters(std::nullopt, false) == "No #{parameters}
are ##{present}");
- REQUIRE(parser.replaceParameters(std::nullopt, true) == "No #{parameters}
are ##{present}");
+ core::NonSensitiveParameterTokenParser non_sensitive_parser("No
##{parameters} are ####{present}");
+ REQUIRE(non_sensitive_parser.replaceParameters(std::nullopt) == "No
#{parameters} are ##{present}");
+ utils::crypto::Bytes secret_key =
utils::string::from_hex("cb76fe6fe4cbfdc3770c0cb0afc910f81ced4d436b11f691395fc2a9dbea27ca");
+ utils::crypto::EncryptionProvider encryption_provider{secret_key};
+ core::SensitiveParameterTokenParser sensitive_parser("No ##{parameters} are
####{present}", encryption_provider);
+ REQUIRE(sensitive_parser.replaceParameters(std::nullopt) == "No
#{parameters} are ##{present}");
+}
+
+TEST_CASE("Test sensitive token replacement") {
+ core::ParameterContext context("test_context");
+ utils::crypto::Bytes secret_key =
utils::string::from_hex("cb76fe6fe4cbfdc3770c0cb0afc910f81ced4d436b11f691395fc2a9dbea27ca");
+ utils::crypto::EncryptionProvider encryption_provider{secret_key};
+ core::SensitiveParameterTokenParser parser("What is #{what}, baby don't hurt
#{who}, don't hurt #{who}, no more", encryption_provider);
+ auto value1 = utils::crypto::property_encryption::encrypt("love",
encryption_provider);
+ auto value2 = utils::crypto::property_encryption::encrypt("me",
encryption_provider);
+ context.addParameter(core::Parameter{"what", "", true, value1});
+ context.addParameter(core::Parameter{"who", "", true, value2});
+ REQUIRE(parser.replaceParameters(context) == "What is love, baby don't hurt
me, don't hurt me, no more");
}
} // namespace org::apache::nifi::minifi::test
diff --git a/libminifi/test/unit/YamlFlowSerializerTests.cpp
b/libminifi/test/unit/YamlFlowSerializerTests.cpp
index d9cd0a249..75062fa92 100644
--- a/libminifi/test/unit/YamlFlowSerializerTests.cpp
+++ b/libminifi/test/unit/YamlFlowSerializerTests.cpp
@@ -82,6 +82,15 @@ Security Properties:
Sensitive Props:
key: ~
algorithm: NIFI_PBKDF2_AES_GCM_256
+Parameter Contexts:
+ - id: 721e10b7-8e00-3188-9a27-476cca376978
+ name: my-context
+ description: my parameter context
+ Parameters:
+ - name: secret_parameter
+ description: ""
+ sensitive: true
+ value: param_value_1
Processors:
- id: aabb6d26-8a8d-4338-92c9-1b8c67ec18e0
name: GenerateFlowFile
@@ -188,17 +197,20 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive
properties") {
const auto processor_id =
utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value();
const auto controller_service_id =
utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value();
+ const auto parameter_id =
utils::Identifier::parse("721e10b7-8e00-3188-9a27-476cca376978").value();
const auto [overrides, expected_results] = GENERATE_REF(
std::make_tuple(OverridesMap{},
- std::array{"very_secure_password", "very_secure_passphrase"}),
+ std::array{"very_secure_password", "very_secure_passphrase",
"param_value_1"}),
std::make_tuple(OverridesMap{{processor_id,
core::flow::Overrides{}.add("invokehttp-proxy-password", "password123")}},
- std::array{"password123", "very_secure_passphrase"}),
+ std::array{"password123", "very_secure_passphrase",
"param_value_1"}),
std::make_tuple(OverridesMap{{controller_service_id,
core::flow::Overrides{}.add("Passphrase", "speak friend and enter")}},
- std::array{"very_secure_password", "speak friend and enter"}),
+ std::array{"very_secure_password", "speak friend and enter",
"param_value_1"}),
std::make_tuple(OverridesMap{{processor_id,
core::flow::Overrides{}.add("invokehttp-proxy-password", "password123")},
{controller_service_id,
core::flow::Overrides{}.add("Passphrase", "speak friend and enter")}},
- std::array{"password123", "speak friend and enter"}));
+ std::array{"password123", "speak friend and enter",
"param_value_1"}),
+ std::make_tuple(OverridesMap{{parameter_id,
core::flow::Overrides{}.add("secret_parameter", "param_value_2")}},
+ std::array{"very_secure_password", "very_secure_passphrase",
"param_value_2"}));
std::string config_yaml_encrypted =
flow_serializer.serialize(*process_group, schema, encryption_provider,
overrides);
@@ -221,6 +233,16 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive
properties") {
std::string encrypted_value = match_results[1];
CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == expected_results[1]);
}
+
+ {
+ std::regex regex{R"_(value: (.*))_"};
+ std::smatch match_results;
+ CHECK(std::regex_search(config_yaml_encrypted, match_results, regex));
+
+ REQUIRE(match_results.size() == 2);
+ std::string encrypted_value = match_results[1];
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value,
encryption_provider) == expected_results[2]);
+ }
}
TEST_CASE("YamlFlowSerializer with an override can add a new property to the
flow config file") {
@@ -278,7 +300,7 @@ TEST_CASE("YamlFlowSerializer with an override can add a
new property to the flo
TEST_CASE("The encrypted flow configuration can be decrypted with the correct
key") {
ConfigurationTestController test_controller;
auto configuration_context = test_controller.getContext();
- configuration_context.sensitive_properties_encryptor = encryption_provider;
+ configuration_context.sensitive_values_encryptor = encryption_provider;
core::flow::AdaptiveConfiguration
yaml_configuration_before{configuration_context};
const auto process_group_before =
yaml_configuration_before.getRootFromPayload(std::string{config_yaml});
@@ -312,12 +334,15 @@ TEST_CASE("The encrypted flow configuration can be
decrypted with the correct ke
REQUIRE(controller_service_after);
CHECK(controller_service_before->getProperties().at("CA
Certificate").getValue() == controller_service_after->getProperties().at("CA
Certificate").getValue());
CHECK(controller_service_before->getProperties().at("Passphrase").getValue()
== controller_service_after->getProperties().at("Passphrase").getValue());
+
+ const auto& param_contexts = yaml_configuration_after.getParameterContexts();
+
CHECK(param_contexts.at("my-context")->getParameter("secret_parameter")->value
== "param_value_1");
}
TEST_CASE("The encrypted flow configuration cannot be decrypted with an
incorrect key") {
ConfigurationTestController test_controller;
auto configuration_context = test_controller.getContext();
- configuration_context.sensitive_properties_encryptor = encryption_provider;
+ configuration_context.sensitive_values_encryptor = encryption_provider;
core::flow::AdaptiveConfiguration
yaml_configuration_before{configuration_context};
const auto process_group_before =
yaml_configuration_before.getRootFromPayload(std::string{config_yaml});
@@ -329,7 +354,7 @@ TEST_CASE("The encrypted flow configuration cannot be
decrypted with an incorrec
std::string config_yaml_encrypted =
flow_serializer.serialize(*process_group_before, schema, encryption_provider,
{});
const utils::crypto::Bytes different_secret_key =
utils::string::from_hex("ea55b7d0edc22280c9547e4d89712b3fae74f96d82f240a004fb9fbd0640eec7");
- configuration_context.sensitive_properties_encryptor =
utils::crypto::EncryptionProvider{different_secret_key};
+ configuration_context.sensitive_values_encryptor =
utils::crypto::EncryptionProvider{different_secret_key};
core::flow::AdaptiveConfiguration
yaml_configuration_after{configuration_context};
REQUIRE_THROWS_AS(yaml_configuration_after.getRootFromPayload(config_yaml_encrypted),
utils::crypto::EncryptionError);
diff --git a/minifi_main/MiNiFiMain.cpp b/minifi_main/MiNiFiMain.cpp
index 1e34a09e3..3b140387a 100644
--- a/minifi_main/MiNiFiMain.cpp
+++ b/minifi_main/MiNiFiMain.cpp
@@ -397,7 +397,7 @@ int main(int argc, char **argv) {
.configuration = configure,
.path =
configure->get(minifi::Configure::nifi_flow_configuration_file),
.filesystem = filesystem,
- .sensitive_properties_encryptor =
utils::crypto::EncryptionProvider::createSensitivePropertiesEncryptor(minifiHome)
+ .sensitive_values_encryptor =
utils::crypto::EncryptionProvider::createSensitivePropertiesEncryptor(minifiHome)
}, nifi_configuration_class_name);
std::vector<std::shared_ptr<core::RepositoryMetricsSource>>
repo_metric_sources{prov_repo, flow_repo, content_repo};
