This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a commit to branch MINIFICPP-2367 in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 042d77a54bd39c50cd37d527f745a90b0ca4f4ba Author: Gabor Gyimesi <gamezb...@gmail.com> AuthorDate: Wed Jun 19 16:23:03 2024 +0200 Fix escape character handling --- .../tests/unit/FlowJsonTests.cpp | 8 +- .../tests/unit/YamlConfigurationTests.cpp | 8 +- libminifi/include/core/ParameterTokenParser.h | 69 ++++++++++++-- libminifi/src/core/ParameterTokenParser.cpp | 52 ++++++++--- .../src/core/flow/StructuredConfiguration.cpp | 40 +++----- libminifi/test/unit/ParameterTokenParserTest.cpp | 103 ++++++++++++++------- 6 files changed, 190 insertions(+), 90 deletions(-) diff --git a/extensions/standard-processors/tests/unit/FlowJsonTests.cpp b/extensions/standard-processors/tests/unit/FlowJsonTests.cpp index 508fdd073..0f1b214e2 100644 --- a/extensions/standard-processors/tests/unit/FlowJsonTests.cpp +++ b/extensions/standard-processors/tests/unit/FlowJsonTests.cpp @@ -388,7 +388,7 @@ TEST_CASE("Cannot use non-sensitive parameter in sensitive property") { } })"; - REQUIRE_THROWS_WITH(config.getRootFromPayload(CONFIG_JSON), "Sensitive property 'Sensitive Property' cannot reference non-sensitive parameters"); + REQUIRE_THROWS_WITH(config.getRootFromPayload(CONFIG_JSON), "Sensitive property cannot reference non-sensitive parameters"); } TEST_CASE("Cannot use non-sensitive parameter in sensitive property value sequence") { @@ -433,7 +433,7 @@ TEST_CASE("Cannot use non-sensitive parameter in sensitive property value sequen } })"; - REQUIRE_THROWS_WITH(config.getRootFromPayload(CONFIG_JSON), "Sensitive property 'Sensitive Property' cannot reference non-sensitive parameters"); + REQUIRE_THROWS_WITH(config.getRootFromPayload(CONFIG_JSON), "Sensitive property cannot reference non-sensitive parameters"); } TEST_CASE("Parameters can be used in nested process groups") { @@ -635,7 +635,7 @@ TEST_CASE("Cannot use parameters if no parameter context is defined") { } })"; - REQUIRE_THROWS_WITH(config.getRootFromPayload(CONFIG_JSON), "Property 'Simple Property' references a parameter in its value, but no parameter context was provided in the process group."); + REQUIRE_THROWS_WITH(config.getRootFromPayload(CONFIG_JSON), "Property references a parameter in its value, but no parameter context was provided."); } TEST_CASE("Cannot use parameters in property value sequences if no parameter context is defined") { @@ -664,7 +664,7 @@ TEST_CASE("Cannot use parameters in property value sequences if no parameter con } })"; - REQUIRE_THROWS_WITH(config.getRootFromPayload(CONFIG_JSON), "Property 'Simple Property' references a parameter in its value, but no parameter context was provided in the process group."); + REQUIRE_THROWS_WITH(config.getRootFromPayload(CONFIG_JSON), "Property references a parameter in its value, but no parameter context was provided."); } TEST_CASE("Property value sequences can use parameters") { diff --git a/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp b/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp index d73650894..c6b2b6d93 100644 --- a/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp +++ b/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp @@ -1314,7 +1314,7 @@ Parameter Context Name: my-context NiFi Properties Overrides: {} )"; - REQUIRE_THROWS_WITH(yaml_config.getRootFromPayload(TEST_CONFIG_YAML), "Sensitive property 'Sensitive Property' cannot reference non-sensitive parameters"); + REQUIRE_THROWS_WITH(yaml_config.getRootFromPayload(TEST_CONFIG_YAML), "Sensitive property cannot reference non-sensitive parameters"); } TEST_CASE("Cannot use non-sensitive parameter in sensitive property value sequence", "[YamlConfiguration]") { @@ -1357,7 +1357,7 @@ Parameter Context Name: my-context NiFi Properties Overrides: {} )"; - REQUIRE_THROWS_WITH(yaml_config.getRootFromPayload(TEST_CONFIG_YAML), "Sensitive property 'Sensitive Property' cannot reference non-sensitive parameters"); + REQUIRE_THROWS_WITH(yaml_config.getRootFromPayload(TEST_CONFIG_YAML), "Sensitive property cannot reference non-sensitive parameters"); } TEST_CASE("Parameters can be used in nested process groups", "[YamlConfiguration]") { @@ -1524,7 +1524,7 @@ Processors: Simple Property: "#{my_value}" )"; - REQUIRE_THROWS_WITH(yaml_config.getRootFromPayload(TEST_CONFIG_YAML), "Property 'Simple Property' references a parameter in its value, but no parameter context was provided in the process group."); + REQUIRE_THROWS_WITH(yaml_config.getRootFromPayload(TEST_CONFIG_YAML), "Property references a parameter in its value, but no parameter context was provided."); } TEST_CASE("Cannot use parameters in property value sequences if no parameter context is defined", "[YamlConfiguration]") { @@ -1550,7 +1550,7 @@ Processors: - value: "#{second_value}" )"; - REQUIRE_THROWS_WITH(yaml_config.getRootFromPayload(TEST_CONFIG_YAML), "Property 'Simple Property' references a parameter in its value, but no parameter context was provided in the process group."); + REQUIRE_THROWS_WITH(yaml_config.getRootFromPayload(TEST_CONFIG_YAML), "Property references a parameter in its value, but no parameter context was provided."); } TEST_CASE("Property value sequences can use parameters", "[YamlConfiguration]") { diff --git a/libminifi/include/core/ParameterTokenParser.h b/libminifi/include/core/ParameterTokenParser.h index f7c4654f2..b5ef47391 100644 --- a/libminifi/include/core/ParameterTokenParser.h +++ b/libminifi/include/core/ParameterTokenParser.h @@ -20,6 +20,8 @@ #include <utility> #include <string> #include <unordered_map> +#include <memory> +#include <optional> #include "ParameterContext.h" #include "Exception.h" @@ -28,9 +30,16 @@ namespace org::apache::nifi::minifi::core { class ParameterToken { public: - ParameterToken(std::string name, uint32_t start, uint32_t size) : name_(std::move(name)), start_(start), size_(size) { + enum class ParameterTokenType { + Escaped, + Replaceable + }; + + ParameterToken(uint32_t start, uint32_t size) : start_(start), size_(size) { } + virtual ~ParameterToken() = default; + uint32_t getStart() const { return start_; } @@ -39,8 +48,17 @@ class ParameterToken { return size_; } - const std::string& getName() const { - return name_; + virtual ParameterTokenType getType() const = 0; + virtual std::optional<std::string> getName() const { + return std::nullopt; + } + + virtual std::optional<std::string> getValue() const { + return std::nullopt; + } + + virtual uint32_t getAdditionalHashmarks() const { + return 0; } private: @@ -49,6 +67,45 @@ class ParameterToken { uint32_t size_; }; +class ReplaceableToken : public ParameterToken { + public: + ReplaceableToken(std::string name, uint32_t additional_hashmarks, uint32_t start, uint32_t size) : ParameterToken(start, size), name_(std::move(name)), additional_hashmarks_(additional_hashmarks) { + } + + std::optional<std::string> getName() const override { + return name_; + } + + ParameterTokenType getType() const override { + return ParameterTokenType::Replaceable; + } + + uint32_t getAdditionalHashmarks() const override { + return additional_hashmarks_; + } + + private: + std::string name_; + uint32_t additional_hashmarks_; +}; + +class EscapedToken : public ParameterToken { + public: + EscapedToken(uint32_t start, uint32_t size, std::string replaced_value) : ParameterToken(start, size), replaced_value_(std::move(replaced_value)) { + }; + + ParameterTokenType getType() const override { + return ParameterTokenType::Escaped; + } + + std::optional<std::string> getValue() const override { + return replaced_value_; + } + + private: + std::string replaced_value_; +}; + class ParameterTokenParser { public: enum class ParseState { @@ -61,17 +118,17 @@ class ParameterTokenParser { parse(); } - const std::vector<ParameterToken>& getTokens() const { + const std::vector<std::unique_ptr<ParameterToken>>& getTokens() const { return tokens_; } - std::string replaceParameters(const ParameterContext& parameter_context) const; + std::string replaceParameters(const std::optional<ParameterContext>& parameter_context, bool is_sensitive) const; private: void parse(); std::string input_; - std::vector<ParameterToken> tokens_; + std::vector<std::unique_ptr<ParameterToken>> tokens_; }; } // namespace org::apache::nifi::minifi::core diff --git a/libminifi/src/core/ParameterTokenParser.cpp b/libminifi/src/core/ParameterTokenParser.cpp index 4210d88cb..4e503170a 100644 --- a/libminifi/src/core/ParameterTokenParser.cpp +++ b/libminifi/src/core/ParameterTokenParser.cpp @@ -26,53 +26,75 @@ void ParameterTokenParser::parse() { minifi::utils::Regex expr("[-a-zA-Z0-9_\\. ]+"); ParseState state = ParseState::OutsideToken; uint32_t token_start = 0; + uint32_t hashmark_length = 0; for (uint32_t i = 0; i < input_.size(); ++i) { if (input_[i] == '#') { if (state == ParseState::OutsideToken) { state = ParseState::InHashMark; - } else if (state == ParseState::InHashMark) { - state = ParseState::OutsideToken; + } + if (state != ParseState::InToken) { + ++hashmark_length; } } else if (input_[i] == '{') { if (state == ParseState::InHashMark) { - token_start = i - 1; + token_start = i - hashmark_length; state = ParseState::InToken; } } else if (input_[i] == '}') { if (state == ParseState::InToken) { state = ParseState::OutsideToken; - auto token_name = input_.substr(token_start + 2, i - token_start - 2); - if (token_name.empty() || !minifi::utils::regexMatch(token_name, expr)) { - throw ParameterException("Invalid token name: '" + token_name + "'. " - "Only alpha-numeric characters (a-z, A-Z, 0-9), hyphens ( - ), underscores ( _ ), periods ( . ), and spaces are allowed in token name."); + gsl_Assert(hashmark_length > 0); + if (hashmark_length % 2 == 0) { + tokens_.push_back(std::make_unique<EscapedToken>(token_start, i - token_start + 1, input_.substr(token_start + (hashmark_length / 2), i - token_start + 1 - (hashmark_length / 2)))); + } else { + auto token_name = input_.substr(token_start + hashmark_length + 1, i - token_start - hashmark_length - 1); + if (token_name.empty() || !minifi::utils::regexMatch(token_name, expr)) { + throw ParameterException("Invalid token name: '" + token_name + "'. " + "Only alpha-numeric characters (a-z, A-Z, 0-9), hyphens ( - ), underscores ( _ ), periods ( . ), and spaces are allowed in token name."); + } + tokens_.push_back(std::make_unique<ReplaceableToken>(token_name, (hashmark_length - 1) / 2, token_start, i - token_start + 1)); + token_start = 0; } - tokens_.push_back(ParameterToken{token_name, token_start, i - token_start + 1}); - token_start = 0; } else { state = ParseState::OutsideToken; } + hashmark_length = 0; } else { if (state != ParseState::InToken) { state = ParseState::OutsideToken; + hashmark_length = 0; } } } } -std::string ParameterTokenParser::replaceParameters(const ParameterContext& parameter_context) const { +std::string ParameterTokenParser::replaceParameters(const std::optional<ParameterContext>& parameter_context, bool is_sensitive) const { if (tokens_.empty()) { return input_; } std::string result; uint32_t last_end = 0; for (const auto& token : tokens_) { - result.append(input_.substr(last_end, token.getStart() - last_end)); - auto parameter = parameter_context.getParameter(token.getName()); + result.append(input_.substr(last_end, token->getStart() - last_end)); + if (token->getType() == ParameterToken::ParameterTokenType::Escaped) { + result.append(token->getValue().value()); + last_end = token->getStart() + token->getSize(); + continue; + } + + if (!parameter_context) { + throw ParameterException("Property references a parameter in its value, but no parameter context was provided."); + } + if (is_sensitive) { + throw ParameterException("Sensitive property cannot reference non-sensitive parameters"); + } + gsl_Assert(token->getName().has_value()); + auto parameter = parameter_context->getParameter(token->getName().value()); if (!parameter.has_value()) { - throw ParameterException("Parameter '" + token.getName() + "' not found"); + throw ParameterException("Parameter '" + token->getName().value() + "' not found"); } - result.append(parameter->value); - last_end = token.getStart() + token.getSize(); + result.append(std::string(token->getAdditionalHashmarks(), '#') + parameter->value); + last_end = token->getStart() + token->getSize(); } result.append(input_.substr(last_end)); return result; diff --git a/libminifi/src/core/flow/StructuredConfiguration.cpp b/libminifi/src/core/flow/StructuredConfiguration.cpp index 78414c567..760300308 100644 --- a/libminifi/src/core/flow/StructuredConfiguration.cpp +++ b/libminifi/src/core/flow/StructuredConfiguration.cpp @@ -673,15 +673,13 @@ void StructuredConfiguration::parsePropertyValueSequence(const std::string& prop if (myProp.isSensitive()) { rawValueString = utils::crypto::property_encryption::decrypt(rawValueString, sensitive_properties_encryptor_); } - core::ParameterTokenParser token_parser(rawValueString); - if (!token_parser.getTokens().empty()) { - if (myProp.isSensitive()) { - throw ParameterException("Sensitive property '" + property_name + "' cannot reference non-sensitive parameters"); - } - if (!parameter_context) { - throw ParameterException("Property '" + property_name + "' references a parameter in its value, but no parameter context was provided in the process group."); - } - rawValueString = token_parser.replaceParameters(*parameter_context); + + try { + core::ParameterTokenParser token_parser(rawValueString); + rawValueString = token_parser.replaceParameters(parameter_context, myProp.isSensitive()); + } catch (const ParameterException& e) { + logger_->log_error("Error while substituting parameters in property '{}': {}", property_name, e.what()); + throw; } logger_->log_debug("Found property {}", property_name); @@ -720,23 +718,15 @@ PropertyValue StructuredConfiguration::getValidatedProcessorPropertyForDefaultTy coercedValue = gsl::narrow<int>(int64_val.value()); } else if (defaultType == Value::BOOL_TYPE && property_value_node.getBool()) { coercedValue = property_value_node.getBool().value(); - } else if (property_from_processor.isSensitive()) { - auto property_value_string = utils::crypto::property_encryption::decrypt(property_value_node.getScalarAsString().value(), sensitive_properties_encryptor_); - core::ParameterTokenParser token_parser(property_value_string); - if (!token_parser.getTokens().empty()) { - throw ParameterException("Sensitive property '" + property_from_processor.getName() + "' cannot reference non-sensitive parameters"); - } - coercedValue = property_value_string; } else { - auto property_value_string = property_value_node.getScalarAsString().value(); - - core::ParameterTokenParser token_parser(property_value_string); - if (!token_parser.getTokens().empty()) { - if (!parameter_context) { - throw ParameterException("Property '" + property_from_processor.getName() + "' references a parameter in its value, but no parameter context was provided in the process group."); - } - property_value_string = token_parser.replaceParameters(*parameter_context); + std::string property_value_string; + if (property_from_processor.isSensitive()) { + property_value_string = utils::crypto::property_encryption::decrypt(property_value_node.getScalarAsString().value(), sensitive_properties_encryptor_); + } else { + property_value_string = property_value_node.getScalarAsString().value(); } + core::ParameterTokenParser token_parser(property_value_string); + property_value_string = token_parser.replaceParameters(parameter_context, property_from_processor.isSensitive()); coercedValue = property_value_string; } return coercedValue; @@ -744,7 +734,7 @@ PropertyValue StructuredConfiguration::getValidatedProcessorPropertyForDefaultTy logger_->log_error("Fetching property failed with a decryption error: {}", e.what()); throw; } catch (const ParameterException& e) { - logger_->log_error("{}", e.what()); + logger_->log_error("Error while substituting parameters in property '{}': {}", property_from_processor.getName(), e.what()); throw; } catch (const std::exception& e) { logger_->log_error("Fetching property failed with an exception of {}", e.what()); diff --git a/libminifi/test/unit/ParameterTokenParserTest.cpp b/libminifi/test/unit/ParameterTokenParserTest.cpp index d03574dbf..5b5e22f1c 100644 --- a/libminifi/test/unit/ParameterTokenParserTest.cpp +++ b/libminifi/test/unit/ParameterTokenParserTest.cpp @@ -28,59 +28,62 @@ TEST_CASE("Empty string has zero parameters") { TEST_CASE("Parse a single token") { core::ParameterTokenParser parser("#{token.1}"); - REQUIRE(parser.getTokens().size() == 1); auto& tokens = parser.getTokens(); - CHECK(tokens.at(0).getName() == "token.1"); - CHECK(tokens.at(0).getStart() == 0); - CHECK(tokens.at(0).getSize() == 10); + REQUIRE(tokens.size() == 1); + CHECK(tokens.at(0)->getName().value() == "token.1"); + CHECK(tokens.at(0)->getStart() == 0); + CHECK(tokens.at(0)->getSize() == 10); } TEST_CASE("Parse multiple tokens") { core::ParameterTokenParser parser("#{token1} #{token-2}"); - REQUIRE(parser.getTokens().size() == 2); auto& tokens = parser.getTokens(); - CHECK(tokens.at(0).getName() == "token1"); - CHECK(tokens.at(0).getStart() == 0); - CHECK(tokens.at(0).getSize() == 9); - CHECK(tokens.at(1).getName() == "token-2"); - CHECK(tokens.at(1).getStart() == 10); - CHECK(tokens.at(1).getSize() == 10); + REQUIRE(tokens.size() == 2); + CHECK(tokens.at(0)->getName().value() == "token1"); + CHECK(tokens.at(0)->getStart() == 0); + CHECK(tokens.at(0)->getSize() == 9); + CHECK(tokens.at(1)->getName().value() == "token-2"); + CHECK(tokens.at(1)->getStart() == 10); + CHECK(tokens.at(1)->getSize() == 10); } TEST_CASE("Parse the same token multiple times") { core::ParameterTokenParser parser("#{token1} #{token-2} #{token1}"); - REQUIRE(parser.getTokens().size() == 3); auto& tokens = parser.getTokens(); - CHECK(tokens.at(0).getName() == "token1"); - CHECK(tokens.at(0).getStart() == 0); - CHECK(tokens.at(0).getSize() == 9); - CHECK(tokens.at(1).getName() == "token-2"); - CHECK(tokens.at(1).getStart() == 10); - CHECK(tokens.at(1).getSize() == 10); - CHECK(tokens.at(2).getName() == "token1"); - CHECK(tokens.at(2).getStart() == 21); - CHECK(tokens.at(2).getSize() == 9); + REQUIRE(tokens.size() == 3); + CHECK(tokens.at(0)->getName().value() == "token1"); + CHECK(tokens.at(0)->getStart() == 0); + CHECK(tokens.at(0)->getSize() == 9); + CHECK(tokens.at(1)->getName().value() == "token-2"); + CHECK(tokens.at(1)->getStart() == 10); + CHECK(tokens.at(1)->getSize() == 10); + CHECK(tokens.at(2)->getName().value() == "token1"); + CHECK(tokens.at(2)->getStart() == 21); + CHECK(tokens.at(2)->getSize() == 9); } TEST_CASE("Tokens can be escaped") { - core::ParameterTokenParser parser("## ##{token1} #{token-2} ###{token_3}# ## #not_a_token"); - REQUIRE(parser.getTokens().size() == 2); + core::ParameterTokenParser parser("## ##{token1} #{token-2} ###{token_3}# ## ##not_a_token"); auto& tokens = parser.getTokens(); - CHECK(tokens.at(0).getName() == "token-2"); - CHECK(tokens.at(0).getStart() == 14); - CHECK(tokens.at(0).getSize() == 10); - CHECK(tokens.at(1).getName() == "token_3"); - CHECK(tokens.at(1).getStart() == 27); - CHECK(tokens.at(1).getSize() == 10); + REQUIRE(tokens.size() == 3); + CHECK(tokens.at(0)->getValue().value() == "#{token1}"); + CHECK(tokens.at(0)->getStart() == 3); + CHECK(tokens.at(0)->getSize() == 10); + CHECK(tokens.at(1)->getName().value() == "token-2"); + CHECK(tokens.at(1)->getStart() == 14); + CHECK(tokens.at(1)->getSize() == 10); + CHECK(tokens.at(2)->getName().value() == "token_3"); + CHECK(tokens.at(2)->getStart() == 25); + CHECK(tokens.at(2)->getSize() == 12); } TEST_CASE("Unfinished token is not a token") { core::ParameterTokenParser parser("this is #{_token_ 1} and #{token-2 not finished"); - REQUIRE(parser.getTokens().size() == 1); auto& tokens = parser.getTokens(); - CHECK(tokens.at(0).getName() == "_token_ 1"); - CHECK(tokens.at(0).getStart() == 8); - CHECK(tokens.at(0).getSize() == 12); + REQUIRE(tokens.size() == 1); + CHECK(tokens.at(0)->getName().value() == "_token_ 1"); + CHECK(tokens.at(0)->getStart() == 8); + CHECK(tokens.at(0)->getSize() == 12); } TEST_CASE("Test invalid token names") { @@ -95,18 +98,46 @@ TEST_CASE("Test invalid token names") { } TEST_CASE("Test token replacement") { - core::ParameterTokenParser parser("What is #{what}, baby don't hurt #{who}, don't hurt #{who}, no more"); + core::ParameterTokenParser parser("## What is #{what}, baby don't hurt #{who}, don't hurt #{who}, no more ##"); core::ParameterContext context("test_context"); context.addParameter(core::Parameter{"what", "", "love"}); context.addParameter(core::Parameter{"who", "", "me"}); - REQUIRE(parser.replaceParameters(context) == "What is love, baby don't hurt me, don't hurt me, no more"); + REQUIRE(parser.replaceParameters(context, false) == "## What is love, baby don't hurt me, don't hurt me, no more ##"); +} + +TEST_CASE("Test replacement with escaped tokens") { + core::ParameterTokenParser parser("### What is #####{what}, baby don't hurt ###{who}, don't hurt ###{who}, no ####{more} ##{"); + REQUIRE(parser.getTokens().size() == 4); + core::ParameterContext context("test_context"); + context.addParameter(core::Parameter{"what", "", "love"}); + context.addParameter(core::Parameter{"who", "", "me"}); + REQUIRE(parser.replaceParameters(context, false) == "### What is ##love, baby don't hurt #me, don't hurt #me, no ##{more} ##{"); } TEST_CASE("Test replacement with missing token in context") { core::ParameterTokenParser parser("What is #{what}, baby don't hurt #{who}, don't hurt #{who}, no more"); core::ParameterContext context("test_context"); context.addParameter(core::Parameter{"what", "", "love"}); - REQUIRE_THROWS_WITH(parser.replaceParameters(context), "Parameter 'who' not found"); + REQUIRE_THROWS_WITH(parser.replaceParameters(context, false), "Parameter 'who' not found"); +} + +TEST_CASE("Sensitive property parameter replacement is not supported") { + core::ParameterTokenParser parser("What is #{what}, baby don't hurt #{who}, don't hurt #{who}, no more"); + core::ParameterContext context("test_context"); + context.addParameter(core::Parameter{"what", "", "love"}); + context.addParameter(core::Parameter{"who", "", "me"}); + REQUIRE_THROWS_WITH(parser.replaceParameters(context, true), "Sensitive property cannot reference non-sensitive parameters"); +} + +TEST_CASE("Parameter context is not provided when parameter is referenced") { + core::ParameterTokenParser parser("What is #{what}, baby don't hurt #{who}, don't hurt #{who}, no more"); + REQUIRE_THROWS_WITH(parser.replaceParameters(std::nullopt, false), "Property references a parameter in its value, but no parameter context was provided."); +} + +TEST_CASE("Replace only escaped tokens") { + core::ParameterTokenParser parser("No ##{parameters} are ####{present}"); + REQUIRE(parser.replaceParameters(std::nullopt, false) == "No #{parameters} are ##{present}"); + REQUIRE(parser.replaceParameters(std::nullopt, true) == "No #{parameters} are ##{present}"); } } // namespace org::apache::nifi::minifi::test