This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 242686b5a9385fb82b416043863379a589aeff63 Author: Martin Zink <[email protected]> AuthorDate: Thu Apr 20 18:25:23 2023 +0200 MINIFICPP-2089 CWEL: prefix EventData in flattened JSON ...so it doesnt need to be included twice Closes #1552 Signed-off-by: Marton Szasz <[email protected]> --- CMakeLists.txt | 1 + .../tests/CWELCustomProviderTests.cpp | 7 ++- .../tests/ConsumeWindowsEventLogTests.cpp | 8 +-- extensions/windows-event-log/wel/JSONUtils.cpp | 68 +++++++++++++++------- 4 files changed, 55 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7eb4edc2a..1bc7f382c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -288,6 +288,7 @@ target_include_directories(concurrentqueue SYSTEM INTERFACE "${CMAKE_CURRENT_SOU # RapidJSON add_library(RapidJSON INTERFACE) target_include_directories(RapidJSON SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/rapidjson-48fbd8cd202ca54031fe799db2ad44ffa8e77c13/include") +target_compile_definitions(RapidJSON INTERFACE RAPIDJSON_HAS_STDSTRING) # cxxopts include(CxxOpts) diff --git a/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp index 1a771ec3a..9a3a61e4c 100644 --- a/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp +++ b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp @@ -172,9 +172,10 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly cus { "Name": ")" + CUSTOM_PROVIDER_NAME + R"(", "Channel": ")" + CUSTOM_CHANNEL /* Channel is not overwritten by data named "Channel" */ + R"(", - "EventData": )" + EVENT_DATA_JSON /* EventData is not discarded */ + R"(, - "param1": "Actual event", - "param2": "Second" + "EventData.param1": "Actual event", + "EventData.param2": "Second", + "EventData.Channel": "Third", + "EventData": "0901" } )"); } diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp index d91174419..d0e4d39d7 100644 --- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp +++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp @@ -390,11 +390,7 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly", " { "Name": "Application", "Channel": "Application", - "EventData": [{ - "Type": "Data", - "Content": "Event one", - "Name": "" - }] + "EventData": "Event one" } )json"); } @@ -526,7 +522,7 @@ TEST_CASE("ConsumeWindowsEventLog Simple JSON works with UserData", "[cwel][json } SECTION("flattened") { const auto flattened_json = jsonToString(toFlattenedJSON(doc)); - const auto expected_json = R"json({"Name":"Microsoft-Windows-AppLocker","Guid":"CBDA4DBF-8D5D-4F69-9578-BE14AA540D22","EventID":"8002","Version":"0","Level":"4","Task":"0","Opcode":"0","Keywords":"0x8000000000000000","SystemTime":"2023-02-06T16:58:09.008534Z","EventRecordID":"46","ProcessID":"1234","ThreadID":"1235","Channel":"Microsoft-Windows-AppLocker/EXE and DLL","Computer":"example.local","EventData":[],"PolicyNameLength":"3","PolicyName":"EXE","RuleNameLength":"9","RuleName":"A [...] + const auto expected_json = R"json({"Name":"Microsoft-Windows-AppLocker","Guid":"CBDA4DBF-8D5D-4F69-9578-BE14AA540D22","EventID":"8002","Version":"0","Level":"4","Task":"0","Opcode":"0","Keywords":"0x8000000000000000","SystemTime":"2023-02-06T16:58:09.008534Z","EventRecordID":"46","ProcessID":"1234","ThreadID":"1235","Channel":"Microsoft-Windows-AppLocker/EXE and DLL","Computer":"example.local","UserData.RuleAndFileData.PolicyNameLength":"3","UserData.RuleAndFileData.PolicyName":"EXE" [...] CHECK(expected_json == flattened_json); } } diff --git a/extensions/windows-event-log/wel/JSONUtils.cpp b/extensions/windows-event-log/wel/JSONUtils.cpp index e6bfc2f7e..0f2913d62 100644 --- a/extensions/windows-event-log/wel/JSONUtils.cpp +++ b/extensions/windows-event-log/wel/JSONUtils.cpp @@ -27,6 +27,7 @@ #include "rapidjson/stringbuffer.h" #include "utils/gsl.h" +#include "utils/StringUtils.h" namespace org::apache::nifi::minifi::wel { namespace { @@ -59,29 +60,51 @@ rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node, rapidjson::Docume return children; } -void simplifiedGenericXmlToJson(const pugi::xml_node& source_node, rapidjson::Value& output_value, rapidjson::Document& allocator_source_document, bool flatten = false) { +void simplifiedGenericXmlToJson(const pugi::xml_node& source_node, + rapidjson::Value& output_value, + rapidjson::Document::AllocatorType& allocator, + std::optional<std::string> prefix_for_flat_structure) { gsl_Expects(source_node.type() == pugi::xml_node_type::node_element); + const bool is_flattened = prefix_for_flat_structure.has_value(); for (const auto& attr : source_node.attributes()) { if (attr.name() == std::string_view{"xmlns"}) { continue; // skip xmlns attribute, because it's metadata } - output_value.AddMember(rapidjson::StringRef(attr.name()), rapidjson::StringRef(attr.value()), allocator_source_document.GetAllocator()); + if (!is_flattened) { + output_value.AddMember(rapidjson::StringRef(attr.name()), rapidjson::StringRef(attr.value()), allocator); + } else { + output_value.AddMember(rapidjson::Value(*prefix_for_flat_structure + attr.name(), allocator).Move(), rapidjson::StringRef(attr.value()), allocator); + } } for (const auto& child: source_node.children()) { if (child.type() == pugi::xml_node_type::node_element) { const auto is_pcdata = [](const pugi::xml_node& node) { return node.type() == pugi::xml_node_type::node_pcdata; }; if (std::all_of(child.children().begin(), child.children().end(), is_pcdata)) { // all children are pcdata (text): leaf node - output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::StringRef(child.text().get()), allocator_source_document.GetAllocator()); + if (!is_flattened) { + output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::StringRef(child.text().get()), allocator); + } else { + output_value.AddMember(rapidjson::Value(*prefix_for_flat_structure + child.name(), allocator).Move(), rapidjson::StringRef(child.text().get()), allocator); + } } else { // there are non-text children: recurse further - auto& child_val = flatten ? output_value : output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::kObjectType, allocator_source_document.GetAllocator())[child.name()]; - simplifiedGenericXmlToJson(child, child_val, allocator_source_document, flatten); + auto& child_val = is_flattened ? output_value : output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::kObjectType, allocator)[child.name()]; + auto new_prefix = is_flattened ? std::optional(*prefix_for_flat_structure + child.name() + ".") : std::nullopt; + simplifiedGenericXmlToJson(child, child_val, allocator, new_prefix); } } } } +std::string createUniqueKey(const std::string& key, const rapidjson::Value& parent) { + auto proposed_key = key; + size_t postfix = 1; + while (parent.HasMember(proposed_key)) { + proposed_key = key + std::to_string(postfix++); + } + return proposed_key; +} + rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) { rapidjson::Document doc{rapidjson::kObjectType}; @@ -135,20 +158,24 @@ rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) { { auto eventData_xml = event_xml.child("EventData"); - // create EventData subarray even if flatten requested - doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator()); - for (const auto& data : eventData_xml.children()) { - auto name_attr = data.attribute("Name"); - rapidjson::Value item(rapidjson::kObjectType); - item.AddMember("Name", rapidjson::StringRef(name_attr.value()), doc.GetAllocator()); - item.AddMember("Content", rapidjson::StringRef(data.text().get()), doc.GetAllocator()); - item.AddMember("Type", rapidjson::StringRef(data.name()), doc.GetAllocator()); - // we need to query EventData because a reference to it wouldn't be stable, as we - // possibly add members to its parent which could result in reallocation - doc["EventData"].PushBack(item, doc.GetAllocator()); - // check collision - if (flatten && !name_attr.empty() && !doc.HasMember(name_attr.value())) { - doc.AddMember(rapidjson::StringRef(name_attr.value()), rapidjson::StringRef(data.text().get()), doc.GetAllocator()); + if (flatten) { + for (const auto& event_data_child : eventData_xml.children()) { + std::string key = "EventData"; + if (auto name_attr = event_data_child.attribute("Name"); !name_attr.empty()) { + key = utils::StringUtils::join_pack(key, ".", name_attr.value()); + } + + doc.AddMember(rapidjson::Value(createUniqueKey(key, doc), doc.GetAllocator()).Move(), rapidjson::StringRef(event_data_child.text().get()), doc.GetAllocator()); + } + } else { + doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator()); + for (const auto& event_data_child : eventData_xml.children()) { + auto name_attr = event_data_child.attribute("Name"); + rapidjson::Value item(rapidjson::kObjectType); + item.AddMember("Name", rapidjson::StringRef(name_attr.value()), doc.GetAllocator()); + item.AddMember("Content", rapidjson::StringRef(event_data_child.text().get()), doc.GetAllocator()); + item.AddMember("Type", rapidjson::StringRef(event_data_child.name()), doc.GetAllocator()); + doc["EventData"].PushBack(item, doc.GetAllocator()); // we need to re-query EventData because a reference to it wouldn't be stable } } } @@ -156,7 +183,8 @@ rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) { const auto userdata_xml = event_xml.child("UserData"); if (!userdata_xml.empty()) { auto& userdata = flatten ? doc : doc.AddMember("UserData", rapidjson::kObjectType, doc.GetAllocator())["UserData"]; - simplifiedGenericXmlToJson(userdata_xml, userdata, doc, flatten); + auto prefix = flatten ? std::optional("UserData.") : std::nullopt; + simplifiedGenericXmlToJson(userdata_xml, userdata, doc.GetAllocator(), prefix); } return doc;
