martinzink commented on a change in pull request #1066: URL: https://github.com/apache/nifi-minifi-cpp/pull/1066#discussion_r631058529
########## File path: libminifi/test/unit/MemoryUsageTest.cpp ########## @@ -20,22 +20,22 @@ #include "utils/OsUtils.h" #include "../TestBase.h" -TEST_CASE("Test memory usage", "[testmemoryusage]") { +TEST_CASE("Test Physical memory usage", "[testphysicalmemoryusage]") { constexpr bool cout_enabled = true; Review comment: These hardware/platform specific functions are hard to test (because they depend on a lot of things). I added cout the these tests so one can manually check them (with 3rd party tools) that they return the correct data. I also wanted to give a simple switch, if sometime we want to mute these tests. ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); + delete counter; + } else { + valid_counters.push_back(counter); + } + } else { + valid_counters.push_back(counter); + } + } + resource_consumption_counters_ = valid_counters; + PdhCollectQueryData(pdh_query_); +} + +void PerformanceDataMonitor::onTrigger(core::ProcessContext* context, core::ProcessSession* session) { + if (resource_consumption_counters_.empty()) { + logger_->log_error("No valid counters for PerformanceDataMonitor"); + return; + } + + std::shared_ptr<core::FlowFile> flowFile = session->create(); + if (!flowFile) { + logger_->log_error("Failed to create flowfile!"); + return; + } + + if (pdh_query_ != nullptr) + PdhCollectQueryData(pdh_query_); + + rapidjson::Document root = rapidjson::Document(rapidjson::kObjectType); + rapidjson::Value& body = prepareJSONBody(root); + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*>(counter); + if (pdh_counter != nullptr && pdh_counter->collectData() != ERROR_SUCCESS) + continue; + counter->addToJson(body, root.GetAllocator()); + } + PerformanceDataMonitor::WriteCallback callback(std::move(root)); + session->write(flowFile, &callback); + session->transfer(flowFile, Success); +} + +void PerformanceDataMonitor::initialize(void) { + setSupportedProperties({ CustomPDHCounters, PredefinedGroups, OutputFormatProperty }); + setSupportedRelationships({ PerformanceDataMonitor::Success }); +} + +rapidjson::Value& PerformanceDataMonitor::prepareJSONBody(rapidjson::Document& root) { + switch (output_format_) { + case (OutputFormat::kOpenTelemetry): + root.AddMember("Name", "PerformanceData", root.GetAllocator()); + root.AddMember("Timestamp", std::time(0), root.GetAllocator()); + root.AddMember("Body", rapidjson::Value{ rapidjson::kObjectType }, root.GetAllocator()); + return root["Body"]; + case (OutputFormat::kJSON): + return root; + default: + return root; + } +} + +void add_cpu_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% Processor Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% User Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% Privileged Time")); +} + +void add_io_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(_Total)\\IO Read Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(_Total)\\IO Write Bytes/sec")); +} + +void add_disk_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\LogicalDisk(*)\\% Free Space")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\LogicalDisk(*)\\Free Megabytes", false)); + + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Read Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Write Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Idle Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Transfer")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Read")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Write")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Write Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Read Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Transfer")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Read")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Write")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Current Disk Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Transfers/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Reads/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Writes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Read Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Write Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Split IO/Sec")); +} + +void add_network_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Bytes Received/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Bytes Sent/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Bytes Total/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Current Bandwidth")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Sent/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Discarded", false)); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Errors", false)); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Unknown", false)); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Non-Unicast/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Unicast/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Sent Unicast/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Sent Non-Unicast/sec")); +} + +void add_memory_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Memory\\% Committed Bytes In Use")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Memory\\Available MBytes")); Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); + delete counter; + } else { + valid_counters.push_back(counter); + } + } else { + valid_counters.push_back(counter); + } + } + resource_consumption_counters_ = valid_counters; + PdhCollectQueryData(pdh_query_); +} + +void PerformanceDataMonitor::onTrigger(core::ProcessContext* context, core::ProcessSession* session) { + if (resource_consumption_counters_.empty()) { + logger_->log_error("No valid counters for PerformanceDataMonitor"); + return; + } + + std::shared_ptr<core::FlowFile> flowFile = session->create(); + if (!flowFile) { + logger_->log_error("Failed to create flowfile!"); + return; + } + + if (pdh_query_ != nullptr) + PdhCollectQueryData(pdh_query_); + + rapidjson::Document root = rapidjson::Document(rapidjson::kObjectType); + rapidjson::Value& body = prepareJSONBody(root); + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*>(counter); + if (pdh_counter != nullptr && pdh_counter->collectData() != ERROR_SUCCESS) + continue; + counter->addToJson(body, root.GetAllocator()); + } + PerformanceDataMonitor::WriteCallback callback(std::move(root)); + session->write(flowFile, &callback); + session->transfer(flowFile, Success); +} + +void PerformanceDataMonitor::initialize(void) { + setSupportedProperties({ CustomPDHCounters, PredefinedGroups, OutputFormatProperty }); + setSupportedRelationships({ PerformanceDataMonitor::Success }); +} + +rapidjson::Value& PerformanceDataMonitor::prepareJSONBody(rapidjson::Document& root) { + switch (output_format_) { + case (OutputFormat::kOpenTelemetry): + root.AddMember("Name", "PerformanceData", root.GetAllocator()); + root.AddMember("Timestamp", std::time(0), root.GetAllocator()); + root.AddMember("Body", rapidjson::Value{ rapidjson::kObjectType }, root.GetAllocator()); + return root["Body"]; + case (OutputFormat::kJSON): + return root; + default: + return root; + } +} + +void add_cpu_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% Processor Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% User Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% Privileged Time")); +} + +void add_io_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(_Total)\\IO Read Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(_Total)\\IO Write Bytes/sec")); +} + +void add_disk_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\LogicalDisk(*)\\% Free Space")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\LogicalDisk(*)\\Free Megabytes", false)); + + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Read Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Write Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Idle Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Transfer")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Read")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Write")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Write Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Read Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Transfer")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Read")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Write")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Current Disk Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Transfers/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Reads/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Writes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Read Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Write Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Split IO/Sec")); +} + +void add_network_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Bytes Received/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Bytes Sent/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Bytes Total/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Current Bandwidth")); Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); + delete counter; + } else { + valid_counters.push_back(counter); + } + } else { + valid_counters.push_back(counter); + } + } + resource_consumption_counters_ = valid_counters; + PdhCollectQueryData(pdh_query_); +} + +void PerformanceDataMonitor::onTrigger(core::ProcessContext* context, core::ProcessSession* session) { + if (resource_consumption_counters_.empty()) { + logger_->log_error("No valid counters for PerformanceDataMonitor"); + return; + } + + std::shared_ptr<core::FlowFile> flowFile = session->create(); + if (!flowFile) { + logger_->log_error("Failed to create flowfile!"); + return; + } + + if (pdh_query_ != nullptr) + PdhCollectQueryData(pdh_query_); + + rapidjson::Document root = rapidjson::Document(rapidjson::kObjectType); + rapidjson::Value& body = prepareJSONBody(root); + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*>(counter); + if (pdh_counter != nullptr && pdh_counter->collectData() != ERROR_SUCCESS) + continue; + counter->addToJson(body, root.GetAllocator()); + } + PerformanceDataMonitor::WriteCallback callback(std::move(root)); + session->write(flowFile, &callback); + session->transfer(flowFile, Success); +} + +void PerformanceDataMonitor::initialize(void) { + setSupportedProperties({ CustomPDHCounters, PredefinedGroups, OutputFormatProperty }); + setSupportedRelationships({ PerformanceDataMonitor::Success }); +} + +rapidjson::Value& PerformanceDataMonitor::prepareJSONBody(rapidjson::Document& root) { + switch (output_format_) { + case (OutputFormat::kOpenTelemetry): + root.AddMember("Name", "PerformanceData", root.GetAllocator()); + root.AddMember("Timestamp", std::time(0), root.GetAllocator()); + root.AddMember("Body", rapidjson::Value{ rapidjson::kObjectType }, root.GetAllocator()); + return root["Body"]; + case (OutputFormat::kJSON): + return root; + default: + return root; + } +} + +void add_cpu_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% Processor Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% User Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% Privileged Time")); +} + +void add_io_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(_Total)\\IO Read Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(_Total)\\IO Write Bytes/sec")); +} + +void add_disk_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\LogicalDisk(*)\\% Free Space")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\LogicalDisk(*)\\Free Megabytes", false)); + + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Read Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Write Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Idle Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Transfer")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Read")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Write")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Write Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Read Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Transfer")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Read")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Write")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Current Disk Queue Length")); Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); + delete counter; + } else { + valid_counters.push_back(counter); + } + } else { + valid_counters.push_back(counter); + } + } + resource_consumption_counters_ = valid_counters; + PdhCollectQueryData(pdh_query_); Review comment: yeah I agree, in changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); + delete counter; + } else { + valid_counters.push_back(counter); + } + } else { + valid_counters.push_back(counter); + } + } + resource_consumption_counters_ = valid_counters; + PdhCollectQueryData(pdh_query_); +} + +void PerformanceDataMonitor::onTrigger(core::ProcessContext* context, core::ProcessSession* session) { + if (resource_consumption_counters_.empty()) { + logger_->log_error("No valid counters for PerformanceDataMonitor"); + return; Review comment: changed both in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); Review comment: yeah thats better, changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.h ########## @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <pdh.h> +#include <string> +#include <vector> +#include <memory> +#include <utility> + +#include "core/Processor.h" + +#include "rapidjson/stream.h" +#include "rapidjson/writer.h" +#include "rapidjson/prettywriter.h" + +#include "PerformanceDataCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// PerformanceDataMonitor Class +class PerformanceDataMonitor : public core::Processor { + public: + static constexpr char const* JSON_FORMAT_STR = "JSON"; + static constexpr char const* OPEN_TELEMETRY_FORMAT_STR = "OpenTelemetry"; + + explicit PerformanceDataMonitor(const std::string& name, utils::Identifier uuid = utils::Identifier()) + : Processor(name, uuid), output_format_(OutputFormat::kJSON), + logger_(logging::LoggerFactory<PerformanceDataMonitor>::getLogger()), + pdh_query_(nullptr), resource_consumption_counters_() { + } + ~PerformanceDataMonitor() override; + static constexpr char const* ProcessorName = "PerformanceDataMonitor"; + // Supported Properties + static core::Property PredefinedGroups; + static core::Property CustomPDHCounters; + static core::Property OutputFormatProperty; + // Supported Relationships + static core::Relationship Success; + // Nest Callback Class for write stream + class WriteCallback : public OutputStreamCallback { + public: + explicit WriteCallback(rapidjson::Document&& root) : root_(std::move(root)) { + } + rapidjson::Document root_; + int64_t process(const std::shared_ptr<io::BaseStream>& stream) { + rapidjson::StringBuffer buffer; + rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer); + root_.Accept(writer); + return stream->write(reinterpret_cast<const uint8_t*>(buffer.GetString()), gsl::narrow<int>(buffer.GetSize())); + } + }; + + public: + void onSchedule(const std::shared_ptr<core::ProcessContext> &context, const std::shared_ptr<core::ProcessSessionFactory> &sessionFactory) override; + void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override; + void initialize(void) override; Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.h ########## @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <pdh.h> +#include <string> +#include <vector> +#include <memory> +#include <utility> + +#include "core/Processor.h" + +#include "rapidjson/stream.h" +#include "rapidjson/writer.h" +#include "rapidjson/prettywriter.h" + +#include "PerformanceDataCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// PerformanceDataMonitor Class +class PerformanceDataMonitor : public core::Processor { + public: + static constexpr char const* JSON_FORMAT_STR = "JSON"; + static constexpr char const* OPEN_TELEMETRY_FORMAT_STR = "OpenTelemetry"; + + explicit PerformanceDataMonitor(const std::string& name, utils::Identifier uuid = utils::Identifier()) + : Processor(name, uuid), output_format_(OutputFormat::kJSON), + logger_(logging::LoggerFactory<PerformanceDataMonitor>::getLogger()), + pdh_query_(nullptr), resource_consumption_counters_() { + } + ~PerformanceDataMonitor() override; + static constexpr char const* ProcessorName = "PerformanceDataMonitor"; + // Supported Properties + static core::Property PredefinedGroups; + static core::Property CustomPDHCounters; + static core::Property OutputFormatProperty; + // Supported Relationships + static core::Relationship Success; + // Nest Callback Class for write stream + class WriteCallback : public OutputStreamCallback { Review comment: good idea, moved it into libminifi/include/utils/JsonCallback.h as JsonOutputCallback in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.h ########## @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <pdh.h> +#include <string> +#include <vector> +#include <memory> +#include <utility> + +#include "core/Processor.h" + +#include "rapidjson/stream.h" +#include "rapidjson/writer.h" +#include "rapidjson/prettywriter.h" + +#include "PerformanceDataCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// PerformanceDataMonitor Class +class PerformanceDataMonitor : public core::Processor { + public: + static constexpr char const* JSON_FORMAT_STR = "JSON"; + static constexpr char const* OPEN_TELEMETRY_FORMAT_STR = "OpenTelemetry"; Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PDHCounters.cpp ########## @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PDHCounters.h" +#include "utils/StringUtils.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +DWORD PDHCounterBase::getDWFormat() const { + return is_double_format_ ? PDH_FMT_DOUBLE : PDH_FMT_LARGE; +} + +PDHCounterBase* PDHCounterBase::createPDHCounter(const std::string& query_name, bool is_double) { + auto groups = utils::StringUtils::split(query_name, "\\"); + if (groups.size() != 2 || query_name.substr(0, 1) != "\\") + return nullptr; + if (query_name.find("(*)") != std::string::npos) { + return new PDHCounterArray(query_name, is_double); + } else { + return new PDHCounter(query_name, is_double); + } +} + +const std::string& PDHCounterBase::getName() const { + return pdh_english_counter_name_; +} + +std::string PDHCounterBase::getObjectName() const { + auto groups = utils::StringUtils::split(pdh_english_counter_name_, "\\"); + return groups[0]; +} + +std::string PDHCounterBase::getCounterName() const { + auto groups = utils::StringUtils::split(pdh_english_counter_name_, "\\"); + return groups[1]; +} + +void PDHCounter::addToJson(rapidjson::Value& body, rapidjson::Document::AllocatorType& alloc) const { + rapidjson::Value key(getCounterName().c_str(), getCounterName().length(), alloc); + rapidjson::Value& group_node = acquireNode(getObjectName(), body, alloc); + group_node.AddMember(key, getValue(), alloc); +} + +PDH_STATUS PDHCounter::addToQuery(PDH_HQUERY& pdh_query) { + return PdhAddEnglishCounter(pdh_query, pdh_english_counter_name_.c_str(), NULL, &counter_); Review comment: good idea, changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/MemoryConsumptionCounter.h ########## @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "PerformanceDataCounter.h" +#include <string> +#include "utils/OsUtils.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +class MemoryConsumptionCounter : public PerformanceDataCounter { + public: + MemoryConsumptionCounter() { + } + + void addToJson(rapidjson::Value& body, rapidjson::Document::AllocatorType& alloc) const override { + rapidjson::Value& group_node = acquireNode(std::string("Memory"), body, alloc); + + rapidjson::Value total_pysical_memory; Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: CMakeLists.txt ########## @@ -501,6 +501,14 @@ if (ENABLE_ALL OR ENABLE_AWS) createExtension(AWS-EXTENSIONS "AWS EXTENSIONS" "This enables AWS support" "extensions/aws" "${TEST_DIR}/aws-tests") endif() +## PDH Extentions Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) (fun fact: there were multiple typos there the correct spelling was the most rare I think, changed them all) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); + delete counter; + } else { + valid_counters.push_back(counter); + } + } else { + valid_counters.push_back(counter); + } + } + resource_consumption_counters_ = valid_counters; + PdhCollectQueryData(pdh_query_); +} + +void PerformanceDataMonitor::onTrigger(core::ProcessContext* context, core::ProcessSession* session) { + if (resource_consumption_counters_.empty()) { + logger_->log_error("No valid counters for PerformanceDataMonitor"); + return; + } + + std::shared_ptr<core::FlowFile> flowFile = session->create(); + if (!flowFile) { + logger_->log_error("Failed to create flowfile!"); + return; + } + + if (pdh_query_ != nullptr) + PdhCollectQueryData(pdh_query_); + + rapidjson::Document root = rapidjson::Document(rapidjson::kObjectType); + rapidjson::Value& body = prepareJSONBody(root); + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*>(counter); + if (pdh_counter != nullptr && pdh_counter->collectData() != ERROR_SUCCESS) + continue; + counter->addToJson(body, root.GetAllocator()); + } + PerformanceDataMonitor::WriteCallback callback(std::move(root)); + session->write(flowFile, &callback); + session->transfer(flowFile, Success); +} + +void PerformanceDataMonitor::initialize(void) { + setSupportedProperties({ CustomPDHCounters, PredefinedGroups, OutputFormatProperty }); + setSupportedRelationships({ PerformanceDataMonitor::Success }); +} + +rapidjson::Value& PerformanceDataMonitor::prepareJSONBody(rapidjson::Document& root) { + switch (output_format_) { + case (OutputFormat::kOpenTelemetry): + root.AddMember("Name", "PerformanceData", root.GetAllocator()); + root.AddMember("Timestamp", std::time(0), root.GetAllocator()); + root.AddMember("Body", rapidjson::Value{ rapidjson::kObjectType }, root.GetAllocator()); + return root["Body"]; + case (OutputFormat::kJSON): + return root; + default: + return root; + } +} + +void add_cpu_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% Processor Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% User Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Processor(*)\\% Privileged Time")); +} + +void add_io_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(_Total)\\IO Read Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(_Total)\\IO Write Bytes/sec")); +} + +void add_disk_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\LogicalDisk(*)\\% Free Space")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\LogicalDisk(*)\\Free Megabytes", false)); + + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Read Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Disk Write Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\% Idle Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Transfer")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Read")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Bytes/Write")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Write Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Read Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Transfer")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Read")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Avg. Disk sec/Write")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Current Disk Queue Length")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Transfers/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Reads/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Writes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Read Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Disk Write Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\PhysicalDisk(*)\\Split IO/Sec")); +} + +void add_network_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Bytes Received/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Bytes Sent/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Bytes Total/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Current Bandwidth")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Sent/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Discarded", false)); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Errors", false)); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Unknown", false)); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Non-Unicast/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Received Unicast/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Sent Unicast/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Network Interface(*)\\Packets Sent Non-Unicast/sec")); +} + +void add_memory_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Memory\\% Committed Bytes In Use")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Memory\\Available MBytes")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Memory\\Page Faults/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Memory\\Pages/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Paging File(_Total)\\% Usage")); + + resource_consumption_counters.push_back(new MemoryConsumptionCounter()); +} + +void add_process_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(*)\\% Processor Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(*)\\Elapsed Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(*)\\ID Process", false)); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\Process(*)\\Private Bytes", false)); +} + +void add_system_related_counters(std::vector<PerformanceDataCounter*>& resource_consumption_counters) { + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\% Registry Quota In Use")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\Context Switches/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\File Control Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\File Control Operations/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\File Read Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\File Read Operations/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\File Write Bytes/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\File Write Operations/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\File Data Operations/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\Processes", false)); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\Processor Queue Length", false)); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\System Calls/sec")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\System Up Time")); + resource_consumption_counters.push_back(PDHCounterBase::createPDHCounter("\\System\\Threads", false)); +} + +void PerformanceDataMonitor::addCountersFromPredefinedGroupsProperty(const std::string& predefined_groups) { + auto groups = utils::StringUtils::split(predefined_groups, ","); + for (const auto& group : groups) { Review comment: good idea changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> Review comment: nice catch changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/tests/PerformanceDataMonitorTests.cpp ########## @@ -0,0 +1,268 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <memory> +#include <string> +#include <vector> +#include <set> +#include <fstream> + +#include "TestBase.h" +#include "processors/PutFile.h" +#include "utils/file/FileUtils.h" +#include "PerformanceDataMonitor.h" +#include "rapidjson/filereadstream.h" + +using PutFile = org::apache::nifi::minifi::processors::PutFile; +using PerformanceDataMonitor = org::apache::nifi::minifi::processors::PerformanceDataMonitor; +using PerformanceDataCounter = org::apache::nifi::minifi::processors::PerformanceDataCounter; + +class PerformanceDataMonitorTester { + public: + PerformanceDataMonitorTester() { + LogTestController::getInstance().setTrace<TestPlan>(); + dir_ = test_controller_.createTempDirectory("/tmp/gt.XXXXXX"); + plan_ = test_controller_.createPlan(); + performance_monitor_ = plan_->addProcessor("PerformanceDataMonitor", "pdhsys"); + putfile_ = plan_->addProcessor("PutFile", "putfile", core::Relationship("success", "description"), true); + plan_->setProperty(putfile_, PutFile::Directory.getName(), dir_); + } + + void runProcessors() { + plan_->runNextProcessor(); // PerformanceMonitor + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + plan_->runCurrentProcessor(); // PerformanceMonitor + plan_->runNextProcessor(); // PutFile + plan_->runCurrentProcessor(); // PutFile + } + + void setPerformanceMonitorProperty(core::Property property, const std::string& value) { + plan_->setProperty(performance_monitor_, property.getName(), value); + } + + TestController test_controller_; + std::string dir_; + std::shared_ptr<TestPlan> plan_; + std::shared_ptr<core::Processor> performance_monitor_; + std::shared_ptr<core::Processor> putfile_; +}; + + +TEST_CASE("PerformanceDataMonitorEmptyPropertiesTest", "[performancedatamonitoremptypropertiestest]") { + PerformanceDataMonitorTester tester; + tester.runProcessors(); + + REQUIRE(tester.test_controller_.getLog().getInstance().contains("No valid counters for PerformanceDataMonitor", std::chrono::seconds(0))); + + uint32_t number_of_flowfiles = 0; + auto lambda = [&number_of_flowfiles](const std::string& path, const std::string& filename) -> bool { + ++number_of_flowfiles; + return true; + }; + + utils::file::FileUtils::list_dir(tester.dir_, lambda, tester.plan_->getLogger(), false); + REQUIRE(number_of_flowfiles == 0); +} + +TEST_CASE("PerformanceDataMonitorPartiallyInvalidGroupPropertyTest", "[performancedatamonitorpartiallyinvalidgrouppropertytest]") { + PerformanceDataMonitorTester tester; + tester.setPerformanceMonitorProperty(PerformanceDataMonitor::PredefinedGroups, "Disk,CPU,Asd"); + tester.setPerformanceMonitorProperty(PerformanceDataMonitor::CustomPDHCounters, "\\Invalid\\Counter,\\System\\Processes"); + tester.runProcessors(); + + REQUIRE(tester.test_controller_.getLog().getInstance().contains("Asd is not a valid predefined group", std::chrono::seconds(0))); + REQUIRE(tester.test_controller_.getLog().getInstance().contains("Error adding \\Invalid\\Counter to query", std::chrono::seconds(0))); + + uint32_t number_of_flowfiles = 0; + + auto lambda = [&number_of_flowfiles](const std::string& path, const std::string& filename) -> bool { + ++number_of_flowfiles; + FILE* fp = fopen((path + utils::file::FileUtils::get_separator() + filename).c_str(), "r"); + REQUIRE(fp != nullptr); + char readBuffer[500]; + rapidjson::FileReadStream is(fp, readBuffer, sizeof(readBuffer)); + rapidjson::Document d; Review comment: yeah thats valid :) changed it to document in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.h ########## @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <pdh.h> +#include <string> +#include <vector> +#include <memory> +#include <utility> + +#include "core/Processor.h" + +#include "rapidjson/stream.h" +#include "rapidjson/writer.h" +#include "rapidjson/prettywriter.h" + +#include "PerformanceDataCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// PerformanceDataMonitor Class +class PerformanceDataMonitor : public core::Processor { + public: + static constexpr char const* JSON_FORMAT_STR = "JSON"; + static constexpr char const* OPEN_TELEMETRY_FORMAT_STR = "OpenTelemetry"; + + explicit PerformanceDataMonitor(const std::string& name, utils::Identifier uuid = utils::Identifier()) + : Processor(name, uuid), output_format_(OutputFormat::kJSON), + logger_(logging::LoggerFactory<PerformanceDataMonitor>::getLogger()), + pdh_query_(nullptr), resource_consumption_counters_() { + } + ~PerformanceDataMonitor() override; + static constexpr char const* ProcessorName = "PerformanceDataMonitor"; + // Supported Properties + static core::Property PredefinedGroups; + static core::Property CustomPDHCounters; + static core::Property OutputFormatProperty; + // Supported Relationships + static core::Relationship Success; + // Nest Callback Class for write stream + class WriteCallback : public OutputStreamCallback { + public: + explicit WriteCallback(rapidjson::Document&& root) : root_(std::move(root)) { + } + rapidjson::Document root_; + int64_t process(const std::shared_ptr<io::BaseStream>& stream) { + rapidjson::StringBuffer buffer; + rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer); + root_.Accept(writer); + return stream->write(reinterpret_cast<const uint8_t*>(buffer.GetString()), gsl::narrow<int>(buffer.GetSize())); + } + }; + + public: + void onSchedule(const std::shared_ptr<core::ProcessContext> &context, const std::shared_ptr<core::ProcessSessionFactory> &sessionFactory) override; + void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override; + void initialize(void) override; + + protected: + enum class OutputFormat { + kJSON, kOpenTelemetry Review comment: Yeah i agree. I changed it to be ALLCAPS (this one was the most common I think) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); + delete counter; + } else { + valid_counters.push_back(counter); + } + } else { + valid_counters.push_back(counter); + } + } + resource_consumption_counters_ = valid_counters; + PdhCollectQueryData(pdh_query_); +} + +void PerformanceDataMonitor::onTrigger(core::ProcessContext* context, core::ProcessSession* session) { + if (resource_consumption_counters_.empty()) { + logger_->log_error("No valid counters for PerformanceDataMonitor"); + return; + } + + std::shared_ptr<core::FlowFile> flowFile = session->create(); + if (!flowFile) { + logger_->log_error("Failed to create flowfile!"); + return; + } + + if (pdh_query_ != nullptr) + PdhCollectQueryData(pdh_query_); + + rapidjson::Document root = rapidjson::Document(rapidjson::kObjectType); + rapidjson::Value& body = prepareJSONBody(root); + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*>(counter); + if (pdh_counter != nullptr && pdh_counter->collectData() != ERROR_SUCCESS) + continue; + counter->addToJson(body, root.GetAllocator()); + } + PerformanceDataMonitor::WriteCallback callback(std::move(root)); + session->write(flowFile, &callback); + session->transfer(flowFile, Success); +} + +void PerformanceDataMonitor::initialize(void) { + setSupportedProperties({ CustomPDHCounters, PredefinedGroups, OutputFormatProperty }); + setSupportedRelationships({ PerformanceDataMonitor::Success }); +} + +rapidjson::Value& PerformanceDataMonitor::prepareJSONBody(rapidjson::Document& root) { + switch (output_format_) { + case (OutputFormat::kOpenTelemetry): Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); + delete counter; Review comment: changed it in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); Review comment: yeah, good idea I changed NULL to nullptr in cases when it was ptr type and into 0 when the argument was integer (e.g. PdhAddEnglishCounterA after many typedefs wants an unsigned __int64). ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); Review comment: Yeah I agree. The whole point of this processor to be able to use the windows built in performance metrics (but there are a few metrics which cannot be find in here (mostly static information e.g. total memory)). I created the parent class so additional metrics can be added from different sources. Since these MemoryConsumptionCounters are not collecting from PDH it would misleading to derived the class from them. I restructed the code a little bit so we dont have to cast only at the begining (when the PDH counters are registered into the pdh query) Now there are 2 virtual functions one for data collection and one for writing the data to the json struct. These functions are valid for the pdh and the non-pdh classes as well, and also follows more strictly the single-responsibility principle. (before data collection and writing was done in the same function) Hope its more readable now. [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataCounter.h ########## @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include <string> +#include "rapidjson/document.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +class PerformanceDataCounter { + public: + PerformanceDataCounter() = default; + virtual ~PerformanceDataCounter() {} Review comment: changed in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PDHCounters.h ########## @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <TCHAR.h> +#include <pdh.h> +#include <pdhmsg.h> +#include <string> + +#include "PerformanceDataCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +class PDHCounterBase : public PerformanceDataCounter { + public: + virtual ~PDHCounterBase() {} + static PDHCounterBase* createPDHCounter(const std::string& query_name, bool is_double = true); + + const std::string& getName() const; + virtual std::string getObjectName() const; + virtual std::string getCounterName() const; + virtual PDH_STATUS addToQuery(PDH_HQUERY& pdh_query) = 0; + virtual PDH_STATUS collectData() = 0; + protected: + PDHCounterBase(const std::string& query_name, bool is_double) : PerformanceDataCounter(), + is_double_format_(is_double), pdh_english_counter_name_(query_name) { + } + DWORD getDWFormat() const; + const bool is_double_format_; + std::string pdh_english_counter_name_; +}; + +class PDHCounter : public PDHCounterBase { + public: + void addToJson(rapidjson::Value& body, rapidjson::Document::AllocatorType& alloc) const override; + + PDH_STATUS addToQuery(PDH_HQUERY& pdh_query) override; + PDH_STATUS collectData() override; + + protected: + friend PDHCounterBase* PDHCounterBase::createPDHCounter(const std::string& query_name, bool is_double); Review comment: Yeah, the pdh counter naming convention is strict so I wanted to prevent these classes to be created with invalid names. This makes accessor functions to be clean (without checking boilerplate) ########## File path: extensions/pdh/PDHCounters.h ########## @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <TCHAR.h> +#include <pdh.h> +#include <pdhmsg.h> +#include <string> + +#include "PerformanceDataCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +class PDHCounterBase : public PerformanceDataCounter { + public: + virtual ~PDHCounterBase() {} + static PDHCounterBase* createPDHCounter(const std::string& query_name, bool is_double = true); + + const std::string& getName() const; + virtual std::string getObjectName() const; + virtual std::string getCounterName() const; + virtual PDH_STATUS addToQuery(PDH_HQUERY& pdh_query) = 0; + virtual PDH_STATUS collectData() = 0; + protected: + PDHCounterBase(const std::string& query_name, bool is_double) : PerformanceDataCounter(), + is_double_format_(is_double), pdh_english_counter_name_(query_name) { + } + DWORD getDWFormat() const; + const bool is_double_format_; + std::string pdh_english_counter_name_; +}; + +class PDHCounter : public PDHCounterBase { + public: + void addToJson(rapidjson::Value& body, rapidjson::Document::AllocatorType& alloc) const override; + + PDH_STATUS addToQuery(PDH_HQUERY& pdh_query) override; + PDH_STATUS collectData() override; + + protected: + friend PDHCounterBase* PDHCounterBase::createPDHCounter(const std::string& query_name, bool is_double); + explicit PDHCounter(const std::string& query_name, bool is_double) : PDHCounterBase(query_name, is_double), + counter_(), current_value_() { Review comment: changed counter_ to be initialized with nullptr, left current_value_ -s explicitly initialised,because its a struct without in-class initializers, causing https://docs.microsoft.com/en-us/cpp/code-quality/c26495 warning ########## File path: extensions/pdh/PDHCounters.h ########## @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <TCHAR.h> +#include <pdh.h> +#include <pdhmsg.h> +#include <string> + +#include "PerformanceDataCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +class PDHCounterBase : public PerformanceDataCounter { Review comment: changed the names in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PDHCounters.cpp ########## @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PDHCounters.h" +#include "utils/StringUtils.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +DWORD PDHCounterBase::getDWFormat() const { + return is_double_format_ ? PDH_FMT_DOUBLE : PDH_FMT_LARGE; +} + +PDHCounterBase* PDHCounterBase::createPDHCounter(const std::string& query_name, bool is_double) { + auto groups = utils::StringUtils::split(query_name, "\\"); + if (groups.size() != 2 || query_name.substr(0, 1) != "\\") + return nullptr; + if (query_name.find("(*)") != std::string::npos) { + return new PDHCounterArray(query_name, is_double); + } else { + return new PDHCounter(query_name, is_double); + } +} + +const std::string& PDHCounterBase::getName() const { + return pdh_english_counter_name_; +} + +std::string PDHCounterBase::getObjectName() const { + auto groups = utils::StringUtils::split(pdh_english_counter_name_, "\\"); + return groups[0]; +} + +std::string PDHCounterBase::getCounterName() const { + auto groups = utils::StringUtils::split(pdh_english_counter_name_, "\\"); + return groups[1]; +} + +void PDHCounter::addToJson(rapidjson::Value& body, rapidjson::Document::AllocatorType& alloc) const { + rapidjson::Value key(getCounterName().c_str(), getCounterName().length(), alloc); + rapidjson::Value& group_node = acquireNode(getObjectName(), body, alloc); + group_node.AddMember(key, getValue(), alloc); +} + +PDH_STATUS PDHCounter::addToQuery(PDH_HQUERY& pdh_query) { + return PdhAddEnglishCounter(pdh_query, pdh_english_counter_name_.c_str(), NULL, &counter_); +} + +PDH_STATUS PDHCounter::collectData() { + return PdhGetFormattedCounterValue(counter_, getDWFormat(), NULL, ¤t_value_); +} Review comment: yeah, good idea I changed NULL to nullptr in cases when it was ptr type and into 0 when the argument was integer (e.g. PdhAddEnglishCounterA after many typedefs wants an unsigned __int64). ########## File path: extensions/pdh/PDHCounters.cpp ########## @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PDHCounters.h" +#include "utils/StringUtils.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +DWORD PDHCounterBase::getDWFormat() const { + return is_double_format_ ? PDH_FMT_DOUBLE : PDH_FMT_LARGE; +} + +PDHCounterBase* PDHCounterBase::createPDHCounter(const std::string& query_name, bool is_double) { Review comment: changed the raw ptrs to be unique_ptr-s in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/MemoryConsumptionCounter.h ########## @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "PerformanceDataCounter.h" +#include <string> +#include "utils/OsUtils.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +class MemoryConsumptionCounter : public PerformanceDataCounter { + public: + MemoryConsumptionCounter() { Review comment: removed in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/PerformanceDataMonitor.cpp ########## @@ -0,0 +1,283 @@ +/** + * @file GenerateFlowFile.cpp + * GenerateFlowFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PerformanceDataMonitor.h" +#include "PDHCounters.h" +#include "MemoryConsumptionCounter.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +core::Relationship PerformanceDataMonitor::Success("success", "All files are routed to success"); + +core::Property PerformanceDataMonitor::PredefinedGroups( + core::PropertyBuilder::createProperty("Predefined Groups")-> + withDescription("Comma separated list to which predefined groups should be included")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::CustomPDHCounters( + core::PropertyBuilder::createProperty("Custom PDH Counters")-> + withDescription("Comma separated list of PDHCounters to collect from")-> + withDefaultValue("")->build()); + +core::Property PerformanceDataMonitor::OutputFormatProperty( + core::PropertyBuilder::createProperty("Output format")-> + withDescription("Format of the created flowfiles")-> + withAllowableValue<std::string>(JSON_FORMAT_STR)-> + withAllowableValue(OPEN_TELEMETRY_FORMAT_STR)-> + withDefaultValue(JSON_FORMAT_STR)->build()); + +PerformanceDataMonitor::~PerformanceDataMonitor() { + if (pdh_query_ != nullptr) + PdhCloseQuery(pdh_query_); + for (PerformanceDataCounter* resource_consumption_counter : resource_consumption_counters_) { + delete resource_consumption_counter; + } +} + +void PerformanceDataMonitor::onSchedule(const std::shared_ptr<core::ProcessContext>& context, const std::shared_ptr<core::ProcessSessionFactory>& sessionFactory) { + setupMembersFromProperties(context); + + std::vector<PerformanceDataCounter*> valid_counters; + + for (PerformanceDataCounter* counter : resource_consumption_counters_) { + PDHCounterBase* pdh_counter = dynamic_cast<PDHCounterBase*> (counter); + if (pdh_counter != nullptr) { + if (pdh_query_ == nullptr) + PdhOpenQuery(NULL, NULL, &pdh_query_); + PDH_STATUS add_to_query_result = pdh_counter->addToQuery(pdh_query_); + if (add_to_query_result != ERROR_SUCCESS) { + logger_->log_error(("Error adding " + pdh_counter->getName() + " to query: " + std::to_string(add_to_query_result)).c_str()); + delete counter; + } else { + valid_counters.push_back(counter); + } + } else { + valid_counters.push_back(counter); + } + } Review comment: changed the raw ptrs to be unique_ptr-s in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) ########## File path: extensions/pdh/tests/PerformanceDataMonitorTests.cpp ########## @@ -0,0 +1,268 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <memory> +#include <string> +#include <vector> +#include <set> +#include <fstream> + +#include "TestBase.h" +#include "processors/PutFile.h" +#include "utils/file/FileUtils.h" +#include "PerformanceDataMonitor.h" +#include "rapidjson/filereadstream.h" + +using PutFile = org::apache::nifi::minifi::processors::PutFile; +using PerformanceDataMonitor = org::apache::nifi::minifi::processors::PerformanceDataMonitor; +using PerformanceDataCounter = org::apache::nifi::minifi::processors::PerformanceDataCounter; + +class PerformanceDataMonitorTester { + public: + PerformanceDataMonitorTester() { + LogTestController::getInstance().setTrace<TestPlan>(); + dir_ = test_controller_.createTempDirectory("/tmp/gt.XXXXXX"); + plan_ = test_controller_.createPlan(); + performance_monitor_ = plan_->addProcessor("PerformanceDataMonitor", "pdhsys"); + putfile_ = plan_->addProcessor("PutFile", "putfile", core::Relationship("success", "description"), true); + plan_->setProperty(putfile_, PutFile::Directory.getName(), dir_); + } + + void runProcessors() { + plan_->runNextProcessor(); // PerformanceMonitor + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + plan_->runCurrentProcessor(); // PerformanceMonitor + plan_->runNextProcessor(); // PutFile + plan_->runCurrentProcessor(); // PutFile + } + + void setPerformanceMonitorProperty(core::Property property, const std::string& value) { + plan_->setProperty(performance_monitor_, property.getName(), value); + } + + TestController test_controller_; + std::string dir_; + std::shared_ptr<TestPlan> plan_; + std::shared_ptr<core::Processor> performance_monitor_; + std::shared_ptr<core::Processor> putfile_; +}; + + +TEST_CASE("PerformanceDataMonitorEmptyPropertiesTest", "[performancedatamonitoremptypropertiestest]") { + PerformanceDataMonitorTester tester; + tester.runProcessors(); + + REQUIRE(tester.test_controller_.getLog().getInstance().contains("No valid counters for PerformanceDataMonitor", std::chrono::seconds(0))); + + uint32_t number_of_flowfiles = 0; + auto lambda = [&number_of_flowfiles](const std::string& path, const std::string& filename) -> bool { + ++number_of_flowfiles; + return true; + }; + + utils::file::FileUtils::list_dir(tester.dir_, lambda, tester.plan_->getLogger(), false); Review comment: changed in [d5959c3](https://github.com/martinzink/nifi-minifi-cpp/commit/d5959c3b1b5d0f377e7b7f2c13cc22a484b2e4a6) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org