This is an automated email from the ASF dual-hosted git repository.
lordgamez pushed a change to branch MINIFICPP-2261
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
omit abaf39c17 MINIFICPP-2261 Add processor for pushing logs to Grafana
Loki through REST API
add 52ab84603 MINIFICPP-2274 Ship Markdown docs in binary packages
add de5baead4 MINIFICPP-2271 update AWS SDK and regions, port patches
add 4a4113713 MINIFICPP-2265 Add AttributeRollingWindow and EL nextInt
add 9c94f960a MINIFICPP-1951 Prevent append overwriting resource
add d80c1d954 MINIFICPP-2244 Update MiNiFi C2 and tests to support NiFi 2.0
add 9332011e1 MINIFICPP-2275 Use utils::string instead of
utils::StringUtils
add eac5e1765 MINIFICPP-2280 link CoreServices to libcURL to support macOS
Sonoma
add e42486db8 MINIFICPP-2229 Encrypt sensitive properties in the flow
configuration
add 0b7c0780f MINIFICPP-2261 Add processor for pushing logs to Grafana
Loki through REST API
add 07d9a0809 Review update
add 835fcacdb Fix after rebase
add 5d54d8107 Make password sensitive
add 0f1a614f8 Review update
add ade0446b4 Add doc for sensitive property
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (abaf39c17)
\
N -- N -- N refs/heads/MINIFICPP-2261 (ade0446b4)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
CMakeLists.txt | 11 +-
CONTROLLERS.md | 48 +--
EXPRESSIONS.md | 25 +-
PROCESSORS.md | 459 +++++++++++----------
bootstrap.sh | 6 +-
cmake/BuildTests.cmake | 9 +-
cmake/BundledAwsSdkCpp.cmake | 14 +-
cmake/BundledLibcURL.cmake | 1 +
controller/MiNiFiController.cpp | 2 +-
controller/tests/ControllerTests.cpp | 4 +-
.../cluster/checkers/GrafanaLokiChecker.py | 24 +-
.../cluster/containers/MinifiC2ServerContainer.py | 6 +-
.../cluster/containers/NifiContainer.py | 32 +-
.../features/MiNiFi_integration_test_driver.py | 2 +-
docker/test/integration/features/steps/steps.py | 1 +
docker/test/integration/minifi/core/Connectable.py | 1 +
.../integration/minifi/core/ControllerService.py | 1 +
.../Minifi_flow_json_serializer.py | 4 +-
.../Minifi_flow_yaml_serializer.py | 6 +-
.../Nifi_flow_json_serializer.py | 226 ++++++++++
.../flow_serialization/Nifi_flow_xml_serializer.py | 348 ----------------
.../kubernetes/pods-etc/minifi.test-pod.yml | 2 +-
.../resources/minifi-c2-server/config-ssl.json | 53 +++
.../resources/minifi-c2-server/config-ssl.yml | 43 --
.../resources/minifi-c2-server/config.json | 41 ++
.../resources/minifi-c2-server/config.yml | 31 --
encrypt-config/EncryptConfig.cpp | 4 +-
encrypt-config/tests/ConfigFileEncryptorTests.cpp | 2 +-
.../aws/controllerservices/AWSCredentialsService.h | 1 +
extensions/aws/processors/PutS3Object.cpp | 10 +-
extensions/aws/processors/S3Processor.h | 19 +-
extensions/aws/s3/MultipartUploadStateStorage.cpp | 6 +-
extensions/aws/s3/S3Wrapper.cpp | 6 +-
extensions/aws/s3/S3Wrapper.h | 2 +-
.../AzureStorageCredentialsService.h | 2 +
.../processors/AzureBlobStorageProcessorBase.h | 5 +-
extensions/azure/storage/AzureDataLakeStorage.cpp | 2 +-
extensions/civetweb/tests/ListenHTTPTests.cpp | 6 +-
extensions/coap/tests/CoapIntegrationBase.h | 12 +-
.../ElasticsearchCredentialsControllerService.h | 2 +
extensions/elasticsearch/PostElasticsearch.cpp | 2 +-
extensions/elasticsearch/tests/MockElastic.h | 2 +-
extensions/expression-language/Expression.cpp | 34 +-
extensions/expression-language/common/Value.h | 6 +-
.../tests/ExpressionLanguageTests.cpp | 1 +
.../GCPCredentialsControllerService.h | 1 +
extensions/gcp/processors/FetchGCSObject.h | 1 +
extensions/gcp/processors/PutGCSObject.h | 1 +
extensions/grafana-loki/PushGrafanaLokiREST.cpp | 40 +-
extensions/grafana-loki/PushGrafanaLokiREST.h | 9 +-
extensions/grafana-loki/tests/MockGrafanaLoki.h | 30 +-
.../grafana-loki/tests/PushGrafanaLokiRESTTest.cpp | 39 +-
extensions/http-curl/client/HTTPClient.cpp | 2 +-
extensions/http-curl/client/HTTPClient.h | 2 +-
extensions/http-curl/processors/InvokeHTTP.cpp | 8 +-
extensions/http-curl/processors/InvokeHTTP.h | 1 +
extensions/http-curl/protocols/RESTSender.cpp | 6 +-
extensions/http-curl/sitetosite/HTTPProtocol.cpp | 2 +-
extensions/http-curl/sitetosite/PeersEntity.h | 6 +-
.../tests/C2ClearCoreComponentStateTest.cpp | 2 +-
extensions/http-curl/tests/C2ConfigEncryption.cpp | 8 +-
.../http-curl/tests/C2DescribeManifestTest.cpp | 2 +-
extensions/http-curl/tests/C2LogHeartbeatTest.cpp | 2 +-
extensions/http-curl/tests/C2MetricsTest.cpp | 6 +-
extensions/http-curl/tests/C2PauseResumeTest.cpp | 20 +-
.../http-curl/tests/C2PropertiesUpdateTests.cpp | 2 +-
extensions/http-curl/tests/C2UpdateAssetTest.cpp | 2 +-
extensions/http-curl/tests/CMakeLists.txt | 3 +-
.../tests/ControllerServiceIntegrationTests.cpp | 22 +-
extensions/http-curl/tests/HTTPHandlers.h | 2 +-
extensions/http-curl/tests/VerifyInvokeHTTP.h | 11 +-
extensions/jni/JVMCreator.h | 4 +-
extensions/jni/jvm/JVMLoader.h | 4 +-
extensions/jni/jvm/JavaControllerService.h | 2 +-
extensions/kubernetes/ApiClient.cpp | 2 +-
extensions/kubernetes/MetricsFilter.cpp | 2 +-
.../processors/CollectKubernetesPodMetrics.cpp | 6 +-
.../tests/KubernetesMetricsFilterTests.cpp | 10 +-
extensions/libarchive/CompressContent.cpp | 6 +-
.../libarchive/tests/CompressContentTests.cpp | 6 +-
extensions/libarchive/tests/MergeFileTests.cpp | 2 +-
extensions/librdkafka/ConsumeKafka.cpp | 18 +-
extensions/librdkafka/KafkaProcessorBase.h | 1 +
extensions/librdkafka/PublishKafka.cpp | 24 +-
extensions/librdkafka/PublishKafka.h | 1 +
extensions/librdkafka/rdkafka_utils.cpp | 13 +-
extensions/lua/LuaScriptEngine.cpp | 4 +-
extensions/lua/LuaScriptExecutor.cpp | 2 +-
.../mqtt/processors/AbstractMQTTProcessor.cpp | 2 +-
extensions/mqtt/processors/AbstractMQTTProcessor.h | 2 +
extensions/mqtt/processors/ConsumeMQTT.cpp | 6 +-
extensions/mqtt/processors/PublishMQTT.cpp | 2 +-
extensions/opc/include/opcbase.h | 1 +
extensions/opc/src/fetchopc.cpp | 6 +-
extensions/opc/src/opc.cpp | 2 +-
extensions/opc/src/opcbase.cpp | 6 +-
extensions/opc/src/putopc.cpp | 10 +-
extensions/opencv/CaptureRTSPFrame.h | 1 +
.../SourceInitiatedSubscriptionListener.cpp | 4 +-
extensions/pcap/CapturePacket.cpp | 4 +-
extensions/pdh/PDHCounters.cpp | 6 +-
extensions/pdh/PerformanceDataMonitor.cpp | 4 +-
.../prometheus/PrometheusMetricsPublisher.cpp | 2 +-
extensions/python/ExecutePythonProcessor.cpp | 2 +-
extensions/python/PythonCreator.h | 8 +-
extensions/python/PythonScriptExecutor.cpp | 2 +-
extensions/python/tests/PythonManifestTests.cpp | 2 +-
.../rocksdb-repos/DatabaseContentRepository.cpp | 10 +-
.../rocksdb-repos/database/RocksDatabase.cpp | 2 +-
.../encryption/RocksDbEncryptionProvider.cpp | 2 +-
.../rocksdb-repos/tests/ContentSessionTests.cpp | 31 +-
.../tests/DBContentRepositoryTests.cpp | 66 ++-
extensions/rocksdb-repos/tests/RepoTests.cpp | 45 +-
extensions/rocksdb-repos/tests/SwapTests.cpp | 1 +
extensions/sftp/client/SFTPClient.cpp | 8 +-
extensions/sftp/processors/FetchSFTP.cpp | 6 +-
extensions/sftp/processors/ListSFTP.cpp | 8 +-
extensions/sftp/processors/PutSFTP.cpp | 10 +-
extensions/sftp/processors/SFTPProcessorBase.cpp | 4 +-
extensions/sftp/processors/SFTPProcessorBase.h | 3 +
extensions/sftp/tests/ListSFTPTests.cpp | 38 +-
extensions/smb/SmbConnectionControllerService.h | 1 +
extensions/splunk/PutSplunkHTTP.cpp | 2 +-
extensions/splunk/SplunkHECProcessor.h | 1 +
extensions/sql/processors/QueryDatabaseTable.cpp | 12 +-
extensions/sql/services/DatabaseService.h | 1 +
extensions/sql/tests/mocks/MockConnectors.cpp | 56 +--
extensions/standard-processors/CMakeLists.txt | 3 +-
extensions/standard-processors/RollingWindow.h | 72 ++++
.../processors/AttributeRollingWindow.cpp | 123 ++++++
.../processors/AttributeRollingWindow.h | 117 ++++++
.../processors/AttributesToJSON.cpp | 2 +-
.../processors/ExecuteProcess.cpp | 25 +-
.../standard-processors/processors/GetFile.cpp | 6 +-
.../standard-processors/processors/GetTCP.cpp | 6 +-
.../standard-processors/processors/HashContent.h | 6 +-
.../processors/LogAttribute.cpp | 2 +-
.../standard-processors/processors/ReplaceText.cpp | 30 +-
.../standard-processors/processors/RouteText.cpp | 10 +-
.../standard-processors/processors/TailFile.cpp | 42 +-
.../tests/unit/AttributeRollingWindowTests.cpp | 101 +++++
.../tests/unit/ManifestTests.cpp | 26 +-
.../tests/unit/ProcessGroupTestUtils.h | 2 +-
.../standard-processors/tests/unit/PutTCPTests.cpp | 2 +-
.../standard-processors/tests/unit/PutUDPTests.cpp | 2 +-
.../tests/unit/RollingWindowTests.cpp | 108 +++++
.../tests/unit/TailFileTests.cpp | 27 +-
.../tests/unit/YamlConfigurationTests.cpp | 282 ++++++++++++-
.../tests/unit/YamlProcessGroupParserTests.cpp | 9 +-
extensions/systemd/libwrapper/DlopenWrapper.cpp | 4 +-
extensions/systemd/tests/ConsumeJournaldTest.cpp | 8 +-
extensions/test-processors/KamikazeProcessor.cpp | 4 +-
.../CollectorInitiatedSubscription.h | 1 +
.../windows-event-log/ConsumeWindowsEventLog.cpp | 14 +-
.../windows-event-log/tests/BookmarkTests.cpp | 2 +-
extensions/windows-event-log/wel/EventPath.cpp | 2 +-
extensions/windows-event-log/wel/JSONUtils.cpp | 2 +-
.../windows-event-log/wel/MetadataWalker.cpp | 2 +-
extensions/windows-event-log/wel/WindowsEventLog.h | 2 +-
libminifi/include/Exception.h | 6 +-
libminifi/include/RemoteProcessorGroupPort.h | 4 +-
libminifi/include/RootProcessGroupWrapper.h | 1 +
libminifi/include/agent/agent_docs.h | 2 +-
libminifi/include/controllers/SSLContextService.h | 3 +-
libminifi/include/core/AbstractProcessor.h | 58 +++
libminifi/include/core/BufferedContentSession.h | 6 +-
libminifi/include/core/ContentRepository.h | 19 +
libminifi/include/core/ContentSession.h | 22 +-
libminifi/include/core/FlowConfiguration.h | 14 +-
libminifi/include/core/ForwardingContentSession.h | 5 +-
libminifi/include/core/ProcessGroup.h | 2 +-
libminifi/include/core/Processor.h | 2 +-
libminifi/include/core/Property.h | 2 +
libminifi/include/core/PropertyDefinition.h | 3 +
libminifi/include/core/PropertyDefinitionBuilder.h | 5 +
libminifi/include/core/PropertyType.h | 4 +-
libminifi/include/core/StreamManager.h | 29 +-
libminifi/include/core/TypedValues.h | 6 +-
libminifi/include/core/VariableRegistry.h | 4 +-
.../include/core/controller/ControllerServiceMap.h | 44 +-
libminifi/include/core/extension/Utils.h | 2 +-
.../include/core/flow/AdaptiveConfiguration.h | 4 +-
libminifi/include/core/flow/Node.h | 4 +-
.../include/core/flow/StructuredConfiguration.h | 12 +-
libminifi/include/core/yaml/YamlConfiguration.h | 2 +
libminifi/include/io/FileStream.h | 4 +-
libminifi/include/properties/Properties.h | 2 +-
libminifi/include/utils/BaseHTTPClient.h | 4 +-
libminifi/include/utils/StringUtils.h | 2 -
libminifi/include/utils/TimeUtil.h | 4 +-
libminifi/include/utils/crypto/EncryptionManager.h | 69 +++-
.../include/utils/crypto/EncryptionProvider.h | 20 +-
libminifi/include/utils/crypto/EncryptionUtils.h | 25 +-
libminifi/include/utils/crypto/ciphers/XSalsa20.h | 22 +-
libminifi/include/utils/file/FileUtils.h | 2 +-
libminifi/src/Configuration.cpp | 4 +-
libminifi/src/Configure.cpp | 4 +-
libminifi/src/FlowController.cpp | 6 +-
libminifi/src/RemoteProcessorGroupPort.cpp | 2 +-
libminifi/src/agent/JsonSchema.cpp | 8 +-
libminifi/src/c2/C2Agent.cpp | 26 +-
libminifi/src/c2/C2MetricsPublisher.cpp | 20 +-
libminifi/src/c2/C2Payload.cpp | 2 +-
libminifi/src/c2/C2Utils.cpp | 4 +-
libminifi/src/c2/ControllerSocketProtocol.cpp | 4 +-
libminifi/src/c2/protocols/RESTProtocol.cpp | 4 +-
.../controllers/LinuxPowerManagementService.cpp | 6 +-
.../src/controllers/NetworkPrioritizerService.cpp | 2 +-
.../controllers/UpdatePolicyControllerService.cpp | 4 +-
libminifi/src/core/BufferedContentSession.cpp | 30 +-
libminifi/src/core/ConfigurableComponent.cpp | 36 +-
libminifi/src/core/ContentRepository.cpp | 19 +
libminifi/src/core/ContentSession.cpp | 49 +++
libminifi/src/core/FlowConfiguration.cpp | 44 +-
libminifi/src/core/ForwardingContentSession.cpp | 14 +-
libminifi/src/core/ProcessGroup.cpp | 2 +-
libminifi/src/core/ProcessSession.cpp | 7 +-
libminifi/src/core/Property.cpp | 15 +-
libminifi/src/core/extension/ExtensionManager.cpp | 7 +-
libminifi/src/core/flow/AdaptiveConfiguration.cpp | 19 +-
libminifi/src/core/flow/CheckRequiredField.cpp | 2 +-
libminifi/src/core/flow/Node.cpp | 4 +-
.../src/core/flow/StructuredConfiguration.cpp | 181 +++++++-
.../src/core/flow/StructuredConnectionParser.cpp | 6 +-
libminifi/src/core/logging/LoggerConfiguration.cpp | 12 +-
libminifi/src/core/logging/Utils.cpp | 14 +-
.../core/repository/VolatileContentRepository.cpp | 2 +-
libminifi/src/core/state/LogMetricsPublisher.cpp | 2 +-
.../src/core/state/MetricsPublisherFactory.cpp | 2 +-
libminifi/src/core/state/Value.cpp | 2 +-
.../src/core/state/nodes/AgentInformation.cpp | 5 +-
.../src/core/state/nodes/ResponseNodeLoader.cpp | 2 +-
libminifi/src/core/yaml/YamlConfiguration.cpp | 5 +-
libminifi/src/io/FileStream.cpp | 8 +-
libminifi/src/properties/Properties.cpp | 12 +-
libminifi/src/properties/PropertiesFile.cpp | 8 +-
libminifi/src/utils/BaseHTTPClient.cpp | 4 +-
libminifi/src/utils/ClassUtils.cpp | 6 +-
libminifi/src/utils/Cron.cpp | 14 +-
libminifi/src/utils/Id.cpp | 4 +-
libminifi/src/utils/ProcessorConfigUtils.cpp | 6 +-
libminifi/src/utils/crypto/EncryptionManager.cpp | 38 +-
libminifi/src/utils/crypto/EncryptionProvider.cpp | 14 +-
libminifi/src/utils/crypto/EncryptionUtils.cpp | 20 +-
libminifi/src/utils/file/FilePattern.cpp | 6 +-
libminifi/src/utils/file/FileReaderCallback.cpp | 6 +-
libminifi/src/utils/file/PathUtils.cpp | 6 +-
libminifi/src/utils/tls/DistinguishedName.cpp | 22 +-
libminifi/src/utils/tls/ExtendedKeyUsage.cpp | 4 +-
libminifi/src/utils/tls/TLSUtils.cpp | 16 +-
libminifi/test/ConfigurationTestController.h | 7 +-
libminifi/test/EmptyFlow.h | 13 +-
libminifi/test/TestBase.cpp | 3 +-
libminifi/test/flow-tests/TestControllerWithFlow.h | 9 +-
libminifi/test/integration/IntegrationBase.h | 26 +-
.../integration/OnScheduleErrorHandlingTests.cpp | 4 +-
.../integration/StateTransactionalityTests.cpp | 10 +-
.../keyvalue-tests/PersistentStateStorageTest.cpp | 10 +-
.../keyvalue-tests/VolatileMapStateStorageTest.cpp | 10 +-
.../test/persistence-tests/PersistenceTests.cpp | 20 +-
libminifi/test/resources/TestHTTPPost.yml | 2 +-
.../test/resources/TestHTTPPostChunkedEncoding.yml | 2 +-
libminifi/test/resources/conf/bootstrap.conf | 1 +
libminifi/test/resources/decrypted.config.json | 91 ++++
.../resources/decrypted.config.json.reformatted | 91 ++++
.../resources/decrypted.config.yml.reformatted | 22 +
libminifi/test/schema-tests/SchemaTests.cpp | 2 +-
libminifi/test/unit/AbstractProcessorTest.cpp | 114 +++++
libminifi/test/unit/CertificateUtilsTests.cpp | 4 +-
libminifi/test/unit/DecryptorTests.cpp | 12 +-
libminifi/test/unit/EncryptionUtilsTests.cpp | 8 +-
libminifi/test/unit/FileSystemRepositoryTests.cpp | 42 ++
libminifi/test/unit/FileSystemTests.cpp | 2 +-
libminifi/test/unit/IdTests.cpp | 2 +-
libminifi/test/unit/LogMetricsPublisherTests.cpp | 2 +
libminifi/test/unit/MinifiConcurrentQueueTests.cpp | 8 +-
libminifi/test/unit/ProcessSessionTests.cpp | 2 +
libminifi/test/unit/PropertyTests.cpp | 12 +-
libminifi/test/unit/Site2SiteTests.cpp | 2 +-
libminifi/test/unit/StagingQueueTests.cpp | 6 +-
libminifi/test/unit/StringUtilsTests.cpp | 2 +-
libminifi/test/unit/ZlibStreamTests.cpp | 2 +-
minifi_main/AgentDocs.cpp | 30 +-
minifi_main/MiNiFiMain.cpp | 10 +-
thirdparty/aws-sdk-cpp/bundle-openssl.patch | 57 ++-
.../aws-sdk-cpp/c++20-compilation-fixes.patch | 12 -
thirdparty/aws-sdk-cpp/dll-export-injection.patch | 118 ++++--
thirdparty/aws-sdk-cpp/openssl3-fix.patch | 54 ---
288 files changed, 3532 insertions(+), 1766 deletions(-)
create mode 100644
docker/test/integration/minifi/flow_serialization/Nifi_flow_json_serializer.py
delete mode 100644
docker/test/integration/minifi/flow_serialization/Nifi_flow_xml_serializer.py
create mode 100644
docker/test/integration/resources/minifi-c2-server/config-ssl.json
delete mode 100644
docker/test/integration/resources/minifi-c2-server/config-ssl.yml
create mode 100644
docker/test/integration/resources/minifi-c2-server/config.json
delete mode 100644
docker/test/integration/resources/minifi-c2-server/config.yml
create mode 100644 extensions/standard-processors/RollingWindow.h
create mode 100644
extensions/standard-processors/processors/AttributeRollingWindow.cpp
create mode 100644
extensions/standard-processors/processors/AttributeRollingWindow.h
create mode 100644
extensions/standard-processors/tests/unit/AttributeRollingWindowTests.cpp
create mode 100644
extensions/standard-processors/tests/unit/RollingWindowTests.cpp
create mode 100644 libminifi/include/core/AbstractProcessor.h
create mode 100644 libminifi/src/core/ContentSession.cpp
create mode 100644 libminifi/test/resources/decrypted.config.json
create mode 100644 libminifi/test/resources/decrypted.config.json.reformatted
create mode 100644 libminifi/test/resources/decrypted.config.yml.reformatted
create mode 100644 libminifi/test/unit/AbstractProcessorTest.cpp
delete mode 100644 thirdparty/aws-sdk-cpp/c++20-compilation-fixes.patch
delete mode 100644 thirdparty/aws-sdk-cpp/openssl3-fix.patch
