This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 26409cfd52c762bb6c5242a594a849edac1b19e7 Author: Gabor Gyimesi <[email protected]> AuthorDate: Tue Sep 30 16:26:51 2025 +0200 MINIFICPP-2614 Add more authentication options to Azure processors - Add credential configuration strategy property with from properties, default credentials, managed identity, and workload identity options - Add managed identity client id property - Do not allow SAS Token and Account Key to be used together BREAKING CHANGE: removing the old `Managed Identity Credentials` property in favor of the new `Credential Configuration Strategy` property which allows multiple new Azure authentication options Closes #2016 Signed-off-by: Marton Szasz <[email protected]> --- CONTROLLERS.md | 17 ++-- PROCESSORS.md | 102 +++++++++++---------- extensions/aws/processors/PutS3Object.h | 6 +- .../AzureStorageCredentialsService.cpp | 13 ++- .../AzureStorageCredentialsService.h | 28 ++++-- .../processors/AzureBlobStorageProcessorBase.cpp | 13 ++- .../processors/AzureBlobStorageProcessorBase.h | 32 +++++-- .../azure/processors/FetchAzureBlobStorage.h | 2 +- .../azure/processors/FetchAzureDataLakeStorage.h | 2 +- .../azure/processors/PutAzureDataLakeStorage.cpp | 13 +-- .../azure/storage/AzureBlobStorageClient.cpp | 10 +- .../azure/storage/AzureDataLakeStorageClient.cpp | 10 +- .../azure/storage/AzureStorageCredentials.cpp | 51 +++++++++-- extensions/azure/storage/AzureStorageCredentials.h | 17 +++- .../azure/tests/DeleteAzureBlobStorageTests.cpp | 74 ++++++++++++++- .../tests/DeleteAzureDataLakeStorageTests.cpp | 32 ++++++- .../azure/tests/FetchAzureBlobStorageTests.cpp | 74 ++++++++++++++- .../azure/tests/FetchAzureDataLakeStorageTests.cpp | 32 ++++++- .../azure/tests/ListAzureBlobStorageTests.cpp | 64 ++++++++++++- .../azure/tests/ListAzureDataLakeStorageTests.cpp | 42 ++++++++- .../azure/tests/PutAzureBlobStorageTests.cpp | 74 ++++++++++++++- .../azure/tests/PutAzureDataLakeStorageTests.cpp | 32 ++++++- extensions/azure/utils/AzureEnums.h | 28 ++++++ 23 files changed, 621 insertions(+), 147 deletions(-) diff --git a/CONTROLLERS.md b/CONTROLLERS.md index 52bf80d40..587470fdd 100644 --- a/CONTROLLERS.md +++ b/CONTROLLERS.md @@ -64,14 +64,15 @@ Manages the credentials for an Azure Storage account. This allows for multiple A In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|----------------------------------------|---------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Storage Account Name | | | The storage account name. | -| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies.<br/>**Sensitive Property: true** | -| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.<br/>**Sensitive Property: true** | -| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions). | -| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used. | -| **Use Managed Identity Credentials** | false | true<br/>false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | +| Name | Default Value | Allowable Values | Description [...] +|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [...] +| Storage Account Name | | | The storage account name. [...] +| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.<br/>**Sensitive Property: true* [...] +| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Credential Configuration Strategy is set to From Properties. If set, Storage Account Key must be empty.<br/>**Sensitive Property: true** [...] +| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions). [...] +| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Credential Configuration Strategy is set to From Properties. [...] +| **Credential Configuration Strategy** | From Properties | From Properties<br/>Default Credential<br/>Managed Identity<br/>Workload Identity | The strategy to use for credential configuration. If set to From Properties, the credentials are parsed from the SAS Token, Storage Account Key, and Connection String properties. In other cases, the selected Azure identity source is used. [...] +| Managed Identity Client ID | | | Client ID of the managed identity. The property is required when User Assigned Managed Identity is used for authentication and multiple user-assigned identities are added to the resource. It must be empty in case of System Assigned Managed Identity and can also be left empty if only one user-assigned identity is present.<br/>**Sensitive Property [...] ## CouchbaseClusterService diff --git a/PROCESSORS.md b/PROCESSORS.md index 5204c2f67..cd4b7141a 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -543,18 +543,19 @@ Deletes the provided blob from Azure Storage In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|----------------------------------------|---------------|------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | -| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.<br/>**Supports Expression Language: true** | -| Storage Account Name | | | The storage account name.<br/>**Supports Expression Language: true** | -| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** | -| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** | -| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions). <br/>**Supports Expression Language: true** | -| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.<br/>**Supports Expression Language: true** | -| **Use Managed Identity Credentials** | false | true<br/>false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | -| Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.<br/>**Supports Expression Language: true** | -| **Delete Snapshots Option** | None | None<br/>Include Snapshots<br/>Delete Snapshots Only | Specifies the snapshot deletion options to be used when deleting a blob. None: Deletes the blob only. Include Snapshots: Delete the blob and its snapshots. Delete Snapshots Only: Delete only the blob's snapshots. | +| Name | Default Value | Allowable Values | Description [...] +|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [...] +| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. [...] +| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.<br/>**Supports Expression Language: true** [...] +| Storage Account Name | | | The storage account name.<br/>**Supports Expression Language: true** [...] +| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.<br/>**Sensitive Property: true* [...] +| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Credential Configuration Strategy is set to From Properties. If set, Storage Account Key must be empty.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** [...] +| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions).<br/>**Supports Expression Language: true** [...] +| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Credential Configuration Strategy is set to From Properties.<br/>**Supports Expression Language: true** [...] +| **Credential Configuration Strategy** | From Properties | From Properties<br/>Default Credential<br/>Managed Identity<br/>Workload Identity | The strategy to use for credential configuration. If set to From Properties, the credentials are parsed from the SAS Token, Storage Account Key, and Connection String properties. In other cases, the selected Azure identity source is used. [...] +| Managed Identity Client ID | | | Client ID of the managed identity. The property is required when User Assigned Managed Identity is used for authentication and multiple user-assigned identities are added to the resource. It must be empty in case of System Assigned Managed Identity and can also be left empty if only one user-assigned identity is present.<br/>**Sensitive Property [...] +| Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.<br/>**Supports Expression Language: true** [...] +| **Delete Snapshots Option** | None | None<br/>Include Snapshots<br/>Delete Snapshots Only | Specifies the snapshot deletion options to be used when deleting a blob. None: Deletes the blob only. Include Snapshots: Delete the blob and its snapshots. Delete Snapshots Only: Delete only the blob's snapshots. [...] ### Relationships @@ -834,19 +835,20 @@ Retrieves contents of an Azure Storage Blob, writing the contents to the content In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|----------------------------------------|---------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | -| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.<br/>**Supports Expression Language: true** | -| Storage Account Name | | | The storage account name.<br/>**Supports Expression Language: true** | -| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** | -| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** | -| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions). <br/>**Supports Expression Language: true** | -| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.<br/>**Supports Expression Language: true** | -| **Use Managed Identity Credentials** | false | true<br/>false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | -| Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.<br/>**Supports Expression Language: true** | -| Range Start | | | The byte position at which to start reading from the blob. An empty value or a value of zero will start reading at the beginning of the blob.<br/>**Supports Expression Language: true** | -| Range Length | | | The number of bytes to download from the blob, starting from the Range Start. An empty value or a value that extends beyond the end of the blob will read to the end of the blob.<br/>**Supports Expression Language: true** | +| Name | Default Value | Allowable Values | Description [...] +|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [...] +| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. [...] +| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.<br/>**Supports Expression Language: true** [...] +| Storage Account Name | | | The storage account name.<br/>**Supports Expression Language: true** [...] +| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.<br/>**Sensitive Property: true* [...] +| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Credential Configuration Strategy is set to From Properties. If set, Storage Account Key must be empty.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** [...] +| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions).<br/>**Supports Expression Language: true** [...] +| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Credential Configuration Strategy is set to From Properties.<br/>**Supports Expression Language: true** [...] +| **Credential Configuration Strategy** | From Properties | From Properties<br/>Default Credential<br/>Managed Identity<br/>Workload Identity | The strategy to use for credential configuration. If set to From Properties, the credentials are parsed from the SAS Token, Storage Account Key, and Connection String properties. In other cases, the selected Azure identity source is used. [...] +| Managed Identity Client ID | | | Client ID of the managed identity. The property is required when User Assigned Managed Identity is used for authentication and multiple user-assigned identities are added to the resource. It must be empty in case of System Assigned Managed Identity and can also be left empty if only one user-assigned identity is present.<br/>**Sensitive Property [...] +| Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.<br/>**Supports Expression Language: true** [...] +| Range Start | | | The byte position at which to start reading from the blob. An empty value or a value of zero will start reading at the beginning of the blob.<br/>**Supports Expression Language: true** [...] +| Range Length | | | The number of bytes to download from the blob, starting from the Range Start. An empty value or a value that extends beyond the end of the blob will read to the end of the blob.<br/>**Supports Expression Language: true** [...] ### Relationships @@ -1400,18 +1402,19 @@ Lists blobs in an Azure Storage container. Listing details are attached to an em In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|----------------------------------------|---------------|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | -| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.<br/>**Supports Expression Language: true** | -| Storage Account Name | | | The storage account name.<br/>**Supports Expression Language: true** | -| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** | -| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** | -| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions). <br/>**Supports Expression Language: true** | -| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.<br/>**Supports Expression Language: true** | -| **Use Managed Identity Credentials** | false | true<br/>false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | -| **Listing Strategy** | timestamps | none<br/>timestamps | Specify how to determine new/updated entities. If 'timestamps' is selected it tracks the latest timestamp of listed entity to determine new/updated entities. If 'none' is selected it lists an entity without any tracking, the same entity will be listed each time on executing this processor. | -| Prefix | | | Search prefix for listing<br/>**Supports Expression Language: true** | +| Name | Default Value | Allowable Values | Description [...] +|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [...] +| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. [...] +| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.<br/>**Supports Expression Language: true** [...] +| Storage Account Name | | | The storage account name.<br/>**Supports Expression Language: true** [...] +| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.<br/>**Sensitive Property: true* [...] +| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Credential Configuration Strategy is set to From Properties. If set, Storage Account Key must be empty.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** [...] +| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions).<br/>**Supports Expression Language: true** [...] +| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Credential Configuration Strategy is set to From Properties.<br/>**Supports Expression Language: true** [...] +| **Credential Configuration Strategy** | From Properties | From Properties<br/>Default Credential<br/>Managed Identity<br/>Workload Identity | The strategy to use for credential configuration. If set to From Properties, the credentials are parsed from the SAS Token, Storage Account Key, and Connection String properties. In other cases, the selected Azure identity source is used. [...] +| Managed Identity Client ID | | | Client ID of the managed identity. The property is required when User Assigned Managed Identity is used for authentication and multiple user-assigned identities are added to the resource. It must be empty in case of System Assigned Managed Identity and can also be left empty if only one user-assigned identity is present.<br/>**Sensitive Property [...] +| **Listing Strategy** | timestamps | none<br/>timestamps | Specify how to determine new/updated entities. If 'timestamps' is selected it tracks the latest timestamp of listed entity to determine new/updated entities. If 'none' is selected it lists an entity without any tracking, the same entity will be listed each time on executing this processor. [...] +| Prefix | | | Search prefix for listing<br/>**Supports Expression Language: true** [...] ### Relationships @@ -2325,18 +2328,19 @@ Puts content into an Azure Storage Blob In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|----------------------------------------|---------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | -| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.<br/>**Supports Expression Language: true** | -| Storage Account Name | | | The storage account name.<br/>**Supports Expression Language: true** | -| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** | -| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** | -| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions). <br/>**Supports Expression Language: true** | -| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.<br/>**Supports Expression Language: true** | -| **Use Managed Identity Credentials** | false | true<br/>false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | -| Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.<br/>**Supports Expression Language: true** | -| **Create Container** | false | true<br/>false | Specifies whether to check if the container exists and to automatically create it if it does not. Permission to list containers is required. If false, this check is not made, but the Put operation will fail if the container does not exist. | +| Name | Default Value | Allowable Values | Description [...] +|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [...] +| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. [...] +| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.<br/>**Supports Expression Language: true** [...] +| Storage Account Name | | | The storage account name.<br/>**Supports Expression Language: true** [...] +| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.<br/>**Sensitive Property: true* [...] +| SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Credential Configuration Strategy is set to From Properties. If set, Storage Account Key must be empty.<br/>**Sensitive Property: true**<br/>**Supports Expression Language: true** [...] +| Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions).<br/>**Supports Expression Language: true** [...] +| Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Credential Configuration Strategy is set to From Properties.<br/>**Supports Expression Language: true** [...] +| **Credential Configuration Strategy** | From Properties | From Properties<br/>Default Credential<br/>Managed Identity<br/>Workload Identity | The strategy to use for credential configuration. If set to From Properties, the credentials are parsed from the SAS Token, Storage Account Key, and Connection String properties. In other cases, the selected Azure identity source is used. [...] +| Managed Identity Client ID | | | Client ID of the managed identity. The property is required when User Assigned Managed Identity is used for authentication and multiple user-assigned identities are added to the resource. It must be empty in case of System Assigned Managed Identity and can also be left empty if only one user-assigned identity is present.<br/>**Sensitive Property [...] +| Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.<br/>**Supports Expression Language: true** [...] +| **Create Container** | false | true<br/>false | Specifies whether to check if the container exists and to automatically create it if it does not. Permission to list containers is required. If false, this check is not made, but the Put operation will fail if the container does not exist. [...] ### Relationships diff --git a/extensions/aws/processors/PutS3Object.h b/extensions/aws/processors/PutS3Object.h index 5fe86bf0d..912c45b7d 100644 --- a/extensions/aws/processors/PutS3Object.h +++ b/extensions/aws/processors/PutS3Object.h @@ -119,21 +119,21 @@ class PutS3Object : public S3Processor { // NOLINT(cppcoreguidelines-special-me .build(); EXTENSIONAPI static constexpr auto MultipartThreshold = core::PropertyDefinitionBuilder<>::createProperty("Multipart Threshold") .withDescription("Specifies the file size threshold for switch from the PutS3Object API to the PutS3MultipartUpload API. " - "Flow files bigger than this limit will be sent using the multipart process. The valid range is 5MB to 5GB.") + "Flow files bigger than this limit will be sent using the multipart process. The valid range is 5MB to 5GB.") .withValidator(core::StandardPropertyValidators::DATA_SIZE_VALIDATOR) .withDefaultValue("5 GB") .isRequired(true) .build(); EXTENSIONAPI static constexpr auto MultipartPartSize = core::PropertyDefinitionBuilder<>::createProperty("Multipart Part Size") .withDescription("Specifies the part size for use when the PutS3Multipart Upload API is used. " - "Flow files will be broken into chunks of this size for the upload process, but the last part sent can be smaller since it is not padded. The valid range is 5MB to 5GB.") + "Flow files will be broken into chunks of this size for the upload process, but the last part sent can be smaller since it is not padded. The valid range is 5MB to 5GB.") .withValidator(core::StandardPropertyValidators::DATA_SIZE_VALIDATOR) .withDefaultValue("5 GB") .isRequired(true) .build(); EXTENSIONAPI static constexpr auto MultipartUploadAgeOffInterval = core::PropertyDefinitionBuilder<>::createProperty("Multipart Upload AgeOff Interval") .withDescription("Specifies the interval at which existing multipart uploads in AWS S3 will be evaluated for ageoff. " - "When processor is triggered it will initiate the ageoff evaluation if this interval has been exceeded.") + "When processor is triggered it will initiate the ageoff evaluation if this interval has been exceeded.") .withValidator(core::StandardPropertyValidators::TIME_PERIOD_VALIDATOR) .withDefaultValue("60 min") .isRequired(true) diff --git a/extensions/azure/controllerservices/AzureStorageCredentialsService.cpp b/extensions/azure/controllerservices/AzureStorageCredentialsService.cpp index 4d0ae2b39..4f128f8f7 100644 --- a/extensions/azure/controllerservices/AzureStorageCredentialsService.cpp +++ b/extensions/azure/controllerservices/AzureStorageCredentialsService.cpp @@ -21,6 +21,7 @@ #include <set> #include "core/Resource.h" +#include "minifi-cpp/Exception.h" namespace org::apache::nifi::minifi::azure::controllers { @@ -29,6 +30,12 @@ void AzureStorageCredentialsService::initialize() { } void AzureStorageCredentialsService::onEnable() { + auto credential_configuration_strategy_str = getProperty(CredentialConfigurationStrategy.name).value_or(std::string{magic_enum::enum_name(CredentialConfigurationStrategyOption::FromProperties)}); + if (auto credential_configuration_strategy = magic_enum::enum_cast<CredentialConfigurationStrategyOption>(credential_configuration_strategy_str)) { + credentials_.setCredentialConfigurationStrategy(*credential_configuration_strategy); + } else { + throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Invalid Credential Configuration Strategy: " + credential_configuration_strategy_str); + } if (auto storage_account_name = getProperty(StorageAccountName.name)) { credentials_.setStorageAccountName(*storage_account_name); } @@ -39,13 +46,13 @@ void AzureStorageCredentialsService::onEnable() { credentials_.setSasToken(*sas_token); } if (auto common_storage_account_endpoint_suffix = getProperty(CommonStorageAccountEndpointSuffix.name)) { - credentials_.setEndpontSuffix(*common_storage_account_endpoint_suffix); + credentials_.setEndpointSuffix(*common_storage_account_endpoint_suffix); } if (auto connection_String = getProperty(ConnectionString.name)) { credentials_.setConnectionString(*connection_String); } - if (auto use_managed_identity_credentials = getProperty(UseManagedIdentityCredentials.name) | utils::andThen(parsing::parseBool)) { - credentials_.setUseManagedIdentityCredentials(*use_managed_identity_credentials); + if (auto managed_identity_client_id = getProperty(ManagedIdentityClientId.name)) { + credentials_.setManagedIdentityClientId(*managed_identity_client_id); } } diff --git a/extensions/azure/controllerservices/AzureStorageCredentialsService.h b/extensions/azure/controllerservices/AzureStorageCredentialsService.h index 0b655edcb..4662bd630 100644 --- a/extensions/azure/controllerservices/AzureStorageCredentialsService.h +++ b/extensions/azure/controllerservices/AzureStorageCredentialsService.h @@ -29,6 +29,7 @@ #include "minifi-cpp/core/PropertyValidator.h" #include "storage/AzureStorageCredentials.h" #include "utils/Export.h" +#include "utils/AzureEnums.h" namespace org::apache::nifi::minifi::azure::controllers { @@ -42,11 +43,13 @@ class AzureStorageCredentialsService : public core::controller::ControllerServic .build(); EXTENSIONAPI static constexpr auto StorageAccountKey = core::PropertyDefinitionBuilder<>::createProperty("Storage Account Key") .withDescription("The storage account key. This is an admin-like password providing access to every container in this account. " - "It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies.") + "It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies " + "if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.") .isSensitive(true) .build(); EXTENSIONAPI static constexpr auto SASToken = core::PropertyDefinitionBuilder<>::createProperty("SAS Token") - .withDescription("Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.") + .withDescription("Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name " + "if Credential Configuration Strategy is set to From Properties. If set, Storage Account Key must be empty.") .isSensitive(true) .build(); EXTENSIONAPI static constexpr auto CommonStorageAccountEndpointSuffix = core::PropertyDefinitionBuilder<>::createProperty("Common Storage Account Endpoint Suffix") @@ -54,13 +57,21 @@ class AzureStorageCredentialsService : public core::controller::ControllerServic "different suffix in certain circumstances (like Azure Stack or non-public Azure regions).") .build(); EXTENSIONAPI static constexpr auto ConnectionString = core::PropertyDefinitionBuilder<>::createProperty("Connection String") - .withDescription("Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.") + .withDescription("Connection string used to connect to Azure Storage service. This overrides all other set credential properties " + "if Credential Configuration Strategy is set to From Properties.") .build(); - EXTENSIONAPI static constexpr auto UseManagedIdentityCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Managed Identity Credentials") - .withDescription("If true Managed Identity credentials will be used together with the Storage Account Name for authentication.") + EXTENSIONAPI static constexpr auto CredentialConfigurationStrategy = + core::PropertyDefinitionBuilder<magic_enum::enum_count<CredentialConfigurationStrategyOption>()>::createProperty("Credential Configuration Strategy") + .withDescription("The strategy to use for credential configuration. If set to From Properties, the credentials are parsed from the SAS Token, Storage Account Key, " + "and Connection String properties. In other cases, the selected Azure identity source is used.") .isRequired(true) - .withValidator(core::StandardPropertyValidators::BOOLEAN_VALIDATOR) - .withDefaultValue("false") + .withDefaultValue(magic_enum::enum_name(CredentialConfigurationStrategyOption::FromProperties)) + .withAllowedValues(magic_enum::enum_names<CredentialConfigurationStrategyOption>()) + .build(); + EXTENSIONAPI static constexpr auto ManagedIdentityClientId = core::PropertyDefinitionBuilder<>::createProperty("Managed Identity Client ID") + .withDescription("Client ID of the managed identity. The property is required when User Assigned Managed Identity is used for authentication and multiple user-assigned identities " + "are added to the resource. It must be empty in case of System Assigned Managed Identity and can also be left empty if only one user-assigned identity is present.") + .isSensitive(true) .build(); EXTENSIONAPI static constexpr auto Properties = std::to_array<core::PropertyReference>({ StorageAccountName, @@ -68,7 +79,8 @@ class AzureStorageCredentialsService : public core::controller::ControllerServic SASToken, CommonStorageAccountEndpointSuffix, ConnectionString, - UseManagedIdentityCredentials + CredentialConfigurationStrategy, + ManagedIdentityClientId }); diff --git a/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp b/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp index f0eee83a7..88ae2a2e9 100644 --- a/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp +++ b/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp @@ -35,10 +35,10 @@ void AzureBlobStorageProcessorBase::onSchedule(core::ProcessContext& context, co return; } - use_managed_identity_credentials_ = minifi::utils::parseBoolProperty(context, UseManagedIdentityCredentials); + credential_configuration_strategy_ = minifi::utils::parseEnumProperty<CredentialConfigurationStrategyOption>(context, CredentialConfigurationStrategy); - if (use_managed_identity_credentials_) { - logger_->log_info("Using Managed Identity for authentication"); + if (credential_configuration_strategy_ != CredentialConfigurationStrategyOption::FromProperties) { + logger_->log_info("Using {} for authentication", magic_enum::enum_name(credential_configuration_strategy_)); return; } @@ -66,6 +66,7 @@ void AzureBlobStorageProcessorBase::onSchedule(core::ProcessContext& context, co storage::AzureStorageCredentials AzureBlobStorageProcessorBase::getAzureCredentialsFromProperties( core::ProcessContext &context, const core::FlowFile* const flow_file) const { storage::AzureStorageCredentials credentials; + credentials.setCredentialConfigurationStrategy(credential_configuration_strategy_); if (const auto value = context.getProperty(StorageAccountName, flow_file)) { credentials.setStorageAccountName(*value); } @@ -76,12 +77,14 @@ storage::AzureStorageCredentials AzureBlobStorageProcessorBase::getAzureCredenti credentials.setSasToken(*value); } if (const auto value = context.getProperty(CommonStorageAccountEndpointSuffix, flow_file)) { - credentials.setEndpontSuffix(*value); + credentials.setEndpointSuffix(*value); } if (const auto value = context.getProperty(ConnectionString, flow_file)) { credentials.setConnectionString(*value); } - credentials.setUseManagedIdentityCredentials(use_managed_identity_credentials_); + if (const auto value = context.getProperty(ManagedIdentityClientId, flow_file)) { + credentials.setManagedIdentityClientId(*value); + } return credentials; } diff --git a/extensions/azure/processors/AzureBlobStorageProcessorBase.h b/extensions/azure/processors/AzureBlobStorageProcessorBase.h index 32eb7f793..88c09bd75 100644 --- a/extensions/azure/processors/AzureBlobStorageProcessorBase.h +++ b/extensions/azure/processors/AzureBlobStorageProcessorBase.h @@ -50,29 +50,40 @@ class AzureBlobStorageProcessorBase : public AzureStorageProcessorBase { .build(); EXTENSIONAPI static constexpr auto StorageAccountKey = core::PropertyDefinitionBuilder<>::createProperty("Storage Account Key") .withDescription("The storage account key. This is an admin-like password providing access to every container in this account. " - "It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies.") + "It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies " + "if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.") .supportsExpressionLanguage(true) .isSensitive(true) .build(); EXTENSIONAPI static constexpr auto SASToken = core::PropertyDefinitionBuilder<>::createProperty("SAS Token") - .withDescription("Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.") + .withDescription("Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name " + "if Credential Configuration Strategy is set to From Properties. If set, Storage Account Key must be empty.") .supportsExpressionLanguage(true) .isSensitive(true) .build(); EXTENSIONAPI static constexpr auto CommonStorageAccountEndpointSuffix = core::PropertyDefinitionBuilder<>::createProperty("Common Storage Account Endpoint Suffix") .withDescription("Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a " - "different suffix in certain circumstances (like Azure Stack or non-public Azure regions). ") + "different suffix in certain circumstances (like Azure Stack or non-public Azure regions).") .supportsExpressionLanguage(true) .build(); EXTENSIONAPI static constexpr auto ConnectionString = core::PropertyDefinitionBuilder<>::createProperty("Connection String") - .withDescription("Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.") + .withDescription("Connection string used to connect to Azure Storage service. This overrides all other set credential properties " + "if Credential Configuration Strategy is set to From Properties.") .supportsExpressionLanguage(true) .build(); - EXTENSIONAPI static constexpr auto UseManagedIdentityCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Managed Identity Credentials") - .withDescription("If true Managed Identity credentials will be used together with the Storage Account Name for authentication.") + EXTENSIONAPI static constexpr auto CredentialConfigurationStrategy = + core::PropertyDefinitionBuilder<magic_enum::enum_count<CredentialConfigurationStrategyOption>()>::createProperty("Credential Configuration Strategy") + .withDescription("The strategy to use for credential configuration. If set to From Properties, the credentials are parsed from the SAS Token, Storage Account Key, " + "and Connection String properties. In other cases, the selected Azure identity source is used.") .isRequired(true) - .withValidator(core::StandardPropertyValidators::BOOLEAN_VALIDATOR) - .withDefaultValue("false") + .withDefaultValue(magic_enum::enum_name(CredentialConfigurationStrategyOption::FromProperties)) + .withAllowedValues(magic_enum::enum_names<CredentialConfigurationStrategyOption>()) + .build(); + EXTENSIONAPI static constexpr auto ManagedIdentityClientId = core::PropertyDefinitionBuilder<>::createProperty("Managed Identity Client ID") + .withDescription("Client ID of the managed identity. The property is required when User Assigned Managed Identity is used for authentication and multiple user-assigned identities " + "are added to the resource. It must be empty in case of System Assigned Managed Identity and can also be left empty if only one user-assigned identity is present.") + .isSensitive(true) + .supportsExpressionLanguage(true) .build(); EXTENSIONAPI static constexpr auto Properties = utils::array_cat(AzureStorageProcessorBase::Properties, std::to_array<core::PropertyReference>({ ContainerName, @@ -81,7 +92,8 @@ class AzureBlobStorageProcessorBase : public AzureStorageProcessorBase { SASToken, CommonStorageAccountEndpointSuffix, ConnectionString, - UseManagedIdentityCredentials + CredentialConfigurationStrategy, + ManagedIdentityClientId })); @@ -105,7 +117,7 @@ class AzureBlobStorageProcessorBase : public AzureStorageProcessorBase { const core::FlowFile* const flow_file); storage::AzureBlobStorage azure_blob_storage_; - bool use_managed_identity_credentials_ = false; + CredentialConfigurationStrategyOption credential_configuration_strategy_ = CredentialConfigurationStrategyOption::FromProperties; }; } // namespace org::apache::nifi::minifi::azure::processors diff --git a/extensions/azure/processors/FetchAzureBlobStorage.h b/extensions/azure/processors/FetchAzureBlobStorage.h index 3270becc4..c9e1ee601 100644 --- a/extensions/azure/processors/FetchAzureBlobStorage.h +++ b/extensions/azure/processors/FetchAzureBlobStorage.h @@ -48,7 +48,7 @@ class FetchAzureBlobStorage final : public AzureBlobStorageSingleBlobProcessorBa EXTENSIONAPI static constexpr auto RangeLength = core::PropertyDefinitionBuilder<>::createProperty("Range Length") .withDescription("The number of bytes to download from the blob, starting from the Range Start. " - "An empty value or a value that extends beyond the end of the blob will read to the end of the blob.") + "An empty value or a value that extends beyond the end of the blob will read to the end of the blob.") .supportsExpressionLanguage(true) .build(); EXTENSIONAPI static constexpr auto Properties = utils::array_cat(AzureBlobStorageSingleBlobProcessorBase::Properties, std::to_array<core::PropertyReference>({ diff --git a/extensions/azure/processors/FetchAzureDataLakeStorage.h b/extensions/azure/processors/FetchAzureDataLakeStorage.h index bb9c4d338..f7ab3a11d 100644 --- a/extensions/azure/processors/FetchAzureDataLakeStorage.h +++ b/extensions/azure/processors/FetchAzureDataLakeStorage.h @@ -42,7 +42,7 @@ class FetchAzureDataLakeStorage final : public AzureDataLakeStorageFileProcessor .build(); EXTENSIONAPI static constexpr auto RangeLength = core::PropertyDefinitionBuilder<>::createProperty("Range Length") .withDescription("The number of bytes to download from the object, starting from the Range Start. " - "An empty value or a value that extends beyond the end of the object will read to the end of the object.") + "An empty value or a value that extends beyond the end of the object will read to the end of the object.") .supportsExpressionLanguage(true) .build(); EXTENSIONAPI static constexpr auto NumberOfRetries = core::PropertyDefinitionBuilder<>::createProperty("Number of Retries") diff --git a/extensions/azure/processors/PutAzureDataLakeStorage.cpp b/extensions/azure/processors/PutAzureDataLakeStorage.cpp index 00f4874cf..c1531de2c 100644 --- a/extensions/azure/processors/PutAzureDataLakeStorage.cpp +++ b/extensions/azure/processors/PutAzureDataLakeStorage.cpp @@ -36,17 +36,6 @@ void PutAzureDataLakeStorage::initialize() { void PutAzureDataLakeStorage::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { AzureDataLakeStorageFileProcessorBase::onSchedule(context, session_factory); - std::optional<storage::AzureStorageCredentials> credentials; - std::tie(std::ignore, credentials) = getCredentialsFromControllerService(context); - if (!credentials) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Azure Storage Credentials Service property missing or invalid"); - } - - if (!credentials->isValid()) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Azure Storage Credentials Service properties are not set or invalid"); - } - - credentials_ = *credentials; conflict_resolution_strategy_ = utils::parseEnumProperty<azure::FileExistsResolutionStrategy>(context, ConflictResolutionStrategy); } @@ -123,7 +112,7 @@ int64_t PutAzureDataLakeStorage::ReadCallback::operator()(const std::shared_ptr< } result_ = azure_data_lake_storage_.uploadFile(params_, buffer); - return read_ret; + return gsl::narrow<int64_t>(read_ret); } REGISTER_RESOURCE(PutAzureDataLakeStorage, Processor); diff --git a/extensions/azure/storage/AzureBlobStorageClient.cpp b/extensions/azure/storage/AzureBlobStorageClient.cpp index 9393008ba..be6186a7f 100644 --- a/extensions/azure/storage/AzureBlobStorageClient.cpp +++ b/extensions/azure/storage/AzureBlobStorageClient.cpp @@ -56,13 +56,13 @@ AzureBlobStorageClient::AzureBlobStorageClient() { } Azure::Storage::Blobs::BlobContainerClient AzureBlobStorageClient::createClient(const AzureStorageCredentials &credentials, const std::string &container_name) { - if (credentials.getUseManagedIdentityCredentials()) { - auto storage_client = Azure::Storage::Blobs::BlobServiceClient( - "https://"; + credentials.getStorageAccountName() + ".blob." + credentials.getEndpointSuffix(), std::make_shared<Azure::Identity::ManagedIdentityCredential>()); - return storage_client.GetBlobContainerClient(container_name); - } else { + if (credentials.getCredentialConfigurationStrategy() == CredentialConfigurationStrategyOption::FromProperties) { return Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(credentials.buildConnectionString(), container_name); } + + auto storage_client = Azure::Storage::Blobs::BlobServiceClient("https://"; + credentials.getStorageAccountName() + ".blob." + credentials.getEndpointSuffix(), + credentials.createAzureTokenCredential()); + return storage_client.GetBlobContainerClient(container_name); } bool AzureBlobStorageClient::createContainerIfNotExists(const PutAzureBlobStorageParameters& params) { diff --git a/extensions/azure/storage/AzureDataLakeStorageClient.cpp b/extensions/azure/storage/AzureDataLakeStorageClient.cpp index 7f588139f..d7618818e 100644 --- a/extensions/azure/storage/AzureDataLakeStorageClient.cpp +++ b/extensions/azure/storage/AzureDataLakeStorageClient.cpp @@ -42,14 +42,14 @@ std::unique_ptr<Azure::Storage::Files::DataLake::DataLakeFileSystemClient> Azure options.Retry.MaxRetries = gsl::narrow<int32_t>(*number_of_retries); } - if (credentials.getUseManagedIdentityCredentials()) { - auto datalake_service_client = Azure::Storage::Files::DataLake::DataLakeServiceClient( - "https://"; + credentials.getStorageAccountName() + ".dfs." + credentials.getEndpointSuffix(), std::make_shared<Azure::Identity::ManagedIdentityCredential>(), options); - return std::make_unique<Azure::Storage::Files::DataLake::DataLakeFileSystemClient>(datalake_service_client.GetFileSystemClient(file_system_name)); - } else { + if (credentials.getCredentialConfigurationStrategy() == CredentialConfigurationStrategyOption::FromProperties) { return std::make_unique<Azure::Storage::Files::DataLake::DataLakeFileSystemClient>( Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString(credentials.buildConnectionString(), file_system_name, options)); } + + auto datalake_service_client = Azure::Storage::Files::DataLake::DataLakeServiceClient( + "https://"; + credentials.getStorageAccountName() + ".dfs." + credentials.getEndpointSuffix(), credentials.createAzureTokenCredential(), options); + return std::make_unique<Azure::Storage::Files::DataLake::DataLakeFileSystemClient>(datalake_service_client.GetFileSystemClient(file_system_name)); } Azure::Storage::Files::DataLake::DataLakeDirectoryClient AzureDataLakeStorageClient::getDirectoryClient(const AzureDataLakeStorageParameters& params) { diff --git a/extensions/azure/storage/AzureStorageCredentials.cpp b/extensions/azure/storage/AzureStorageCredentials.cpp index 070dfe860..7af832d22 100644 --- a/extensions/azure/storage/AzureStorageCredentials.cpp +++ b/extensions/azure/storage/AzureStorageCredentials.cpp @@ -34,7 +34,7 @@ void AzureStorageCredentials::setSasToken(const std::string& sas_token) { sas_token_ = sas_token; } -void AzureStorageCredentials::setEndpontSuffix(const std::string& endpoint_suffix) { +void AzureStorageCredentials::setEndpointSuffix(const std::string& endpoint_suffix) { endpoint_suffix_ = endpoint_suffix; } @@ -42,8 +42,12 @@ void AzureStorageCredentials::setConnectionString(const std::string& connection_ connection_string_ = connection_string; } -void AzureStorageCredentials::setUseManagedIdentityCredentials(bool use_managed_identity_credentials) { - use_managed_identity_credentials_ = use_managed_identity_credentials; +void AzureStorageCredentials::setCredentialConfigurationStrategy(CredentialConfigurationStrategyOption credential_configuration_strategy) { + credential_configuration_strategy_ = credential_configuration_strategy; +} + +void AzureStorageCredentials::setManagedIdentityClientId(const std::string& managed_identity_client_id) { + managed_identity_client_id_ = managed_identity_client_id; } std::string AzureStorageCredentials::getStorageAccountName() const { @@ -54,12 +58,16 @@ std::string AzureStorageCredentials::getEndpointSuffix() const { return endpoint_suffix_.empty() ? "core.windows.net" : endpoint_suffix_; } -bool AzureStorageCredentials::getUseManagedIdentityCredentials() const { - return use_managed_identity_credentials_; +CredentialConfigurationStrategyOption AzureStorageCredentials::getCredentialConfigurationStrategy() const { + return credential_configuration_strategy_; +} + +std::string AzureStorageCredentials::getManagedIdentityClientId() const { + return managed_identity_client_id_; } std::string AzureStorageCredentials::buildConnectionString() const { - if (use_managed_identity_credentials_) { + if (credential_configuration_strategy_ != CredentialConfigurationStrategyOption::FromProperties) { return ""; } @@ -67,6 +75,10 @@ std::string AzureStorageCredentials::buildConnectionString() const { return connection_string_; } + if (!storage_account_key_.empty() && !sas_token_.empty()) { + return ""; + } + if (storage_account_name_.empty() || (storage_account_key_.empty() && sas_token_.empty())) { return ""; } @@ -90,16 +102,35 @@ std::string AzureStorageCredentials::buildConnectionString() const { } bool AzureStorageCredentials::isValid() const { - return (getUseManagedIdentityCredentials() && !getStorageAccountName().empty()) || - (!getUseManagedIdentityCredentials() && !buildConnectionString().empty()); + return (credential_configuration_strategy_ != CredentialConfigurationStrategyOption::FromProperties && !getStorageAccountName().empty()) || + (credential_configuration_strategy_ == CredentialConfigurationStrategyOption::FromProperties && !buildConnectionString().empty()); +} + +std::shared_ptr<Azure::Core::Credentials::TokenCredential> AzureStorageCredentials::createAzureTokenCredential() const { + std::shared_ptr<Azure::Core::Credentials::TokenCredential> credential; + if (credential_configuration_strategy_ == CredentialConfigurationStrategyOption::ManagedIdentity) { + Azure::Identity::ManagedIdentityCredentialOptions options; + if (managed_identity_client_id_.empty()) { + options.IdentityId = Azure::Identity::ManagedIdentityId::SystemAssigned(); + } else { + options.IdentityId = Azure::Identity::ManagedIdentityId::FromUserAssignedClientId(managed_identity_client_id_); + } + credential = std::make_shared<Azure::Identity::ManagedIdentityCredential>(options); + } else if (credential_configuration_strategy_ == CredentialConfigurationStrategyOption::DefaultCredential) { + credential = std::make_shared<Azure::Identity::DefaultAzureCredential>(); + } else if (credential_configuration_strategy_ == CredentialConfigurationStrategyOption::WorkloadIdentity) { + credential = std::make_shared<Azure::Identity::WorkloadIdentityCredential>(); + } + + return credential; } bool AzureStorageCredentials::operator==(const AzureStorageCredentials& other) const { - if (other.use_managed_identity_credentials_ != use_managed_identity_credentials_) { + if (other.credential_configuration_strategy_ != credential_configuration_strategy_) { return false; } - if (use_managed_identity_credentials_) { + if (credential_configuration_strategy_ != CredentialConfigurationStrategyOption::FromProperties) { return storage_account_name_ == other.storage_account_name_ && endpoint_suffix_ == other.endpoint_suffix_; } else { return buildConnectionString() == other.buildConnectionString(); diff --git a/extensions/azure/storage/AzureStorageCredentials.h b/extensions/azure/storage/AzureStorageCredentials.h index ea14d2fb8..b156b5361 100644 --- a/extensions/azure/storage/AzureStorageCredentials.h +++ b/extensions/azure/storage/AzureStorageCredentials.h @@ -20,6 +20,10 @@ #pragma once #include <string> +#include <memory> + +#include "utils/AzureEnums.h" +#include "azure/identity.hpp" namespace org::apache::nifi::minifi::azure::storage { @@ -28,16 +32,20 @@ class AzureStorageCredentials { void setStorageAccountName(const std::string& storage_account_name); void setStorageAccountKey(const std::string& storage_account_key); void setSasToken(const std::string& sas_token); - void setEndpontSuffix(const std::string& endpoint_suffix); + void setEndpointSuffix(const std::string& endpoint_suffix); void setConnectionString(const std::string& connection_string); - void setUseManagedIdentityCredentials(bool use_managed_identity_credentials); + void setCredentialConfigurationStrategy(CredentialConfigurationStrategyOption credential_configuration_strategy); + void setManagedIdentityClientId(const std::string& managed_identity_client_id); std::string getStorageAccountName() const; std::string getEndpointSuffix() const; - bool getUseManagedIdentityCredentials() const; + CredentialConfigurationStrategyOption getCredentialConfigurationStrategy() const; + std::string getManagedIdentityClientId() const; std::string buildConnectionString() const; bool isValid() const; + std::shared_ptr<Azure::Core::Credentials::TokenCredential> createAzureTokenCredential() const; + bool operator==(const AzureStorageCredentials& other) const; private: @@ -46,7 +54,8 @@ class AzureStorageCredentials { std::string sas_token_; std::string endpoint_suffix_; std::string connection_string_; - bool use_managed_identity_credentials_ = false; + std::string managed_identity_client_id_; + CredentialConfigurationStrategyOption credential_configuration_strategy_ = CredentialConfigurationStrategyOption::FromProperties; }; } // namespace org::apache::nifi::minifi::azure::storage diff --git a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp index 7c7d790fd..b554021bb 100644 --- a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp @@ -134,23 +134,61 @@ TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test credentials settings" REQUIRE(failed_flowfiles[0] == TEST_DATA); } - SECTION("Account name and managed identity are used in properties") { + SECTION("Account name and Azure default identity sources are used in properties") { + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } + plan_->setProperty(azure_blob_storage_processor_, "Storage Account Name", STORAGE_ACCOUNT_NAME); - plan_->setProperty(azure_blob_storage_processor_, "Use Managed Identity Credentials", "true"); + plan_->setProperty(azure_blob_storage_processor_, "Credential Configuration Strategy", credential_configuration_strategy_string); + plan_->setProperty(azure_blob_storage_processor_, "Managed Identity Client ID", managed_identity_client_id); test_controller_.runSession(plan_, true); CHECK(getFailedFlowFileContents().empty()); auto passed_params = mock_blob_storage_ptr_->getPassedDeleteParams(); CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == STORAGE_ACCOUNT_NAME); CHECK(passed_params.credentials.getEndpointSuffix() == "core.windows.net"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(passed_params.container_name == CONTAINER_NAME); } - SECTION("Account name and managed identity are used from Azure Storage Credentials Service") { + SECTION("Account name and Azure default identity sources are used from Azure Storage Credentials Service") { + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } + auto azure_storage_cred_service = plan_->addController("AzureStorageCredentialsService", "AzureStorageCredentialsService"); plan_->setProperty(azure_storage_cred_service, "Storage Account Name", STORAGE_ACCOUNT_NAME); - plan_->setProperty(azure_storage_cred_service, "Use Managed Identity Credentials", "true"); + plan_->setProperty(azure_storage_cred_service, "Credential Configuration Strategy", credential_configuration_strategy_string); plan_->setProperty(azure_storage_cred_service, "Common Storage Account Endpoint Suffix", "core.chinacloudapi.cn"); + plan_->setProperty(azure_storage_cred_service, "Managed Identity Client ID", managed_identity_client_id); plan_->setProperty(azure_blob_storage_processor_, "Azure Storage Credentials Service", "AzureStorageCredentialsService"); test_controller_.runSession(plan_, true); CHECK(getFailedFlowFileContents().empty()); @@ -158,6 +196,8 @@ TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test credentials settings" CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == STORAGE_ACCOUNT_NAME); CHECK(passed_params.credentials.getEndpointSuffix() == "core.chinacloudapi.cn"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(passed_params.container_name == CONTAINER_NAME); } @@ -209,6 +249,32 @@ TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test credentials settings" REQUIRE(failed_flowfiles.size() == 1); REQUIRE(failed_flowfiles[0] == TEST_DATA); } + + SECTION("Both SAS Token and Storage Account Key cannot be set in credentials service") { + auto azure_storage_cred_service = plan_->addController("AzureStorageCredentialsService", "AzureStorageCredentialsService"); + plan_->setProperty(azure_storage_cred_service, "Storage Account Name", STORAGE_ACCOUNT_NAME); + plan_->setProperty(azure_storage_cred_service, "Storage Account Key", STORAGE_ACCOUNT_KEY); + plan_->setProperty(azure_storage_cred_service, "SAS Token", SAS_TOKEN); + plan_->setProperty(azure_blob_storage_processor_, "Azure Storage Credentials Service", "AzureStorageCredentialsService"); + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedDeleteParams(); + REQUIRE(passed_params.credentials.buildConnectionString().empty()); + auto failed_flowfiles = getFailedFlowFileContents(); + REQUIRE(failed_flowfiles.size() == 1); + REQUIRE(failed_flowfiles[0] == TEST_DATA); + } + + SECTION("Both SAS Token and Storage Account Key cannot be set in properties") { + plan_->setProperty(azure_blob_storage_processor_, "Storage Account Name", STORAGE_ACCOUNT_NAME); + plan_->setProperty(azure_blob_storage_processor_, "Storage Account Key", STORAGE_ACCOUNT_KEY); + plan_->setProperty(azure_blob_storage_processor_, "SAS Token", SAS_TOKEN); + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedDeleteParams(); + REQUIRE(passed_params.credentials.buildConnectionString().empty()); + auto failed_flowfiles = getFailedFlowFileContents(); + REQUIRE(failed_flowfiles.size() == 1); + REQUIRE(failed_flowfiles[0] == TEST_DATA); + } } TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test Azure blob delete failure in case Blob is not set and filename is empty", "[azureBlobStorageDelete]") { diff --git a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp index 40bacdca5..a241429c2 100644 --- a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp @@ -54,19 +54,47 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Test Azure credentials CHECK(getFailedFlowFileContents().empty()); } -TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Test Azure credentials with managed identity use", "[azureDataLakeStorageParameters]") { +TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Test Azure credentials with Azure default identity sources", "[azureDataLakeStorageParameters]") { setDefaultProperties(); + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ConnectionString, "test"); - plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::UseManagedIdentityCredentials, "true"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::CredentialConfigurationStrategy, credential_configuration_strategy_string); plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountName, "TEST_ACCOUNT"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ManagedIdentityClientId, managed_identity_client_id); test_controller_.runSession(plan_, true); auto passed_params = mock_data_lake_storage_client_ptr_->getPassedDeleteParams(); CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == "TEST_ACCOUNT"); CHECK(passed_params.credentials.getEndpointSuffix() == "core.windows.net"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(getFailedFlowFileContents().empty()); } +TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Both SAS Token and Storage Account Key cannot be set in credentials service") { + setDefaultProperties(); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ConnectionString, ""); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::SASToken, "token"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountName, "TEST_ACCOUNT"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountKey, "TEST_KEY"); + REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); +} + TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Filesystem name is not set", "[azureDataLakeStorageParameters]") { plan_->setDynamicProperty(update_attribute_, "test.filesystemname", ""); test_controller_.runSession(plan_, true); diff --git a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp index a1e901744..cd57961cb 100644 --- a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp @@ -130,23 +130,61 @@ TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Test credentials settings", REQUIRE(failed_flowfiles[0] == TEST_DATA); } - SECTION("Account name and managed identity are used in properties") { + SECTION("Account name and Azure default identity sources are used in properties") { + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } + plan_->setProperty(azure_blob_storage_processor_, "Storage Account Name", STORAGE_ACCOUNT_NAME); - plan_->setProperty(azure_blob_storage_processor_, "Use Managed Identity Credentials", "true"); + plan_->setProperty(azure_blob_storage_processor_, "Credential Configuration Strategy", credential_configuration_strategy_string); + plan_->setProperty(azure_blob_storage_processor_, "Managed Identity Client ID", managed_identity_client_id); test_controller_.runSession(plan_, true); CHECK(getFailedFlowFileContents().empty()); auto passed_params = mock_blob_storage_ptr_->getPassedFetchParams(); CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == STORAGE_ACCOUNT_NAME); CHECK(passed_params.credentials.getEndpointSuffix() == "core.windows.net"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(passed_params.container_name == CONTAINER_NAME); } - SECTION("Account name and managed identity are used from Azure Storage Credentials Service") { + SECTION("Account name and Azure default identity sources are used from Azure Storage Credentials Service") { + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } + auto azure_storage_cred_service = plan_->addController("AzureStorageCredentialsService", "AzureStorageCredentialsService"); plan_->setProperty(azure_storage_cred_service, "Storage Account Name", STORAGE_ACCOUNT_NAME); - plan_->setProperty(azure_storage_cred_service, "Use Managed Identity Credentials", "true"); + plan_->setProperty(azure_storage_cred_service, "Credential Configuration Strategy", credential_configuration_strategy_string); plan_->setProperty(azure_storage_cred_service, "Common Storage Account Endpoint Suffix", "core.chinacloudapi.cn"); + plan_->setProperty(azure_storage_cred_service, "Managed Identity Client ID", managed_identity_client_id); plan_->setProperty(azure_blob_storage_processor_, "Azure Storage Credentials Service", "AzureStorageCredentialsService"); test_controller_.runSession(plan_, true); CHECK(getFailedFlowFileContents().empty()); @@ -154,6 +192,8 @@ TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Test credentials settings", CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == STORAGE_ACCOUNT_NAME); CHECK(passed_params.credentials.getEndpointSuffix() == "core.chinacloudapi.cn"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(passed_params.container_name == CONTAINER_NAME); } @@ -205,6 +245,32 @@ TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Test credentials settings", REQUIRE(failed_flowfiles.size() == 1); REQUIRE(failed_flowfiles[0] == TEST_DATA); } + + SECTION("Both SAS Token and Storage Account Key cannot be set in credentials service") { + auto azure_storage_cred_service = plan_->addController("AzureStorageCredentialsService", "AzureStorageCredentialsService"); + plan_->setProperty(azure_storage_cred_service, "Storage Account Name", STORAGE_ACCOUNT_NAME); + plan_->setProperty(azure_storage_cred_service, "Storage Account Key", STORAGE_ACCOUNT_KEY); + plan_->setProperty(azure_storage_cred_service, "SAS Token", SAS_TOKEN); + plan_->setProperty(azure_blob_storage_processor_, "Azure Storage Credentials Service", "AzureStorageCredentialsService"); + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedFetchParams(); + REQUIRE(passed_params.credentials.buildConnectionString().empty()); + auto failed_flowfiles = getFailedFlowFileContents(); + REQUIRE(failed_flowfiles.size() == 1); + REQUIRE(failed_flowfiles[0] == TEST_DATA); + } + + SECTION("Both SAS Token and Storage Account Key cannot be set in properties") { + plan_->setProperty(azure_blob_storage_processor_, "Storage Account Name", STORAGE_ACCOUNT_NAME); + plan_->setProperty(azure_blob_storage_processor_, "Storage Account Key", STORAGE_ACCOUNT_KEY); + plan_->setProperty(azure_blob_storage_processor_, "SAS Token", SAS_TOKEN); + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedFetchParams(); + REQUIRE(passed_params.credentials.buildConnectionString().empty()); + auto failed_flowfiles = getFailedFlowFileContents(); + REQUIRE(failed_flowfiles.size() == 1); + REQUIRE(failed_flowfiles[0] == TEST_DATA); + } } TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Test Azure blob fetch failure in case Blob is not set and filename is empty", "[azureBlobStorageFetch]") { diff --git a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp index 70d63bc6d..c5505f49e 100644 --- a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp @@ -54,19 +54,47 @@ TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Test Azure credentials CHECK(getFailedFlowFileContents().empty()); } -TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Test Azure credentials with managed identity use", "[azureDataLakeStorageParameters]") { +TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Test Azure credentials with Azure default identity sources", "[azureDataLakeStorageParameters]") { setDefaultProperties(); + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ConnectionString, "test"); - plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::UseManagedIdentityCredentials, "true"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::CredentialConfigurationStrategy, credential_configuration_strategy_string); plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountName, "TEST_ACCOUNT"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ManagedIdentityClientId, managed_identity_client_id); test_controller_.runSession(plan_, true); auto passed_params = mock_data_lake_storage_client_ptr_->getPassedFetchParams(); CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == "TEST_ACCOUNT"); CHECK(passed_params.credentials.getEndpointSuffix() == "core.windows.net"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(getFailedFlowFileContents().empty()); } +TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Both SAS Token and Storage Account Key cannot be set in credentials service") { + setDefaultProperties(); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ConnectionString, ""); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::SASToken, "token"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountName, "TEST_ACCOUNT"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountKey, "TEST_KEY"); + REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); +} + TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Filesystem name is not set", "[azureDataLakeStorageParameters]") { plan_->setDynamicProperty(update_attribute_, "test.filesystemname", ""); test_controller_.runSession(plan_, true); diff --git a/extensions/azure/tests/ListAzureBlobStorageTests.cpp b/extensions/azure/tests/ListAzureBlobStorageTests.cpp index 2717469ec..31bdad909 100644 --- a/extensions/azure/tests/ListAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/ListAzureBlobStorageTests.cpp @@ -168,28 +168,68 @@ TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "Test credentials settings", REQUIRE(passed_params.credentials.buildConnectionString() == CONNECTION_STRING); } - SECTION("Account name and managed identity are used in properties") { + SECTION("Account name and Azure default identity sources are used in properties") { + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } + plan_->setProperty(list_azure_blob_storage_, "Storage Account Name", STORAGE_ACCOUNT_NAME); - plan_->setProperty(list_azure_blob_storage_, "Use Managed Identity Credentials", "true"); + plan_->setProperty(list_azure_blob_storage_, "Credential Configuration Strategy", credential_configuration_strategy_string); + plan_->setProperty(list_azure_blob_storage_, "Managed Identity Client ID", managed_identity_client_id); test_controller_.runSession(plan_, true); auto passed_params = mock_blob_storage_ptr_->getPassedListParams(); CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == STORAGE_ACCOUNT_NAME); CHECK(passed_params.credentials.getEndpointSuffix() == "core.windows.net"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(passed_params.container_name == CONTAINER_NAME); } - SECTION("Account name and managed identity are used from Azure Storage Credentials Service") { + SECTION("Account name and Azure default identity sources are used from Azure Storage Credentials Service") { + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } + auto azure_storage_cred_service = plan_->addController("AzureStorageCredentialsService", "AzureStorageCredentialsService"); plan_->setProperty(azure_storage_cred_service, "Storage Account Name", STORAGE_ACCOUNT_NAME); - plan_->setProperty(azure_storage_cred_service, "Use Managed Identity Credentials", "true"); + plan_->setProperty(azure_storage_cred_service, "Credential Configuration Strategy", credential_configuration_strategy_string); plan_->setProperty(azure_storage_cred_service, "Common Storage Account Endpoint Suffix", "core.chinacloudapi.cn"); + plan_->setProperty(azure_storage_cred_service, "Managed Identity Client ID", managed_identity_client_id); plan_->setProperty(list_azure_blob_storage_, "Azure Storage Credentials Service", "AzureStorageCredentialsService"); test_controller_.runSession(plan_, true); auto passed_params = mock_blob_storage_ptr_->getPassedListParams(); CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == STORAGE_ACCOUNT_NAME); CHECK(passed_params.credentials.getEndpointSuffix() == "core.chinacloudapi.cn"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(passed_params.container_name == CONTAINER_NAME); } @@ -224,6 +264,22 @@ TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "Test credentials settings", plan_->setProperty(list_azure_blob_storage_, "Storage Account Key", STORAGE_ACCOUNT_KEY); REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); } + + SECTION("Both SAS Token and Storage Account Key cannot be set in credentials service") { + auto azure_storage_cred_service = plan_->addController("AzureStorageCredentialsService", "AzureStorageCredentialsService"); + plan_->setProperty(azure_storage_cred_service, "Storage Account Name", STORAGE_ACCOUNT_NAME); + plan_->setProperty(azure_storage_cred_service, "Storage Account Key", STORAGE_ACCOUNT_KEY); + plan_->setProperty(azure_storage_cred_service, "SAS Token", SAS_TOKEN); + plan_->setProperty(list_azure_blob_storage_, "Azure Storage Credentials Service", "AzureStorageCredentialsService"); + REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); + } + + SECTION("Both SAS Token and Storage Account Key cannot be set in properties") { + plan_->setProperty(list_azure_blob_storage_, "Storage Account Name", STORAGE_ACCOUNT_NAME); + plan_->setProperty(list_azure_blob_storage_, "Storage Account Key", STORAGE_ACCOUNT_KEY); + plan_->setProperty(list_azure_blob_storage_, "SAS Token", SAS_TOKEN); + REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); + } } TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "List all files every time", "[ListAzureBlobStorage]") { diff --git a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp index 4cc0fa5c7..4f2a19b51 100644 --- a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp @@ -43,7 +43,7 @@ class ListAzureDataLakeStorageTestsFixture { auto mock_data_lake_storage_client = std::make_unique<MockDataLakeStorageClient>(); mock_data_lake_storage_client_ptr_ = mock_data_lake_storage_client.get(); auto uuid = utils::IdGenerator::getIdGenerator()->generate(); - auto impl = std::unique_ptr<minifi::azure::processors::ListAzureDataLakeStorage>( + auto impl = std::unique_ptr<minifi::azure::processors::ListAzureDataLakeStorage>( // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) new minifi::azure::processors::ListAzureDataLakeStorage({ .uuid = uuid, .name = "ListAzureDataLakeStorage", .logger = logging::LoggerFactory<minifi::azure::processors::ListAzureDataLakeStorage>::getLogger(uuid)}, std::move(mock_data_lake_storage_client))); @@ -216,4 +216,44 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Throw on invalid path fi REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); } +TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Test Azure credentials with Azure default identity sources", "[azureDataLakeStorageParameters]") { + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ConnectionString, "test"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::CredentialConfigurationStrategy, credential_configuration_strategy_string); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountName, "TEST_ACCOUNT"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ManagedIdentityClientId, managed_identity_client_id); + test_controller_.runSession(plan_, true); + auto passed_params = mock_data_lake_storage_client_ptr_->getPassedListParams(); + CHECK(passed_params.credentials.buildConnectionString().empty()); + CHECK(passed_params.credentials.getStorageAccountName() == "TEST_ACCOUNT"); + CHECK(passed_params.credentials.getEndpointSuffix() == "core.windows.net"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); +} + +TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Both SAS Token and Storage Account Key cannot be set in credentials service") { + setDefaultProperties(); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ConnectionString, ""); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::SASToken, "token"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountName, "TEST_ACCOUNT"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountKey, "TEST_KEY"); + REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); +} + } // namespace diff --git a/extensions/azure/tests/PutAzureBlobStorageTests.cpp b/extensions/azure/tests/PutAzureBlobStorageTests.cpp index 1b4e6cc7b..a4a0ed19e 100644 --- a/extensions/azure/tests/PutAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/PutAzureBlobStorageTests.cpp @@ -134,23 +134,61 @@ TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test credentials settings", " REQUIRE(failed_flowfiles[0] == TEST_DATA); } - SECTION("Account name and managed identity are used in properties") { + SECTION("Account name and Azure default identity sources are used in properties") { + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } + plan_->setProperty(azure_blob_storage_processor_, "Storage Account Name", STORAGE_ACCOUNT_NAME); - plan_->setProperty(azure_blob_storage_processor_, "Use Managed Identity Credentials", "true"); + plan_->setProperty(azure_blob_storage_processor_, "Credential Configuration Strategy", credential_configuration_strategy_string); + plan_->setProperty(azure_blob_storage_processor_, "Managed Identity Client ID", managed_identity_client_id); test_controller_.runSession(plan_, true); CHECK(getFailedFlowFileContents().empty()); auto passed_params = mock_blob_storage_ptr_->getPassedPutParams(); CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == STORAGE_ACCOUNT_NAME); CHECK(passed_params.credentials.getEndpointSuffix() == "core.windows.net"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(passed_params.container_name == CONTAINER_NAME); } - SECTION("Account name and managed identity are used from Azure Storage Credentials Service") { + SECTION("Account name and Azure default identity sources are used from Azure Storage Credentials Service") { + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } + auto azure_storage_cred_service = plan_->addController("AzureStorageCredentialsService", "AzureStorageCredentialsService"); plan_->setProperty(azure_storage_cred_service, "Storage Account Name", STORAGE_ACCOUNT_NAME); - plan_->setProperty(azure_storage_cred_service, "Use Managed Identity Credentials", "true"); + plan_->setProperty(azure_storage_cred_service, "Credential Configuration Strategy", credential_configuration_strategy_string); plan_->setProperty(azure_storage_cred_service, "Common Storage Account Endpoint Suffix", "core.chinacloudapi.cn"); + plan_->setProperty(azure_storage_cred_service, "Managed Identity Client ID", managed_identity_client_id); plan_->setProperty(azure_blob_storage_processor_, "Azure Storage Credentials Service", "AzureStorageCredentialsService"); test_controller_.runSession(plan_, true); CHECK(getFailedFlowFileContents().empty()); @@ -158,6 +196,8 @@ TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test credentials settings", " CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == STORAGE_ACCOUNT_NAME); CHECK(passed_params.credentials.getEndpointSuffix() == "core.chinacloudapi.cn"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); CHECK(passed_params.container_name == CONTAINER_NAME); } @@ -209,6 +249,32 @@ TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test credentials settings", " REQUIRE(failed_flowfiles.size() == 1); REQUIRE(failed_flowfiles[0] == TEST_DATA); } + + SECTION("Both SAS Token and Storage Account Key cannot be set in credentials service") { + auto azure_storage_cred_service = plan_->addController("AzureStorageCredentialsService", "AzureStorageCredentialsService"); + plan_->setProperty(azure_storage_cred_service, "Storage Account Name", STORAGE_ACCOUNT_NAME); + plan_->setProperty(azure_storage_cred_service, "Storage Account Key", STORAGE_ACCOUNT_KEY); + plan_->setProperty(azure_storage_cred_service, "SAS Token", SAS_TOKEN); + plan_->setProperty(azure_blob_storage_processor_, "Azure Storage Credentials Service", "AzureStorageCredentialsService"); + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedPutParams(); + REQUIRE(passed_params.credentials.buildConnectionString().empty()); + auto failed_flowfiles = getFailedFlowFileContents(); + REQUIRE(failed_flowfiles.size() == 1); + REQUIRE(failed_flowfiles[0] == TEST_DATA); + } + + SECTION("Both SAS Token and Storage Account Key cannot be set in properties") { + plan_->setProperty(azure_blob_storage_processor_, "Storage Account Name", STORAGE_ACCOUNT_NAME); + plan_->setProperty(azure_blob_storage_processor_, "Storage Account Key", STORAGE_ACCOUNT_KEY); + plan_->setProperty(azure_blob_storage_processor_, "SAS Token", SAS_TOKEN); + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedPutParams(); + REQUIRE(passed_params.credentials.buildConnectionString().empty()); + auto failed_flowfiles = getFailedFlowFileContents(); + REQUIRE(failed_flowfiles.size() == 1); + REQUIRE(failed_flowfiles[0] == TEST_DATA); + } } TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test Azure blob upload failure in case Blob is not set and filename is empty", "[azureBlobStorageUpload]") { diff --git a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp index 9ea94222d..6938dc841 100644 --- a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp @@ -54,19 +54,47 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Test Azure credentials wi REQUIRE(getFailedFlowFileContents().empty()); } -TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Test Azure credentials with managed identity use", "[azureDataLakeStorageParameters]") { +TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Test Azure credentials with Azure default identity sources", "[azureDataLakeStorageParameters]") { setDefaultProperties(); + minifi::azure::CredentialConfigurationStrategyOption expected_configuration_strategy_option{}; + std::string credential_configuration_strategy_string; + std::string managed_identity_client_id; + SECTION("Managed Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::ManagedIdentity; + credential_configuration_strategy_string = "Managed Identity"; + managed_identity_client_id = "test-managed-identity-client-id"; + } + SECTION("Default Credential") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::DefaultCredential; + credential_configuration_strategy_string = "Default Credential"; + } + SECTION("Workload Identity") { + expected_configuration_strategy_option = minifi::azure::CredentialConfigurationStrategyOption::WorkloadIdentity; + credential_configuration_strategy_string = "Workload Identity"; + } plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ConnectionString, "test"); - plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::UseManagedIdentityCredentials, "true"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::CredentialConfigurationStrategy, credential_configuration_strategy_string); plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountName, "TEST_ACCOUNT"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ManagedIdentityClientId, managed_identity_client_id); test_controller_.runSession(plan_, true); auto passed_params = mock_data_lake_storage_client_ptr_->getPassedPutParams(); CHECK(passed_params.credentials.buildConnectionString().empty()); CHECK(passed_params.credentials.getStorageAccountName() == "TEST_ACCOUNT"); CHECK(passed_params.credentials.getEndpointSuffix() == "core.windows.net"); + CHECK(passed_params.credentials.getCredentialConfigurationStrategy() == expected_configuration_strategy_option); + CHECK(passed_params.credentials.getManagedIdentityClientId() == managed_identity_client_id); REQUIRE(getFailedFlowFileContents().empty()); } +TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Both SAS Token and Storage Account Key cannot be set in credentials service") { + setDefaultProperties(); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::ConnectionString, ""); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::SASToken, "token"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountName, "TEST_ACCOUNT"); + plan_->setProperty(azure_storage_cred_service_, minifi::azure::controllers::AzureStorageCredentialsService::StorageAccountKey, "TEST_KEY"); + REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); +} + TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Filesystem name is not set", "[azureDataLakeStorageParameters]") { plan_->setDynamicProperty(update_attribute_, "test.filesystemname", ""); test_controller_.runSession(plan_, true); diff --git a/extensions/azure/utils/AzureEnums.h b/extensions/azure/utils/AzureEnums.h index 28a54535b..ab87836e1 100644 --- a/extensions/azure/utils/AzureEnums.h +++ b/extensions/azure/utils/AzureEnums.h @@ -19,6 +19,8 @@ */ #pragma once +#include "magic_enum.hpp" + namespace org::apache::nifi::minifi::azure { enum class EntityTracking { @@ -26,4 +28,30 @@ enum class EntityTracking { timestamps }; +enum class CredentialConfigurationStrategyOption { + FromProperties, + DefaultCredential, + ManagedIdentity, + WorkloadIdentity +}; + } // namespace org::apache::nifi::minifi::azure + +namespace magic_enum::customize { +using CredentialConfigurationStrategyOption = org::apache::nifi::minifi::azure::CredentialConfigurationStrategyOption; + +template <> +constexpr customize_t enum_name<CredentialConfigurationStrategyOption>(CredentialConfigurationStrategyOption value) noexcept { + switch (value) { + case CredentialConfigurationStrategyOption::FromProperties: + return "From Properties"; + case CredentialConfigurationStrategyOption::DefaultCredential: + return "Default Credential"; + case CredentialConfigurationStrategyOption::ManagedIdentity: + return "Managed Identity"; + case CredentialConfigurationStrategyOption::WorkloadIdentity: + return "Workload Identity"; + } + return invalid_tag; +} +} // namespace magic_enum::customize
