adamdebreceni commented on a change in pull request #975:
URL: https://github.com/apache/nifi-minifi-cpp/pull/975#discussion_r561703605
##########
File path: extensions/aws/s3/S3Wrapper.h
##########
@@ -31,11 +47,190 @@ namespace minifi {
namespace aws {
namespace s3 {
-class S3Wrapper : public S3WrapperBase {
+static const std::unordered_map<std::string, Aws::S3::Model::StorageClass>
STORAGE_CLASS_MAP {
+ {"Standard", Aws::S3::Model::StorageClass::STANDARD},
+ {"ReducedRedundancy", Aws::S3::Model::StorageClass::REDUCED_REDUNDANCY},
+ {"StandardIA", Aws::S3::Model::StorageClass::STANDARD_IA},
+ {"OnezoneIA", Aws::S3::Model::StorageClass::ONEZONE_IA},
+ {"IntelligentTiering", Aws::S3::Model::StorageClass::INTELLIGENT_TIERING},
+ {"Glacier", Aws::S3::Model::StorageClass::GLACIER},
+ {"DeepArchive", Aws::S3::Model::StorageClass::DEEP_ARCHIVE}
+};
+
+static const std::unordered_map<Aws::S3::Model::ObjectStorageClass,
std::string> OBJECT_STORAGE_CLASS_MAP {
+ {Aws::S3::Model::ObjectStorageClass::STANDARD, "Standard"},
+ {Aws::S3::Model::ObjectStorageClass::REDUCED_REDUNDANCY,
"ReducedRedundancy"},
+ {Aws::S3::Model::ObjectStorageClass::STANDARD_IA, "StandardIA"},
+ {Aws::S3::Model::ObjectStorageClass::ONEZONE_IA, "OnezoneIA"},
+ {Aws::S3::Model::ObjectStorageClass::INTELLIGENT_TIERING,
"IntelligentTiering"},
+ {Aws::S3::Model::ObjectStorageClass::GLACIER, "Glacier"},
+ {Aws::S3::Model::ObjectStorageClass::DEEP_ARCHIVE, "DeepArchive"}
+};
+
+static const std::unordered_map<Aws::S3::Model::ObjectVersionStorageClass,
std::string> VERSION_STORAGE_CLASS_MAP {
+ {Aws::S3::Model::ObjectVersionStorageClass::STANDARD, "Standard"}
+};
+
+static const std::unordered_map<std::string,
Aws::S3::Model::ServerSideEncryption> SERVER_SIDE_ENCRYPTION_MAP {
+ {"None", Aws::S3::Model::ServerSideEncryption::NOT_SET},
+ {"AES256", Aws::S3::Model::ServerSideEncryption::AES256},
+ {"aws_kms", Aws::S3::Model::ServerSideEncryption::aws_kms},
+};
+
+static const std::unordered_map<std::string, Aws::S3::Model::ObjectCannedACL>
CANNED_ACL_MAP {
+ {"BucketOwnerFullControl",
Aws::S3::Model::ObjectCannedACL::bucket_owner_full_control},
+ {"BucketOwnerRead", Aws::S3::Model::ObjectCannedACL::bucket_owner_read},
+ {"AuthenticatedRead", Aws::S3::Model::ObjectCannedACL::authenticated_read},
+ {"PublicReadWrite", Aws::S3::Model::ObjectCannedACL::public_read_write},
+ {"PublicRead", Aws::S3::Model::ObjectCannedACL::public_read},
+ {"Private", Aws::S3::Model::ObjectCannedACL::private_},
+ {"AwsExecRead", Aws::S3::Model::ObjectCannedACL::aws_exec_read},
+};
+
+struct Expiration {
+ std::string expiration_time;
+ std::string expiration_time_rule_id;
+};
+
+struct PutObjectResult {
+ std::string version;
+ std::string etag;
+ std::string expiration;
+ std::string ssealgorithm;
+};
+
+struct PutObjectRequestParameters {
+ std::string bucket;
+ std::string object_key;
+ std::string storage_class;
+ std::string server_side_encryption;
+ std::string content_type;
+ std::map<std::string, std::string> user_metadata_map;
+ std::string fullcontrol_user_list;
+ std::string read_permission_user_list;
+ std::string read_acl_user_list;
+ std::string write_acl_user_list;
+ std::string canned_acl;
+};
+
+struct GetObjectRequestParameters {
+ std::string bucket;
+ std::string object_key;
+ std::string version;
+ bool requester_pays = false;
+};
+
+struct HeadObjectResult {
+ std::string path;
+ std::string absolute_path;
+ std::string filename;
+ std::string mime_type;
+ std::string etag;
+ Expiration expiration;
+ std::string ssealgorithm;
+ std::string version;
+ std::map<std::string, std::string> user_metadata_map;
+
+ void setFilePaths(const std::string& key);
+};
+
+struct GetObjectResult : public HeadObjectResult {
+ int64_t write_size = 0;
+};
+
+struct ListRequestParameters {
+ std::string bucket;
+ std::string delimiter;
+ std::string prefix;
+ bool use_versions = false;
+ uint64_t min_object_age = 0;
+};
+
+struct ListedObjectAttributes {
+ std::string filename;
+ std::string etag;
+ bool is_latest = false;
+ int64_t last_modified = 0;
+ int length = 0;
+ std::string store_class;
+ std::string version;
Review comment:
it seems like `version` is an optional field and we treat the empty
string as the `null`, now there are many examples of this throughout our
codebase so it is not unheard of, but do you think using `optional` to make
this explicit could be beneficial?
##########
File path: extensions/aws/processors/ListS3.cpp
##########
@@ -0,0 +1,294 @@
+/**
+ * @file ListS3.cpp
+ * ListS3 class implementation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ListS3.h"
+
+#include <tuple>
+#include <algorithm>
+#include <set>
+#include <utility>
+#include <memory>
+
+#include "utils/StringUtils.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace aws {
+namespace processors {
+
+const std::string ListS3::LATEST_LISTED_KEY_PREFIX = "listed_key.";
+const std::string ListS3::LATEST_LISTED_KEY_TIMESTAMP = "listed_key.timestamp";
+
+const core::Property ListS3::Delimiter(
+ core::PropertyBuilder::createProperty("Delimiter")
+ ->withDescription("The string used to delimit directories within the
bucket. Please consult the AWS documentation for the correct use of this
field.")
+ ->build());
+const core::Property ListS3::Prefix(
+ core::PropertyBuilder::createProperty("Prefix")
+ ->withDescription("The prefix used to filter the object list. In most
cases, it should end with a forward slash ('/').")
+ ->build());
+const core::Property ListS3::UseVersions(
+ core::PropertyBuilder::createProperty("Use Versions")
+ ->isRequired(true)
+ ->withDefaultValue<bool>(false)
+ ->withDescription("Specifies whether to use S3 versions, if applicable. If
false, only the latest version of each object will be returned.")
+ ->build());
+const core::Property ListS3::MinimumObjectAge(
+ core::PropertyBuilder::createProperty("Minimum Object Age")
+ ->isRequired(true)
+ ->withDefaultValue<core::TimePeriodValue>("0 sec")
+ ->withDescription("The minimum age that an S3 object must be in order to
be considered; any object younger than this amount of time (according to last
modification date) will be ignored.")
+ ->build());
+const core::Property ListS3::WriteObjectTags(
+ core::PropertyBuilder::createProperty("Write Object Tags")
+ ->isRequired(true)
+ ->withDefaultValue<bool>(false)
+ ->withDescription("If set to 'true', the tags associated with the S3
object will be written as FlowFile attributes.")
+ ->build());
+const core::Property ListS3::WriteUserMetadata(
+ core::PropertyBuilder::createProperty("Write User Metadata")
+ ->isRequired(true)
+ ->withDefaultValue<bool>(false)
+ ->withDescription("If set to 'true', the user defined metadata associated
with the S3 object will be added to FlowFile attributes/records.")
+ ->build());
+const core::Property ListS3::RequesterPays(
+ core::PropertyBuilder::createProperty("Requester Pays")
+ ->isRequired(true)
+ ->withDefaultValue<bool>(false)
+ ->withDescription("If true, indicates that the requester consents to pay
any charges associated with listing the S3 bucket. This sets the
'x-amz-request-payer' header to 'requester'. "
+ "Note that this setting is only used if Write User
Metadata is true.")
+ ->build());
+
+const core::Relationship ListS3::Success("success", "FlowFiles are routed to
success relationship");
+
+void ListS3::initialize() {
+ // Add new supported properties
+ updateSupportedProperties({Delimiter, Prefix, UseVersions, MinimumObjectAge,
WriteObjectTags, WriteUserMetadata, RequesterPays});
+ // Set the supported relationships
+ setSupportedRelationships({Success});
+}
+
+void ListS3::onSchedule(const std::shared_ptr<core::ProcessContext> &context,
const std::shared_ptr<core::ProcessSessionFactory> &sessionFactory) {
+ S3Processor::onSchedule(context, sessionFactory);
+
+ state_manager_ = context->getStateManager();
+ if (state_manager_ == nullptr) {
+ throw Exception(PROCESSOR_EXCEPTION, "Failed to get StateManager");
+ }
+
+ auto common_properties = getCommonELSupportedProperties(context, nullptr);
Review comment:
passing `nullptr` will call a possible expression script with `nullptr`,
will that properly throw an exception? e.g. it seems like `Bucket` property
supports expression language, what happens if the user accidentally supplies a
script value for that property that tries to access the null flowFile?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
