Copilot commented on code in PR #13646:
URL: https://github.com/apache/gluten/pull/13646#discussion_r4583947016


##########
cpp/velox/tests/GlutenS3FileSystemTest.cc:
##########
@@ -68,5 +68,39 @@ TEST(GlutenS3FileSystemTest, extractsAsyncUploadConfigs) {
   EXPECT_EQ(hiveConfig->get<std::string>(prefix + "upload-threads"), "32");
 }
 
+TEST(GlutenS3FileSystemTest, extractsPerBucketConfigs) {
+  using Keys = filesystems::S3Config::Keys;
+  auto config = 
std::make_shared<velox::config::ConfigBase>(std::unordered_map<std::string, 
std::string>{
+      {"spark.hadoop.fs.s3a.bucket.plain-bucket.endpoint", 
"http://plain:9000"},
+      {"spark.hadoop.fs.s3a.bucket.my.data.bucket.endpoint", 
"http://dotted:9000"},
+      {"spark.hadoop.fs.s3a.bucket.my.data.bucket.access.key", 
"dotted-access"},
+      {"spark.hadoop.fs.s3a.bucket.my.data.bucket.endpoint.region", 
"us-west-2"},
+      {"spark.hadoop.fs.s3a.bucket.my.data.bucket.iam.role.session.name", 
"dotted-session"},
+      {"spark.hadoop.fs.s3a.bucket.my.data.bucket.unknown.option", 
"ignored"}});
+
+  const auto hiveConfig = createHiveConnectorConfig(config, 
FileSystemType::kS3);
+
+  auto bucketValue = [&](Keys key, std::string_view bucket) {
+    return 
hiveConfig->get<std::string>(filesystems::S3Config::bucketConfigKey(key, 
bucket));
+  };
+  EXPECT_EQ(bucketValue(Keys::kEndpoint, "plain-bucket"), "http://plain:9000";);
+  EXPECT_EQ(bucketValue(Keys::kEndpoint, "my.data.bucket"), 
"http://dotted:9000";);
+  EXPECT_EQ(bucketValue(Keys::kAccessKey, "my.data.bucket"), "dotted-access");
+  EXPECT_EQ(bucketValue(Keys::kEndpointRegion, "my.data.bucket"), "us-west-2");
+  EXPECT_EQ(bucketValue(Keys::kIamRoleSessionName, "my.data.bucket"), 
"dotted-session");
+
+  // Nothing is registered under a bucket name truncated at the first dot.
+  EXPECT_FALSE(bucketValue(Keys::kEndpoint, "my").has_value());
+
+  // Only the five recognized options above produce per-bucket Velox configs.
+  size_t numBucketConfigs = 0;
+  for (const auto& [key, value] : hiveConfig->rawConfigs()) {
+    if (key.rfind(filesystems::S3Config::kS3BucketPrefix, 0) == 0) {
+      ++numBucketConfigs;
+    }
+  }
+  EXPECT_EQ(numBucketConfigs, 5);

Review Comment:
   This assertion is brittle: if `sparkSuffixes` gains additional recognized 
per-bucket options later (or if more per-bucket entries are emitted for other 
reasons), the test would fail even though parsing remains correct. Prefer 
asserting absence of the specific unknown option-derived key (or deriving the 
expected count from the set of keys explicitly asserted earlier) instead of 
hard-coding `5`.



##########
cpp/velox/utils/ConfigExtractor.cc:
##########
@@ -84,14 +84,25 @@ void getS3HiveConfig(
       {S3Config::Keys::kIMDSEnabled, std::make_pair("aws.imds.enabled", 
"true")},
   };
 
-  // get Velox S3 config key from Spark Suffix.
-  auto getVeloxKey = [&](std::string_view suffix) {
+  // Split "<bucket>.<suffix>" into the bucket name and the Velox S3 config 
key. Bucket names may contain dots,
+  // so match a known Spark suffix at the end instead of splitting at the 
first dot. The longest matching suffix
+  // wins, e.g. "endpoint.region" over "endpoint".
+  auto parseBucketConfig = [&](std::string_view bucketAndSuffix) {
+    std::optional<std::pair<S3Config::Keys, std::string_view>> result;
+    size_t matchedSuffixSize = 0;
     for (const auto& [key, value] : sparkSuffixes) {
-      if (value.first == suffix) {
-        return std::optional<S3Config::Keys>(key);
+      const std::string_view suffix = value.first;
+      // At least one character for the bucket name plus the separating dot.
+      if (bucketAndSuffix.size() < suffix.size() + 2 || suffix.size() <= 
matchedSuffixSize) {
+        continue;
+      }
+      const auto dotPos = bucketAndSuffix.size() - suffix.size() - 1;
+      if (bucketAndSuffix[dotPos] == '.' && bucketAndSuffix.substr(dotPos + 1) 
== suffix) {
+        result = std::make_pair(key, bucketAndSuffix.substr(0, dotPos));
+        matchedSuffixSize = suffix.size();
       }
     }
-    return std::optional<S3Config::Keys>(std::nullopt);
+    return result;
   };

Review Comment:
   The “longest matching suffix wins” logic is correct but a bit harder to 
follow/extend with the `matchedSuffixSize` tracking and full scan on every 
config key. Consider precomputing a list of `(suffix, key)` entries sorted by 
descending suffix length (once, e.g., as a `static const` vector) and then 
returning on the first match; this simplifies the control flow and makes it 
easier to reason about future suffix additions.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to