This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 5f8db8cc79 [GLUTEN-9625][VL] Add explicit GCS auth type check for
service account key file (#9627)
5f8db8cc79 is described below
commit 5f8db8cc79509793d29cea9386a596cf0429b294
Author: Pratham Manja <[email protected]>
AuthorDate: Thu May 15 01:51:46 2025 +0530
[GLUTEN-9625][VL] Add explicit GCS auth type check for service account key
file (#9627)
Have added validation for the case when spark.hadoop.fs.gs.auth.type is not
set, or invalid. This would fail the application with prompt msg indicating the
missing or invalid config.
Fixes: #9625
---
cpp/velox/utils/ConfigExtractor.cc | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/cpp/velox/utils/ConfigExtractor.cc
b/cpp/velox/utils/ConfigExtractor.cc
index a5730ae589..995706bde2 100644
--- a/cpp/velox/utils/ConfigExtractor.cc
+++ b/cpp/velox/utils/ConfigExtractor.cc
@@ -186,20 +186,20 @@ std::shared_ptr<facebook::velox::config::ConfigBase>
getHiveConfig(
//
https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcs/CONFIGURATION.md#authentication
auto gsAuthType = conf->get<std::string>("spark.hadoop.fs.gs.auth.type");
- if (gsAuthType.hasValue()) {
- std::string type = gsAuthType.value();
- if (type == "SERVICE_ACCOUNT_JSON_KEYFILE") {
- auto gsAuthServiceAccountJsonKeyfile =
-
conf->get<std::string>("spark.hadoop.fs.gs.auth.service.account.json.keyfile");
- if (gsAuthServiceAccountJsonKeyfile.hasValue()) {
-
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGcsCredentialsPath] =
- gsAuthServiceAccountJsonKeyfile.value();
- } else {
- LOG(WARNING) << "STARTUP: conf spark.hadoop.fs.gs.auth.type is set to
SERVICE_ACCOUNT_JSON_KEYFILE, "
- "however conf
spark.hadoop.fs.gs.auth.service.account.json.keyfile is not set";
- throw GlutenException("Conf
spark.hadoop.fs.gs.auth.service.account.json.keyfile is not set");
- }
+ auto gsAuthServiceAccountJsonKeyfile =
conf->get<std::string>("spark.hadoop.fs.gs.auth.service.account.json.keyfile");
+ if (gsAuthType.hasValue() && gsAuthType.value() ==
"SERVICE_ACCOUNT_JSON_KEYFILE") {
+ if (gsAuthServiceAccountJsonKeyfile.hasValue()) {
+
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGcsCredentialsPath] =
+ gsAuthServiceAccountJsonKeyfile.value();
+ } else {
+ LOG(WARNING) << "STARTUP: conf spark.hadoop.fs.gs.auth.type is set to
SERVICE_ACCOUNT_JSON_KEYFILE, "
+ "however conf
spark.hadoop.fs.gs.auth.service.account.json.keyfile is not set";
+ throw GlutenException("Conf
spark.hadoop.fs.gs.auth.service.account.json.keyfile is not set");
}
+ } else if (gsAuthServiceAccountJsonKeyfile.hasValue()) {
+ LOG(WARNING) << "STARTUP: conf
spark.hadoop.fs.gs.auth.service.account.json.keyfile is set, "
+ "but conf spark.hadoop.fs.gs.auth.type is not
SERVICE_ACCOUNT_JSON_KEYFILE";
+ throw GlutenException("Conf spark.hadoop.fs.gs.auth.type is missing or
incorrect");
}
#endif
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
