This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 76b375d998 [GLUTEN-6887][VL] Daily Update Velox Version (2024_11_08)
(#7854)
76b375d998 is described below
commit 76b375d9983d096111a438cadf71017124297194
Author: Gluten Performance Bot
<[email protected]>
AuthorDate: Fri Nov 8 20:36:36 2024 +0800
[GLUTEN-6887][VL] Daily Update Velox Version (2024_11_08) (#7854)
Upstream Velox's New Commits:
d76c05c7d by Deepak Majeti, Add support for S3 bucket config (11321)
327fa89fd by Christian Zentgraf, fix: Installation of HDFS on Ubuntu (11420)
8c69b8b76 by Chengcheng Jin, Support prefix sort in spilling (11384)
64661a31d by Jaime Pan, Return null for invalid datetime format when legacy
date formatter is used (11131)
d1bf9da3d by Bikramjeet Vig, Add support for using negative array indices
in json path (11451)
789ce652f by Ke, Rename tableStorageFormat in HiveInsertTableHandle (11458)
aba7ba815 by Kevin Wilfong, Fix inconsistencies between Velox and Presto
Java when formatting time zone short/long names
---
cpp/velox/utils/ConfigExtractor.cc | 39 +++++++++++++++++++++++++-------------
ep/build-velox/src/get_velox.sh | 2 +-
2 files changed, 27 insertions(+), 14 deletions(-)
diff --git a/cpp/velox/utils/ConfigExtractor.cc
b/cpp/velox/utils/ConfigExtractor.cc
index a83045b03e..3175c375fa 100644
--- a/cpp/velox/utils/ConfigExtractor.cc
+++ b/cpp/velox/utils/ConfigExtractor.cc
@@ -22,6 +22,7 @@
#include "utils/Exception.h"
#include "velox/connectors/hive/HiveConfig.h"
+#include "velox/connectors/hive/storage_adapters/s3fs/S3Config.h"
namespace {
@@ -98,27 +99,39 @@ std::shared_ptr<facebook::velox::config::ConfigBase>
getHiveConfig(
}
if (useInstanceCredentials) {
-
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3UseInstanceCredentials]
= "true";
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+
facebook::velox::filesystems::S3Config::Keys::kUseInstanceCredentials)] =
"true";
} else if (!iamRole.empty()) {
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3IamRole] =
iamRole;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kIamRole)] = iamRole;
if (!iamRoleSessionName.empty()) {
-
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3IamRoleSessionName]
= iamRoleSessionName;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kIamRoleSessionName)]
= iamRoleSessionName;
}
} else {
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3AwsAccessKey]
= awsAccessKey;
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3AwsSecretKey]
= awsSecretKey;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kAccessKey)] =
awsAccessKey;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kSecretKey)] =
awsSecretKey;
}
// Only need to set s3 endpoint when not use instance credentials.
if (!useInstanceCredentials) {
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3Endpoint] =
awsEndpoint;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kEndpoint)] =
awsEndpoint;
}
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3SSLEnabled] =
sslEnabled ? "true" : "false";
-
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3PathStyleAccess] =
pathStyleAccess ? "true" : "false";
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3LogLevel] =
awsSdkLogLevel;
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3MaxAttempts] =
retryMaxAttempts;
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3RetryMode] =
retryMode;
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3MaxConnections]
= maxConnections;
- hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3ConnectTimeout]
= connectTimeout;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kSSLEnabled)] = sslEnabled
? "true" : "false";
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kPathStyleAccess)] =
pathStyleAccess ? "true" : "false";
+ hiveConfMap[facebook::velox::filesystems::S3Config::kS3LogLevel] =
awsSdkLogLevel;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kMaxAttempts)] =
retryMaxAttempts;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kRetryMode)] = retryMode;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kMaxConnections)] =
maxConnections;
+ hiveConfMap[facebook::velox::filesystems::S3Config::baseConfigKey(
+ facebook::velox::filesystems::S3Config::Keys::kConnectTimeout)] =
connectTimeout;
#endif
#ifdef ENABLE_GCS
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 37dc20aedb..60c7ea0a48 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_11_07
+VELOX_BRANCH=2024_11_08
VELOX_HOME=""
OS=`uname -s`
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
