This is an automated email from the ASF dual-hosted git repository.
rui pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new e5652ee80d [VL] Pass timestamp timezone to Parquet write options
(#8317)
e5652ee80d is described below
commit e5652ee80d9cf6564af03180cd04ac36d6ca6d37
Author: Rui Mo <[email protected]>
AuthorDate: Tue Feb 11 11:35:35 2025 +0000
[VL] Pass timestamp timezone to Parquet write options (#8317)
---
.../sql/execution/datasources/velox/VeloxParquetWriterInjects.scala | 5 +++++
cpp/velox/operators/writer/VeloxParquetDataSource.cc | 1 +
ep/build-velox/src/get_velox.sh | 2 +-
3 files changed, 7 insertions(+), 1 deletion(-)
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala
index a75fec0285..a66b271cb1 100644
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala
@@ -39,6 +39,11 @@ class VeloxParquetWriterInjects extends
VeloxFormatWriterInjects {
GlutenConfig.PARQUET_BLOCK_ROWS,
GlutenConfig.get.columnarParquetWriteBlockRows.toString)
sparkOptions.put(GlutenConfig.PARQUET_BLOCK_ROWS, blockRows)
+ sparkOptions.put(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key,
+ options.getOrElse(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key,
+ SQLConf.SESSION_LOCAL_TIMEZONE.defaultValueString))
options
.get(GlutenConfig.PARQUET_GZIP_WINDOW_SIZE)
.foreach(sparkOptions.put(GlutenConfig.PARQUET_GZIP_WINDOW_SIZE, _))
diff --git a/cpp/velox/operators/writer/VeloxParquetDataSource.cc
b/cpp/velox/operators/writer/VeloxParquetDataSource.cc
index af8e233ad0..15082b8b82 100644
--- a/cpp/velox/operators/writer/VeloxParquetDataSource.cc
+++ b/cpp/velox/operators/writer/VeloxParquetDataSource.cc
@@ -97,6 +97,7 @@ void VeloxParquetDataSource::init(const
std::unordered_map<std::string, std::str
return std::make_unique<velox::parquet::LambdaFlushPolicy>(
maxRowGroupRows_, maxRowGroupBytes_, [&]() { return false; });
};
+ writeOption.parquetWriteTimestampTimeZone = getConfigValue(sparkConfs,
kSessionTimezone, std::nullopt);
auto schema = gluten::fromArrowSchema(schema_);
parquetWriter_ = std::make_unique<velox::parquet::Writer>(std::move(sink_),
writeOption, pool_, asRowType(schema));
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 7c51af3fd2..5b094136d5 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2025_02_11
+VELOX_BRANCH=2025_02_11_fix
VELOX_HOME=""
OS=`uname -s`
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
