jackylee-ch commented on code in PR #12213:
URL: https://github.com/apache/gluten/pull/12213#discussion_r3335209252
##########
cpp/velox/utils/VeloxWriterUtils.cc:
##########
@@ -49,6 +49,17 @@ std::unique_ptr<WriterOptions> makeParquetWriteOption(const
std::unordered_map<s
}
auto writeOption = std::make_unique<WriterOptions>();
writeOption->parquetWriteTimestampUnit = TimestampPrecision::kMicroseconds
/*micro*/;
+ bool writeLegacyParquetFormat = false;
+ if (auto it = sparkConfs.find(kParquetStoreDecimalAsInteger); it !=
sparkConfs.end()) {
+ writeLegacyParquetFormat = boost::iequals(it->second, "true");
+ }
+ // Maps spark.sql.parquet.writeLegacyFormat to Velox
enableStoreDecimalAsInteger (inverted).
+ // Controls how DECIMAL values are stored by the Writer:
+ // - true (default when legacy format is off): store as integer using
INT32/INT64 for short
+ // DECIMAL precisions; higher precisions use FIXED_LEN_BYTE_ARRAY.
+ // - false (when spark.sql.parquet.writeLegacyFormat is true): store as
FIXED_LEN_BYTE_ARRAY
+ // regardless of precision (Spark legacy Parquet decimal layout).
+ writeOption->enableStoreDecimalAsInteger = !writeLegacyParquetFormat;
Review Comment:
do we have any suite tests for this? if not, please add some.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
