alamb commented on code in PR #8540:
URL: https://github.com/apache/arrow-rs/pull/8540#discussion_r2399703980
##########
parquet/src/arrow/schema/primitive.rs:
##########
@@ -132,35 +133,24 @@ fn from_parquet(parquet_type: &Type) -> Result<DataType> {
}
fn decimal_type(scale: i32, precision: i32) -> Result<DataType> {
- if precision <= DECIMAL128_MAX_PRECISION as i32 {
- decimal_128_type(scale, precision)
- } else {
- decimal_256_type(scale, precision)
- }
-}
-
-fn decimal_128_type(scale: i32, precision: i32) -> Result<DataType> {
+ // Convert and validate inputs once
let scale = scale
.try_into()
.map_err(|_| arrow_err!("scale cannot be negative: {}", scale))?;
-
let precision = precision
.try_into()
.map_err(|_| arrow_err!("precision cannot be negative: {}",
precision))?;
- Ok(DataType::Decimal128(precision, scale))
-}
-
-fn decimal_256_type(scale: i32, precision: i32) -> Result<DataType> {
- let scale = scale
- .try_into()
- .map_err(|_| arrow_err!("scale cannot be negative: {}", scale))?;
-
- let precision = precision
- .try_into()
- .map_err(|_| arrow_err!("precision cannot be negative: {}",
precision))?;
-
- Ok(DataType::Decimal256(precision, scale))
+ // Dispatch based on precision thresholds using DecimalType trait constants
+ if precision <= Decimal32Type::MAX_PRECISION {
Review Comment:
Is this change needed for this PR?
Decimal32/Decimal64 were just added recently and it is not widely supported
by all the kernels, or downstream systems (e.g. DataFusion)
I think this change means that some parquet files that currently are read as
Decimal128 would come back as Decimal32/Decimal64 which is likely to be a
pretty big change to some consumers.
I would suggest backing out this part of the change, if possble, and filing
a separate ticket to discuss changing how decimal data is read from existing
parquet files
If we want to proceed, we should probably need to add additional tests that
show what happens when decimals are read from parquet.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
