This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 6165cd94c feat(spec): replace rust_decimal with fastnum for 38-digit
precision (#2063)
6165cd94c is described below
commit 6165cd94c61c4e25af8d2e742c8d1671c9a5d01c
Author: Nathan Metzger <[email protected]>
AuthorDate: Thu Jan 29 18:20:57 2026 +0100
feat(spec): replace rust_decimal with fastnum for 38-digit precision (#2063)
## Summary
Replace rust_decimal with fastnum::D128 to support 38-digit decimal
precision as required by the Iceberg spec, addressing issue #669.
## Changes
- Add `crates/iceberg/src/spec/values/decimal_utils.rs` with:
- Compatibility layer providing rust_decimal-like API using
fastnum::D128
- `decimal_from_i128_with_scale()`, `decimal_mantissa()`,
`decimal_scale()` helpers
- `i128_from_be_bytes()` / `i128_to_be_bytes_min()` for binary
serialization
- Unit tests for all helper functions including 38-digit precision
validation
- Update `Cargo.toml` (workspace and iceberg crate):
- Add fastnum dependency with std and serde features
- Remove rust_decimal, num-bigint, num-traits dependencies
- Update `crates/iceberg/src/spec/values/datum.rs`:
- Replace rust_decimal imports with decimal_utils
- Update `from_bytes()` and `to_bytes()` for new decimal API
- Update `decimal_with_precision()` validation
- Update `crates/iceberg/src/spec/values/literal.rs`:
- Update JSON serialization/deserialization for decimals
- Update `crates/iceberg/src/transform/bucket.rs` and `truncate.rs`:
- Update decimal transform implementations
- Update `crates/iceberg/src/arrow/schema.rs` and `parquet_writer.rs`:
- Update Arrow/Parquet decimal statistics handling
- Replace `BigInt` byte conversions with helper functions
- Update `.cargo/audit.toml`:
- Remove RUSTSEC-2024-0399 ignore (rust_decimal vulnerability no longer
applies)
## Notes
fastnum::D128 provides exactly 38-digit precision with stack-based
storage (no heap allocation), meeting the Iceberg spec requirement that
rust_decimal (28-digit max) could not satisfy.
The decimal_utils module abstracts the API differences between
rust_decimal and fastnum, making the migration transparent to the rest
of the codebase.
Closes #669
---
.cargo/audit.toml | 3 -
Cargo.lock | 34 ++-
Cargo.toml | 3 +-
bindings/python/Cargo.toml | 5 +-
crates/iceberg/Cargo.toml | 3 +-
crates/iceberg/src/arrow/schema.rs | 20 +-
crates/iceberg/src/error.rs | 6 -
crates/iceberg/src/spec/mod.rs | 1 +
crates/iceberg/src/spec/transform.rs | 13 +-
crates/iceberg/src/spec/values/datum.rs | 66 ++---
crates/iceberg/src/spec/values/decimal_utils.rs | 322 +++++++++++++++++++++
crates/iceberg/src/spec/values/literal.rs | 21 +-
crates/iceberg/src/spec/values/mod.rs | 2 +
crates/iceberg/src/spec/values/tests.rs | 23 +-
crates/iceberg/src/transform/bucket.rs | 3 +-
crates/iceberg/src/transform/truncate.rs | 11 +-
.../src/writer/file_writer/parquet_writer.rs | 27 +-
17 files changed, 447 insertions(+), 116 deletions(-)
diff --git a/.cargo/audit.toml b/.cargo/audit.toml
index d403f0ac5..09e2d35c5 100644
--- a/.cargo/audit.toml
+++ b/.cargo/audit.toml
@@ -33,7 +33,4 @@ ignore = [
#
# Introduced by object_store, see
https://github.com/apache/arrow-rs-object-store/issues/564
"RUSTSEC-2025-0134",
-
- # Tracked here: https://github.com/paupino/rust-decimal/issues/766
- "RUSTSEC-2026-0001",
]
diff --git a/Cargo.lock b/Cargo.lock
index 211b6416c..577229645 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1058,6 +1058,16 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "bnum"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10"
+dependencies = [
+ "serde",
+ "serde-big-array",
+]
+
[[package]]
name = "bon"
version = "3.8.1"
@@ -2703,6 +2713,18 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+[[package]]
+name = "fastnum"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4089ab2dfd45d8ddc92febb5ca80644389d5ebb954f40231274a3f18341762e2"
+dependencies = [
+ "bnum",
+ "num-integer",
+ "num-traits",
+ "serde",
+]
+
[[package]]
name = "fastrand"
version = "2.3.0"
@@ -3367,6 +3389,7 @@ dependencies = [
"chrono",
"derive_builder",
"expect-test",
+ "fastnum",
"flate2",
"fnv",
"futures",
@@ -3376,7 +3399,6 @@ dependencies = [
"mockall",
"moka",
"murmur3",
- "num-bigint",
"once_cell",
"opendal",
"ordered-float 4.6.0",
@@ -3387,7 +3409,6 @@ dependencies = [
"reqsign",
"reqwest",
"roaring",
- "rust_decimal",
"serde",
"serde_bytes",
"serde_derive",
@@ -5966,6 +5987,15 @@ dependencies = [
"serde_derive",
]
+[[package]]
+name = "serde-big-array"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "serde_bytes"
version = "0.11.19"
diff --git a/Cargo.toml b/Cargo.toml
index 46ac4736b..900377c22 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -97,7 +97,6 @@ mockall = "0.13.1"
mockito = "1"
motore-macros = "0.4.3"
murmur3 = "0.5.2"
-num-bigint = "0.4.6"
once_cell = "1.20"
opendal = "0.55.0"
ordered-float = "4"
@@ -108,7 +107,7 @@ rand = "0.8.5"
regex = "1.11.3"
reqwest = { version = "0.12.12", default-features = false, features = ["json"]
}
roaring = { version = "0.11" }
-rust_decimal = { version = "1.39", default-features = false, features =
["std"] }
+fastnum = { version = "0.7", default-features = false, features = ["std",
"serde"] }
serde = { version = "1.0.219", features = ["rc"] }
serde_bytes = "0.11.17"
serde_derive = "1.0.219"
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index 8346d0270..c179e89f9 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -37,8 +37,6 @@ pyo3 = { version = "0.26", features = ["extension-module",
"abi3-py310"] }
iceberg-datafusion = { path = "../../crates/integrations/datafusion" }
datafusion-ffi = { version = "51.0" }
tokio = { version = "1.46.1", default-features = false }
-# Security: disable rkyv feature to avoid RUSTSEC-2026-0001 (rkyv 0.7.45
vulnerability)
-rust_decimal = { version = "1.39", default-features = false, features =
["std"] }
[profile.release]
codegen-units = 1
@@ -48,5 +46,4 @@ opt-level = "z"
strip = true
[package.metadata.cargo-machete]
-# rust_decimal is included to override feature flags for security (disable
rkyv)
-ignored = ["rust_decimal"]
+ignored = []
diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml
index 3835e77d1..d6d931c86 100644
--- a/crates/iceberg/Cargo.toml
+++ b/crates/iceberg/Cargo.toml
@@ -67,7 +67,6 @@ futures = { workspace = true }
itertools = { workspace = true }
moka = { version = "0.12.10", features = ["future"] }
murmur3 = { workspace = true }
-num-bigint = { workspace = true }
once_cell = { workspace = true }
opendal = { workspace = true }
ordered-float = { workspace = true }
@@ -76,7 +75,7 @@ rand = { workspace = true }
reqsign = { version = "0.16.3", optional = true, default-features = false }
reqwest = { workspace = true }
roaring = { workspace = true }
-rust_decimal = { workspace = true }
+fastnum = { workspace = true }
serde = { workspace = true }
serde_bytes = { workspace = true }
serde_derive = { workspace = true }
diff --git a/crates/iceberg/src/arrow/schema.rs
b/crates/iceberg/src/arrow/schema.rs
index b15e141c8..e00f79a3b 100644
--- a/crates/iceberg/src/arrow/schema.rs
+++ b/crates/iceberg/src/arrow/schema.rs
@@ -27,13 +27,12 @@ use arrow_array::{
TimestampMicrosecondArray,
};
use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema, TimeUnit};
-use num_bigint::BigInt;
use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
use parquet::file::statistics::Statistics;
-use rust_decimal::prelude::ToPrimitive;
use uuid::Uuid;
use crate::error::Result;
+use crate::spec::decimal_utils::i128_from_be_bytes;
use crate::spec::{
Datum, FIRST_FIELD_ID, ListType, MapType, NestedField, NestedFieldRef,
PrimitiveLiteral,
PrimitiveType, Schema, SchemaVisitor, StructType, Type,
@@ -680,7 +679,8 @@ impl SchemaVisitor for ToArrowSchemaConverter {
DataType::FixedSizeBinary(16),
)),
crate::spec::PrimitiveType::Fixed(len) =>
Ok(ArrowSchemaOrFieldOrType::Type(
- len.to_i32()
+ i32::try_from(*len)
+ .ok()
.map(DataType::FixedSizeBinary)
.unwrap_or(DataType::LargeBinary),
)),
@@ -722,10 +722,10 @@ pub(crate) fn get_arrow_datum(datum: &Datum) ->
Result<Arc<dyn ArrowDatum + Send
Ok(Arc::new(Int64Array::new_scalar(*value)))
}
(PrimitiveType::Float, PrimitiveLiteral::Float(value)) => {
- Ok(Arc::new(Float32Array::new_scalar(value.to_f32().unwrap())))
+ Ok(Arc::new(Float32Array::new_scalar(value.into_inner())))
}
(PrimitiveType::Double, PrimitiveLiteral::Double(value)) => {
- Ok(Arc::new(Float64Array::new_scalar(value.to_f64().unwrap())))
+ Ok(Arc::new(Float64Array::new_scalar(value.into_inner())))
}
(PrimitiveType::String, PrimitiveLiteral::String(value)) => {
Ok(Arc::new(StringArray::new_scalar(value.as_str())))
@@ -835,10 +835,9 @@ pub(crate) fn get_parquet_stat_min_as_datum(
let Some(bytes) = stats.min_bytes_opt() else {
return Ok(None);
};
- let unscaled_value = BigInt::from_signed_bytes_be(bytes);
Some(Datum::new(
primitive_type.clone(),
-
PrimitiveLiteral::Int128(unscaled_value.to_i128().ok_or_else(|| {
+
PrimitiveLiteral::Int128(i128_from_be_bytes(bytes).ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
format!("Can't convert bytes to i128: {bytes:?}"),
@@ -982,10 +981,9 @@ pub(crate) fn get_parquet_stat_max_as_datum(
let Some(bytes) = stats.max_bytes_opt() else {
return Ok(None);
};
- let unscaled_value = BigInt::from_signed_bytes_be(bytes);
Some(Datum::new(
primitive_type.clone(),
-
PrimitiveLiteral::Int128(unscaled_value.to_i128().ok_or_else(|| {
+
PrimitiveLiteral::Int128(i128_from_be_bytes(bytes).ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
format!("Can't convert bytes to i128: {bytes:?}"),
@@ -1295,9 +1293,9 @@ mod tests {
use std::sync::Arc;
use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit};
- use rust_decimal::Decimal;
use super::*;
+ use crate::spec::decimal_utils::decimal_new;
use crate::spec::{Literal, Schema};
/// Create a simple field with metadata.
@@ -2127,7 +2125,7 @@ mod tests {
assert_eq!(array.value(0), 42);
}
{
- let datum = Datum::decimal_with_precision(Decimal::new(123, 2),
30).unwrap();
+ let datum = Datum::decimal_with_precision(decimal_new(123, 2),
30).unwrap();
let arrow_datum = get_arrow_datum(&datum).unwrap();
let (array, is_scalar) = arrow_datum.get();
let array =
array.as_any().downcast_ref::<Decimal128Array>().unwrap();
diff --git a/crates/iceberg/src/error.rs b/crates/iceberg/src/error.rs
index 6ab3a78c8..8810ef500 100644
--- a/crates/iceberg/src/error.rs
+++ b/crates/iceberg/src/error.rs
@@ -408,12 +408,6 @@ define_from_err!(
"Failed to parse json string"
);
-define_from_err!(
- rust_decimal::Error,
- ErrorKind::DataInvalid,
- "Failed to convert decimal literal to rust decimal"
-);
-
define_from_err!(
parquet::errors::ParquetError,
ErrorKind::Unexpected,
diff --git a/crates/iceberg/src/spec/mod.rs b/crates/iceberg/src/spec/mod.rs
index a2b540f08..707ebbb63 100644
--- a/crates/iceberg/src/spec/mod.rs
+++ b/crates/iceberg/src/spec/mod.rs
@@ -52,6 +52,7 @@ pub use table_metadata::*;
pub(crate) use table_metadata_builder::FIRST_FIELD_ID;
pub use table_properties::*;
pub use transform::*;
+pub(crate) use values::decimal_utils;
pub use values::*;
pub use view_metadata::*;
pub use view_version::*;
diff --git a/crates/iceberg/src/spec/transform.rs
b/crates/iceberg/src/spec/transform.rs
index 026b12613..73fd290ee 100644
--- a/crates/iceberg/src/spec/transform.rs
+++ b/crates/iceberg/src/spec/transform.rs
@@ -24,6 +24,7 @@ use std::str::FromStr;
use fnv::FnvHashSet;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use super::values::decimal_utils::decimal_from_i128_with_scale;
use super::{Datum, PrimitiveLiteral};
use crate::ErrorKind;
use crate::error::{Error, Result};
@@ -660,7 +661,7 @@ impl Transform {
(PrimitiveType::Int, PrimitiveLiteral::Int(v)) =>
Some(Datum::int(v - 1)),
(PrimitiveType::Long, PrimitiveLiteral::Long(v)) =>
Some(Datum::long(v - 1)),
(PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v))
=> {
- Some(Datum::decimal(v - 1)?)
+ Some(Datum::decimal(decimal_from_i128_with_scale(v - 1,
0))?)
}
(PrimitiveType::Date, PrimitiveLiteral::Int(v)) =>
Some(Datum::date(v - 1)),
(PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => {
@@ -672,7 +673,7 @@ impl Transform {
(PrimitiveType::Int, PrimitiveLiteral::Int(v)) =>
Some(Datum::int(v + 1)),
(PrimitiveType::Long, PrimitiveLiteral::Long(v)) =>
Some(Datum::long(v + 1)),
(PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v))
=> {
- Some(Datum::decimal(v + 1)?)
+ Some(Datum::decimal(decimal_from_i128_with_scale(v + 1,
0))?)
}
(PrimitiveType::Date, PrimitiveLiteral::Int(v)) =>
Some(Datum::date(v + 1)),
(PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => {
@@ -806,7 +807,9 @@ impl Transform {
match (datum.data_type(), datum.literal()) {
(PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Ok(Datum::int(v
+ 1)),
(PrimitiveType::Long, PrimitiveLiteral::Long(v)) =>
Ok(Datum::long(v + 1)),
- (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) =>
Datum::decimal(v + 1),
+ (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => {
+ Datum::decimal(decimal_from_i128_with_scale(v + 1, 0))
+ }
(PrimitiveType::Date, PrimitiveLiteral::Int(v)) =>
Ok(Datum::date(v + 1)),
(PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => {
Ok(Datum::timestamp_micros(v + 1))
@@ -842,7 +845,9 @@ impl Transform {
match (datum.data_type(), datum.literal()) {
(PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Ok(Datum::int(v
- 1)),
(PrimitiveType::Long, PrimitiveLiteral::Long(v)) =>
Ok(Datum::long(v - 1)),
- (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) =>
Datum::decimal(v - 1),
+ (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => {
+ Datum::decimal(decimal_from_i128_with_scale(v - 1, 0))
+ }
(PrimitiveType::Date, PrimitiveLiteral::Int(v)) =>
Ok(Datum::date(v - 1)),
(PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => {
Ok(Datum::timestamp_micros(v - 1))
diff --git a/crates/iceberg/src/spec/values/datum.rs
b/crates/iceberg/src/spec/values/datum.rs
index 88209ae95..3d4abc019 100644
--- a/crates/iceberg/src/spec/values/datum.rs
+++ b/crates/iceberg/src/spec/values/datum.rs
@@ -22,15 +22,16 @@ use std::fmt::{Display, Formatter};
use std::str::FromStr;
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc};
-use num_bigint::BigInt;
use ordered_float::{Float, OrderedFloat};
-use rust_decimal::Decimal;
-use rust_decimal::prelude::ToPrimitive;
use serde::de::{self, MapAccess};
use serde::ser::SerializeStruct;
use serde::{Deserialize, Serialize};
use serde_bytes::ByteBuf;
+use super::decimal_utils::{
+ Decimal, decimal_from_i128_with_scale, decimal_from_str_exact,
decimal_mantissa, decimal_scale,
+ i128_from_be_bytes, i128_to_be_bytes_min,
+};
use super::literal::Literal;
use super::primitive::PrimitiveLiteral;
use super::serde::_serde::RawLiteral;
@@ -268,8 +269,8 @@ impl PartialOrd for Datum {
scale: other_scale,
},
) => {
- let val = Decimal::from_i128_with_scale(*val, *scale);
- let other_val = Decimal::from_i128_with_scale(*other_val,
*other_scale);
+ let val = decimal_from_i128_with_scale(*val, *scale);
+ let other_val = decimal_from_i128_with_scale(*other_val,
*other_scale);
val.partial_cmp(&other_val)
}
_ => None,
@@ -315,7 +316,7 @@ impl Display for Datum {
},
PrimitiveLiteral::Int128(val),
) => {
- write!(f, "{}", Decimal::from_i128_with_scale(*val, *scale))
+ write!(f, "{}", decimal_from_i128_with_scale(*val, *scale))
}
(_, _) => {
unreachable!()
@@ -407,8 +408,7 @@ impl Datum {
PrimitiveType::Fixed(_) =>
PrimitiveLiteral::Binary(Vec::from(bytes)),
PrimitiveType::Binary =>
PrimitiveLiteral::Binary(Vec::from(bytes)),
PrimitiveType::Decimal { .. } => {
- let unscaled_value = BigInt::from_signed_bytes_be(bytes);
-
PrimitiveLiteral::Int128(unscaled_value.to_i128().ok_or_else(|| {
+
PrimitiveLiteral::Int128(i128_from_be_bytes(bytes).ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
format!("Can't convert bytes to i128: {bytes:?}"),
@@ -461,10 +461,8 @@ impl Datum {
};
// The primitive literal is unscaled value.
- let unscaled_value = BigInt::from(*val);
- // Convert into two's-complement byte representation of the
BigInt
- // in big-endian byte order.
- let mut bytes = unscaled_value.to_signed_bytes_be();
+ // Convert into two's-complement byte representation in
big-endian byte order.
+ let mut bytes = i128_to_be_bytes_min(*val);
// Truncate with required bytes to make sure.
bytes.truncate(required_bytes as usize);
@@ -993,22 +991,18 @@ impl Datum {
}
}
- /// Creates decimal literal from string. See [`Decimal::from_str_exact`].
+ /// Creates decimal literal from string.
///
/// Example:
///
/// ```rust
/// use iceberg::spec::Datum;
- /// use itertools::assert_equal;
- /// use rust_decimal::Decimal;
/// let t = Datum::decimal_from_str("123.45").unwrap();
///
/// assert_eq!(&format!("{t}"), "123.45");
/// ```
pub fn decimal_from_str<S: AsRef<str>>(s: S) -> Result<Self> {
- let decimal = Decimal::from_str_exact(s.as_ref()).map_err(|e| {
- Error::new(ErrorKind::DataInvalid, "Can't parse
decimal.").with_source(e)
- })?;
+ let decimal = decimal_from_str_exact(s.as_ref())?;
Self::decimal(decimal)
}
@@ -1019,21 +1013,19 @@ impl Datum {
///
/// ```rust
/// use iceberg::spec::Datum;
- /// use rust_decimal::Decimal;
///
- /// let t = Datum::decimal(Decimal::new(123, 2)).unwrap();
+ /// let t = Datum::decimal_from_str("1.23").unwrap();
///
/// assert_eq!(&format!("{t}"), "1.23");
/// ```
- pub fn decimal(value: impl Into<Decimal>) -> Result<Self> {
- let decimal = value.into();
- let scale = decimal.scale();
+ pub fn decimal(value: Decimal) -> Result<Self> {
+ let scale = decimal_scale(&value);
let r#type = Type::decimal(MAX_DECIMAL_PRECISION, scale)?;
if let Type::Primitive(p) = r#type {
Ok(Self {
r#type: p,
- literal: PrimitiveLiteral::Int128(decimal.mantissa()),
+ literal: PrimitiveLiteral::Int128(decimal_mantissa(&value)),
})
} else {
unreachable!("Decimal type must be primitive.")
@@ -1042,27 +1034,19 @@ impl Datum {
/// Try to create a decimal literal from [`Decimal`] with precision.
///
- /// Example:
- ///
- /// ```rust
- /// use iceberg::spec::Datum;
- /// use rust_decimal::Decimal;
- ///
- /// let t = Datum::decimal_with_precision(Decimal::new(123, 2),
30).unwrap();
- ///
- /// assert_eq!(&format!("{t}"), "1.23");
- /// ```
- pub fn decimal_with_precision(value: impl Into<Decimal>, precision: u32)
-> Result<Self> {
- let decimal = value.into();
- let scale = decimal.scale();
+ /// This method allows specifying a custom precision for the decimal type,
+ /// which is useful when you need to control the storage requirements.
+ /// Use [`Datum::decimal`] if you want to use the maximum precision (38).
+ pub fn decimal_with_precision(value: Decimal, precision: u32) ->
Result<Self> {
+ let scale = decimal_scale(&value);
+ let mantissa = decimal_mantissa(&value);
let available_bytes = Type::decimal_required_bytes(precision)? as
usize;
- let unscaled_value = BigInt::from(decimal.mantissa());
- let actual_bytes = unscaled_value.to_signed_bytes_be();
+ let actual_bytes = i128_to_be_bytes_min(mantissa);
if actual_bytes.len() > available_bytes {
return Err(Error::new(
ErrorKind::DataInvalid,
- format!("Decimal value {decimal} is too large for precision
{precision}"),
+ format!("Decimal value {value} is too large for precision
{precision}"),
));
}
@@ -1070,7 +1054,7 @@ impl Datum {
if let Type::Primitive(p) = r#type {
Ok(Self {
r#type: p,
- literal: PrimitiveLiteral::Int128(decimal.mantissa()),
+ literal: PrimitiveLiteral::Int128(mantissa),
})
} else {
unreachable!("Decimal type must be primitive.")
diff --git a/crates/iceberg/src/spec/values/decimal_utils.rs
b/crates/iceberg/src/spec/values/decimal_utils.rs
new file mode 100644
index 000000000..88e3f72f6
--- /dev/null
+++ b/crates/iceberg/src/spec/values/decimal_utils.rs
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Compatibility layer for decimal operations.
+//!
+//! Provides rust_decimal-compatible API using fastnum's D128 internally.
+//! D128 supports 38-digit precision, meeting the Iceberg spec requirement.
+
+use fastnum::D128;
+use fastnum::decimal::Context;
+
+use crate::{Error, ErrorKind, Result};
+
+/// Re-export D128 as the Decimal type for use throughout the crate.
+pub type Decimal = D128;
+
+/// Create a D128 from mantissa (i128) and scale (u32).
+///
+/// This is equivalent to rust_decimal's `Decimal::from_i128_with_scale`.
+/// The value is computed as: mantissa * 10^(-scale)
+///
+/// For example:
+/// - mantissa=12345, scale=2 => 123.45
+/// - mantissa=-456, scale=3 => -0.456
+pub fn decimal_from_i128_with_scale(mantissa: i128, scale: u32) -> Decimal {
+ if scale == 0 {
+ return D128::from_i128(mantissa).expect("i128 always fits in D128");
+ }
+
+ // Convert mantissa to string and insert decimal point at the right
position
+ let is_negative = mantissa < 0;
+ let abs_str = mantissa.unsigned_abs().to_string();
+ let scale_usize = scale as usize;
+
+ let decimal_str = if abs_str.len() <= scale_usize {
+ // Need leading zeros: e.g., mantissa=456, scale=3 => "0.456"
+ // Or mantissa=5, scale=3 => "0.005"
+ let zeros_needed = scale_usize - abs_str.len();
+ format!(
+ "{}0.{}{}",
+ if is_negative { "-" } else { "" },
+ "0".repeat(zeros_needed),
+ abs_str
+ )
+ } else {
+ // Insert decimal point: e.g., mantissa=12345, scale=2 => "123.45"
+ let decimal_pos = abs_str.len() - scale_usize;
+ format!(
+ "{}{}.{}",
+ if is_negative { "-" } else { "" },
+ &abs_str[..decimal_pos],
+ &abs_str[decimal_pos..]
+ )
+ };
+
+ D128::from_str(&decimal_str, Context::default())
+ .expect("constructed decimal string is always valid")
+}
+
+/// Try to create a D128 from mantissa and scale, with validation.
+///
+/// This is equivalent to rust_decimal's `Decimal::try_from_i128_with_scale`.
+/// Currently always succeeds for 38-digit decimals.
+pub fn try_decimal_from_i128_with_scale(mantissa: i128, scale: u32) ->
Result<Decimal> {
+ // For now, always succeeds since D128 supports full 38-digit precision
+ Ok(decimal_from_i128_with_scale(mantissa, scale))
+}
+
+/// Create a D128 from i64 mantissa and scale.
+///
+/// This is equivalent to rust_decimal's `Decimal::new`.
+#[allow(dead_code)]
+pub fn decimal_new(mantissa: i64, scale: u32) -> Decimal {
+ decimal_from_i128_with_scale(mantissa as i128, scale)
+}
+
+/// Parse a decimal from string with exact representation.
+///
+/// This is equivalent to rust_decimal's `Decimal::from_str_exact`.
+pub fn decimal_from_str_exact(s: &str) -> Result<Decimal> {
+ D128::from_str(s, Context::default())
+ .map_err(|e| Error::new(ErrorKind::DataInvalid, format!("Can't parse
decimal: {e}")))
+}
+
+/// Get the mantissa (unscaled coefficient) as i128.
+///
+/// This is equivalent to rust_decimal's `decimal.mantissa()`.
+///
+/// The mantissa is signed: negative decimals return negative mantissa.
+pub fn decimal_mantissa(d: &Decimal) -> i128 {
+ // digits() returns unsigned coefficient as UInt<N>
+ // For Iceberg decimals (max 38 digits), this always fits in u128/i128
+ let digits = d.digits();
+
+ // Convert UInt<2> to u128 - this always succeeds for Iceberg-compliant
decimals
+ // since 38 digits requires ~127 bits and u128 has 128 bits
+ let unsigned: u128 = digits
+ .to_u128()
+ .expect("Iceberg decimals (max 38 digits) always fit in u128");
+
+ let signed = unsigned as i128;
+ if d.is_sign_negative() {
+ -signed
+ } else {
+ signed
+ }
+}
+
+/// Get the scale (number of digits after decimal point).
+///
+/// This is equivalent to rust_decimal's `decimal.scale()`.
+pub fn decimal_scale(d: &Decimal) -> u32 {
+ let frac = d.fractional_digits_count();
+ if frac < 0 { 0 } else { frac as u32 }
+}
+
+/// Rescale a decimal to the given scale, returning the rescaled value.
+///
+/// This is equivalent to rust_decimal's `decimal.rescale(scale)`.
+pub fn decimal_rescale(d: Decimal, scale: u32) -> Decimal {
+ d.rescale(scale as i16)
+}
+
+/// Convert big-endian signed bytes to i128.
+///
+/// This handles variable-length byte arrays (up to 16 bytes) with sign
extension.
+/// Returns None if the byte array is longer than 16 bytes.
+pub fn i128_from_be_bytes(bytes: &[u8]) -> Option<i128> {
+ if bytes.is_empty() {
+ return Some(0);
+ }
+ if bytes.len() > 16 {
+ return None; // Too large for i128
+ }
+
+ // Check sign bit (most significant bit of first byte)
+ let is_negative = bytes[0] & 0x80 != 0;
+
+ // Pad to 16 bytes with sign extension
+ let mut padded = if is_negative { [0xFF; 16] } else { [0; 16] };
+ let start = 16 - bytes.len();
+ padded[start..].copy_from_slice(bytes);
+
+ Some(i128::from_be_bytes(padded))
+}
+
+/// Convert i128 to big-endian signed bytes with minimum length.
+///
+/// This produces the shortest two's complement representation of the value.
+/// The result is suitable for Iceberg decimal binary serialization.
+pub fn i128_to_be_bytes_min(value: i128) -> Vec<u8> {
+ let bytes = value.to_be_bytes();
+
+ // Find the first significant byte
+ // For positive numbers, skip leading 0x00 bytes (but keep sign bit)
+ // For negative numbers, skip leading 0xFF bytes (but keep sign bit)
+ let is_negative = value < 0;
+ let skip_byte = if is_negative { 0xFF } else { 0x00 };
+
+ let mut start = 0;
+ while start < 15 && bytes[start] == skip_byte {
+ // Check if the next byte has the correct sign bit
+ let next_byte = bytes[start + 1];
+ let next_is_negative = (next_byte & 0x80) != 0;
+ if next_is_negative == is_negative {
+ start += 1;
+ } else {
+ break;
+ }
+ }
+
+ bytes[start..].to_vec()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_decimal_from_i128_with_scale() {
+ let d = decimal_from_i128_with_scale(12345, 2);
+ assert_eq!(d.to_string(), "123.45");
+
+ let d = decimal_from_i128_with_scale(-12345, 2);
+ assert_eq!(d.to_string(), "-123.45");
+
+ let d = decimal_from_i128_with_scale(0, 5);
+ assert_eq!(d.to_string(), "0.00000");
+ }
+
+ #[test]
+ fn test_decimal_new() {
+ let d = decimal_new(123, 2);
+ assert_eq!(d.to_string(), "1.23");
+
+ let d = decimal_new(-456, 3);
+ assert_eq!(d.to_string(), "-0.456");
+ }
+
+ #[test]
+ fn test_decimal_from_str_exact() {
+ let d = decimal_from_str_exact("123.45").unwrap();
+ assert_eq!(d.to_string(), "123.45");
+
+ let d = decimal_from_str_exact("-0.001").unwrap();
+ assert_eq!(d.to_string(), "-0.001");
+
+ let d =
decimal_from_str_exact("99999999999999999999999999999999999999").unwrap();
+ assert_eq!(d.to_string(), "99999999999999999999999999999999999999");
+ }
+
+ #[test]
+ fn test_decimal_mantissa() {
+ let d = decimal_from_i128_with_scale(12345, 2);
+ assert_eq!(decimal_mantissa(&d), 12345);
+
+ let d = decimal_from_i128_with_scale(-12345, 2);
+ assert_eq!(decimal_mantissa(&d), -12345);
+ }
+
+ #[test]
+ fn test_decimal_scale() {
+ let d = decimal_from_i128_with_scale(12345, 2);
+ assert_eq!(decimal_scale(&d), 2);
+
+ let d = decimal_from_i128_with_scale(12345, 0);
+ assert_eq!(decimal_scale(&d), 0);
+ }
+
+ #[test]
+ fn test_decimal_rescale() {
+ let d = decimal_from_str_exact("123.45").unwrap();
+ let rescaled = decimal_rescale(d, 4);
+ assert_eq!(decimal_scale(&rescaled), 4);
+ assert_eq!(decimal_mantissa(&rescaled), 1234500);
+ }
+
+ #[test]
+ fn test_38_digit_precision() {
+ // Test that we can handle 38-digit decimals (Iceberg spec requirement)
+ let max_38_digits = "99999999999999999999999999999999999999";
+ let d = decimal_from_str_exact(max_38_digits).unwrap();
+ assert_eq!(d.to_string(), max_38_digits);
+
+ let min_38_digits = "-99999999999999999999999999999999999999";
+ let d = decimal_from_str_exact(min_38_digits).unwrap();
+ assert_eq!(d.to_string(), min_38_digits);
+ }
+
+ #[test]
+ fn test_i128_from_be_bytes() {
+ // Empty bytes
+ assert_eq!(i128_from_be_bytes(&[]), Some(0));
+
+ // Positive values
+ assert_eq!(i128_from_be_bytes(&[0x01]), Some(1));
+ assert_eq!(i128_from_be_bytes(&[0x7F]), Some(127));
+ assert_eq!(i128_from_be_bytes(&[0x00, 0xFF]), Some(255));
+ assert_eq!(i128_from_be_bytes(&[0x04, 0xD2]), Some(1234));
+
+ // Negative values (sign extension)
+ assert_eq!(i128_from_be_bytes(&[0xFF]), Some(-1));
+ assert_eq!(i128_from_be_bytes(&[0x80]), Some(-128));
+ assert_eq!(i128_from_be_bytes(&[0xFB, 0x2E]), Some(-1234));
+
+ // Too large (> 16 bytes)
+ assert_eq!(i128_from_be_bytes(&[0; 17]), None);
+ }
+
+ #[test]
+ fn test_i128_to_be_bytes_min() {
+ // Positive values
+ assert_eq!(i128_to_be_bytes_min(0), vec![0x00]);
+ assert_eq!(i128_to_be_bytes_min(1), vec![0x01]);
+ assert_eq!(i128_to_be_bytes_min(127), vec![0x7F]);
+ assert_eq!(i128_to_be_bytes_min(128), vec![0x00, 0x80]);
+ assert_eq!(i128_to_be_bytes_min(255), vec![0x00, 0xFF]);
+ assert_eq!(i128_to_be_bytes_min(1234), vec![0x04, 0xD2]);
+
+ // Negative values
+ assert_eq!(i128_to_be_bytes_min(-1), vec![0xFF]);
+ assert_eq!(i128_to_be_bytes_min(-128), vec![0x80]);
+ assert_eq!(i128_to_be_bytes_min(-129), vec![0xFF, 0x7F]);
+ assert_eq!(i128_to_be_bytes_min(-1234), vec![0xFB, 0x2E]);
+
+ // Round trip test
+ for val in [
+ 0i128,
+ 1,
+ -1,
+ 127,
+ -128,
+ 255,
+ -256,
+ 12345,
+ -12345,
+ i128::MAX,
+ i128::MIN,
+ ] {
+ let bytes = i128_to_be_bytes_min(val);
+ assert_eq!(
+ i128_from_be_bytes(&bytes),
+ Some(val),
+ "Round trip failed for {val}"
+ );
+ }
+ }
+}
diff --git a/crates/iceberg/src/spec/values/literal.rs
b/crates/iceberg/src/spec/values/literal.rs
index d6e502e8f..e82fa197c 100644
--- a/crates/iceberg/src/spec/values/literal.rs
+++ b/crates/iceberg/src/spec/values/literal.rs
@@ -22,11 +22,13 @@ use std::str::FromStr;
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc};
use ordered_float::OrderedFloat;
-use rust_decimal::Decimal;
use serde_json::{Map as JsonMap, Number, Value as JsonValue};
use uuid::Uuid;
use super::Map;
+use super::decimal_utils::{
+ decimal_from_str_exact, decimal_mantissa, decimal_rescale,
try_decimal_from_i128_with_scale,
+};
use super::primitive::PrimitiveLiteral;
use super::struct_value::Struct;
use super::temporal::{date, time, timestamp, timestamptz};
@@ -396,23 +398,20 @@ impl Literal {
Self::Primitive(PrimitiveLiteral::Int128(decimal))
}
- /// Creates decimal literal from string. See [`Decimal::from_str_exact`].
+ /// Creates decimal literal from string.
///
/// Example:
///
/// ```rust
/// use iceberg::spec::Literal;
- /// use rust_decimal::Decimal;
/// let t1 = Literal::decimal(12345);
/// let t2 = Literal::decimal_from_str("123.45").unwrap();
///
/// assert_eq!(t1, t2);
/// ```
pub fn decimal_from_str<S: AsRef<str>>(s: S) -> Result<Self> {
- let decimal = Decimal::from_str_exact(s.as_ref()).map_err(|e| {
- Error::new(ErrorKind::DataInvalid, "Can't parse
decimal.").with_source(e)
- })?;
- Ok(Self::decimal(decimal.mantissa()))
+ let decimal = decimal_from_str_exact(s.as_ref())?;
+ Ok(Self::decimal(decimal_mantissa(&decimal)))
}
/// Attempts to convert the Literal to a PrimitiveLiteral
@@ -509,10 +508,10 @@ impl Literal {
},
JsonValue::String(s),
) => {
- let mut decimal = Decimal::from_str_exact(&s)?;
- decimal.rescale(*scale);
+ let decimal = decimal_from_str_exact(&s)?;
+ let rescaled = decimal_rescale(decimal, *scale);
Ok(Some(Literal::Primitive(PrimitiveLiteral::Int128(
- decimal.mantissa(),
+ decimal_mantissa(&rescaled),
))))
}
(_, JsonValue::Null) => Ok(None),
@@ -672,7 +671,7 @@ impl Literal {
precision: _precision,
scale,
}) => {
- let decimal = Decimal::try_from_i128_with_scale(val,
*scale)?;
+ let decimal = try_decimal_from_i128_with_scale(val,
*scale)?;
Ok(JsonValue::String(decimal.to_string()))
}
_ => Err(Error::new(
diff --git a/crates/iceberg/src/spec/values/mod.rs
b/crates/iceberg/src/spec/values/mod.rs
index 2bc967191..65374c74b 100644
--- a/crates/iceberg/src/spec/values/mod.rs
+++ b/crates/iceberg/src/spec/values/mod.rs
@@ -18,6 +18,7 @@
//! This module contains Iceberg value types
pub(crate) mod datum;
+pub(crate) mod decimal_utils;
mod literal;
mod map;
mod primitive;
@@ -30,6 +31,7 @@ mod tests;
// Re-export all public types
pub use datum::Datum;
+pub use decimal_utils::Decimal;
pub use literal::Literal;
pub use map::Map;
pub use primitive::PrimitiveLiteral;
diff --git a/crates/iceberg/src/spec/values/tests.rs
b/crates/iceberg/src/spec/values/tests.rs
index bb10701d8..41238ed89 100644
--- a/crates/iceberg/src/spec/values/tests.rs
+++ b/crates/iceberg/src/spec/values/tests.rs
@@ -20,11 +20,11 @@
use apache_avro::to_value;
use apache_avro::types::Value;
use ordered_float::OrderedFloat;
-use rust_decimal::Decimal;
use serde_bytes::ByteBuf;
use serde_json::Value as JsonValue;
use uuid::Uuid;
+use super::decimal_utils::{decimal_from_i128_with_scale, decimal_new};
use crate::ErrorKind;
use crate::avro::schema_to_avro_schema;
use crate::spec::Schema;
@@ -395,11 +395,8 @@ fn avro_bytes_decimal() {
for (input_bytes, decimal_num, expect_scale, expect_precision) in cases {
check_avro_bytes_serde(
input_bytes,
- Datum::decimal_with_precision(
- Decimal::new(decimal_num, expect_scale),
- expect_precision,
- )
- .unwrap(),
+ Datum::decimal_with_precision(decimal_new(decimal_num,
expect_scale), expect_precision)
+ .unwrap(),
&PrimitiveType::Decimal {
precision: expect_precision,
scale: expect_scale,
@@ -414,10 +411,8 @@ fn avro_bytes_decimal_expect_error() {
let cases = vec![(1234, 2, 1)];
for (decimal_num, expect_scale, expect_precision) in cases {
- let result = Datum::decimal_with_precision(
- Decimal::new(decimal_num, expect_scale),
- expect_precision,
- );
+ let result =
+ Datum::decimal_with_precision(decimal_new(decimal_num,
expect_scale), expect_precision);
assert!(result.is_err(), "expect error but got {result:?}");
assert_eq!(
result.unwrap_err().kind(),
@@ -1053,7 +1048,7 @@ fn test_datum_ser_deser() {
test_fn(datum);
let datum =
Datum::uuid(Uuid::parse_str("f79c3e09-677c-4bbd-a479-3f349cb785e7").unwrap());
test_fn(datum);
- let datum = Datum::decimal(1420).unwrap();
+ let datum = Datum::decimal(decimal_new(1420, 0)).unwrap();
test_fn(datum);
let datum = Datum::binary(vec![1, 2, 3, 4, 5]);
test_fn(datum);
@@ -1140,7 +1135,7 @@ fn test_datum_long_convert_to_timestamptz() {
#[test]
fn test_datum_decimal_convert_to_long() {
- let datum = Datum::decimal(12345).unwrap();
+ let datum = Datum::decimal(decimal_new(12345, 0)).unwrap();
let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap();
@@ -1151,7 +1146,7 @@ fn test_datum_decimal_convert_to_long() {
#[test]
fn test_datum_decimal_convert_to_long_above_max() {
- let datum = Datum::decimal(LONG_MAX as i128 + 1).unwrap();
+ let datum = Datum::decimal(decimal_from_i128_with_scale(LONG_MAX as i128 +
1, 0)).unwrap();
let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap();
@@ -1162,7 +1157,7 @@ fn test_datum_decimal_convert_to_long_above_max() {
#[test]
fn test_datum_decimal_convert_to_long_below_min() {
- let datum = Datum::decimal(LONG_MIN as i128 - 1).unwrap();
+ let datum = Datum::decimal(decimal_from_i128_with_scale(LONG_MIN as i128 -
1, 0)).unwrap();
let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap();
diff --git a/crates/iceberg/src/transform/bucket.rs
b/crates/iceberg/src/transform/bucket.rs
index e6786a70c..52fb72f1c 100644
--- a/crates/iceberg/src/transform/bucket.rs
+++ b/crates/iceberg/src/transform/bucket.rs
@@ -294,6 +294,7 @@ mod test {
TimestampNs, Timestamptz, TimestamptzNs, Uuid,
};
use crate::spec::Type::{Primitive, Struct};
+ use crate::spec::decimal_utils::decimal_new;
use crate::spec::{Datum, NestedField, PrimitiveType, StructType,
Transform, Type};
use crate::transform::TransformFunction;
use crate::transform::test::{TestProjectionFixture, TestTransformFixture};
@@ -848,7 +849,7 @@ mod test {
let bucket = Bucket::new(10);
assert_eq!(
bucket
- .transform_literal(&Datum::decimal(1420).unwrap())
+ .transform_literal(&Datum::decimal(decimal_new(1420,
0)).unwrap())
.unwrap()
.unwrap(),
Datum::int(9)
diff --git a/crates/iceberg/src/transform/truncate.rs
b/crates/iceberg/src/transform/truncate.rs
index 84ef7c0da..4fac48f7d 100644
--- a/crates/iceberg/src/transform/truncate.rs
+++ b/crates/iceberg/src/transform/truncate.rs
@@ -22,6 +22,7 @@ use arrow_schema::DataType;
use super::TransformFunction;
use crate::Error;
+use crate::spec::decimal_utils::decimal_from_i128_with_scale;
use crate::spec::{Datum, PrimitiveLiteral};
#[derive(Debug)]
@@ -163,7 +164,10 @@ impl TransformFunction for Truncate {
})),
PrimitiveLiteral::Int128(v) => Ok(Some({
let width = self.width as i128;
- Datum::decimal(Self::truncate_decimal_i128(*v, width))?
+ Datum::decimal(decimal_from_i128_with_scale(
+ Self::truncate_decimal_i128(*v, width),
+ 0,
+ ))?
})),
PrimitiveLiteral::String(v) => Ok(Some({
let len = self.width as usize;
@@ -195,6 +199,7 @@ mod test {
TimestampNs, Timestamptz, TimestamptzNs, Uuid,
};
use crate::spec::Type::{Primitive, Struct};
+ use crate::spec::decimal_utils::decimal_new;
use crate::spec::{Datum, NestedField, PrimitiveType, StructType,
Transform, Type};
use crate::transform::TransformFunction;
use crate::transform::test::{TestProjectionFixture, TestTransformFixture};
@@ -831,12 +836,12 @@ mod test {
#[test]
fn test_decimal_literal() {
- let input = Datum::decimal(1065).unwrap();
+ let input = Datum::decimal(decimal_new(1065, 0)).unwrap();
let res = super::Truncate::new(50)
.transform_literal(&input)
.unwrap()
.unwrap();
- assert_eq!(res, Datum::decimal(1050).unwrap(),);
+ assert_eq!(res, Datum::decimal(decimal_new(1050, 0)).unwrap(),);
}
#[test]
diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs
b/crates/iceberg/src/writer/file_writer/parquet_writer.rs
index 8fe40df71..a75c74b3b 100644
--- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs
+++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs
@@ -622,13 +622,13 @@ mod tests {
use arrow_select::concat::concat_batches;
use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
use parquet::file::statistics::ValueStatistics;
- use rust_decimal::Decimal;
use tempfile::TempDir;
use uuid::Uuid;
use super::*;
use crate::arrow::schema_to_arrow_schema;
use crate::io::FileIOBuilder;
+ use crate::spec::decimal_utils::{decimal_mantissa, decimal_new,
decimal_scale};
use crate::spec::{PrimitiveLiteral, Struct, *};
use crate::writer::file_writer::location_generator::{
DefaultFileNameGenerator, DefaultLocationGenerator, FileNameGenerator,
LocationGenerator,
@@ -1378,12 +1378,12 @@ mod tests {
.unwrap();
assert_eq!(
data_file.upper_bounds().get(&0),
- Some(Datum::decimal_with_precision(Decimal::new(22000000000_i64,
10), 28).unwrap())
+ Some(Datum::decimal_with_precision(decimal_new(22000000000_i64,
10), 28).unwrap())
.as_ref()
);
assert_eq!(
data_file.lower_bounds().get(&0),
- Some(Datum::decimal_with_precision(Decimal::new(11000000000_i64,
10), 28).unwrap())
+ Some(Datum::decimal_with_precision(decimal_new(11000000000_i64,
10), 28).unwrap())
.as_ref()
);
@@ -1430,19 +1430,22 @@ mod tests {
.unwrap();
assert_eq!(
data_file.upper_bounds().get(&0),
- Some(Datum::decimal_with_precision(Decimal::new(-11000000000_i64,
10), 28).unwrap())
+ Some(Datum::decimal_with_precision(decimal_new(-11000000000_i64,
10), 28).unwrap())
.as_ref()
);
assert_eq!(
data_file.lower_bounds().get(&0),
- Some(Datum::decimal_with_precision(Decimal::new(-22000000000_i64,
10), 28).unwrap())
+ Some(Datum::decimal_with_precision(decimal_new(-22000000000_i64,
10), 28).unwrap())
.as_ref()
);
- // test max and min of rust_decimal
- let decimal_max = Decimal::MAX;
- let decimal_min = Decimal::MIN;
- assert_eq!(decimal_max.scale(), decimal_min.scale());
+ // test 38-digit precision decimal values (Iceberg spec max)
+ // Note: fastnum D128::MAX/MIN have impractical exponents, so we use
meaningful values
+ use crate::spec::decimal_utils::decimal_from_str_exact;
+ let decimal_max =
decimal_from_str_exact("99999999999999999999999999999999999999").unwrap();
+ let decimal_min =
+
decimal_from_str_exact("-99999999999999999999999999999999999999").unwrap();
+ assert_eq!(decimal_scale(&decimal_max), decimal_scale(&decimal_min));
let schema = Arc::new(
Schema::builder()
.with_fields(vec![
@@ -1451,7 +1454,7 @@ mod tests {
"decimal",
Type::Primitive(PrimitiveType::Decimal {
precision: 38,
- scale: decimal_max.scale(),
+ scale: decimal_scale(&decimal_max),
}),
)
.into(),
@@ -1468,8 +1471,8 @@ mod tests {
.await?;
let col0 = Arc::new(
Decimal128Array::from(vec![
- Some(decimal_max.mantissa()),
- Some(decimal_min.mantissa()),
+ Some(decimal_mantissa(&decimal_max)),
+ Some(decimal_mantissa(&decimal_min)),
])
.with_data_type(DataType::Decimal128(38, 0)),
) as ArrayRef;
