This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 1dd887cdff Support Decimal256 column in create external table (#7866)
1dd887cdff is described below
commit 1dd887cdff518ede1d1de457f4b20c22a9c7228f
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Fri Oct 20 00:19:49 2023 -0700
Support Decimal256 column in create external table (#7866)
* Support Decimal256 column in create external table
* Update test
* More
---
datafusion/sql/src/utils.rs | 13 ++++--
datafusion/sql/tests/sql_integration.rs | 16 +++++--
datafusion/sqllogictest/test_files/decimal.slt | 64 ++++++++++++++++++++++++++
3 files changed, 86 insertions(+), 7 deletions(-)
diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs
index 28eaf241fa..616a2fc749 100644
--- a/datafusion/sql/src/utils.rs
+++ b/datafusion/sql/src/utils.rs
@@ -17,7 +17,9 @@
//! SQL Utility Functions
-use arrow_schema::{DataType, DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE};
+use arrow_schema::{
+ DataType, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION,
DECIMAL_DEFAULT_SCALE,
+};
use datafusion_common::tree_node::{Transformed, TreeNode};
use sqlparser::ast::Ident;
@@ -221,14 +223,17 @@ pub(crate) fn make_decimal_type(
(None, None) => (DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE),
};
- // Arrow decimal is i128 meaning 38 maximum decimal digits
if precision == 0
- || precision > DECIMAL128_MAX_PRECISION
+ || precision > DECIMAL256_MAX_PRECISION
|| scale.unsigned_abs() > precision
{
plan_err!(
- "Decimal(precision = {precision}, scale = {scale}) should satisfy
`0 < precision <= 38`, and `scale <= precision`."
+ "Decimal(precision = {precision}, scale = {scale}) should satisfy
`0 < precision <= 76`, and `scale <= precision`."
)
+ } else if precision > DECIMAL128_MAX_PRECISION
+ && precision <= DECIMAL256_MAX_PRECISION
+ {
+ Ok(DataType::Decimal256(precision, scale))
} else {
Ok(DataType::Decimal128(precision, scale))
}
diff --git a/datafusion/sql/tests/sql_integration.rs
b/datafusion/sql/tests/sql_integration.rs
index 653d2ec52d..2446ee0a58 100644
--- a/datafusion/sql/tests/sql_integration.rs
+++ b/datafusion/sql/tests/sql_integration.rs
@@ -201,7 +201,7 @@ fn cast_to_invalid_decimal_type_precision_0() {
let sql = "SELECT CAST(10 AS DECIMAL(0))";
let err = logical_plan(sql).expect_err("query should have failed");
assert_eq!(
- "Error during planning: Decimal(precision = 0, scale = 0) should
satisfy `0 < precision <= 38`, and `scale <= precision`.",
+ "Error during planning: Decimal(precision = 0, scale = 0) should
satisfy `0 < precision <= 76`, and `scale <= precision`.",
err.strip_backtrace()
);
}
@@ -212,9 +212,19 @@ fn cast_to_invalid_decimal_type_precision_gt_38() {
// precision > 38
{
let sql = "SELECT CAST(10 AS DECIMAL(39))";
+ let plan = "Projection: CAST(Int64(10) AS Decimal256(39, 0))\n
EmptyRelation";
+ quick_test(sql, plan);
+ }
+}
+
+#[test]
+fn cast_to_invalid_decimal_type_precision_gt_76() {
+ // precision > 76
+ {
+ let sql = "SELECT CAST(10 AS DECIMAL(79))";
let err = logical_plan(sql).expect_err("query should have failed");
assert_eq!(
- "Error during planning: Decimal(precision = 39, scale = 0) should
satisfy `0 < precision <= 38`, and `scale <= precision`.",
+ "Error during planning: Decimal(precision = 79, scale = 0) should
satisfy `0 < precision <= 76`, and `scale <= precision`.",
err.strip_backtrace()
);
}
@@ -227,7 +237,7 @@ fn cast_to_invalid_decimal_type_precision_lt_scale() {
let sql = "SELECT CAST(10 AS DECIMAL(5, 10))";
let err = logical_plan(sql).expect_err("query should have failed");
assert_eq!(
- "Error during planning: Decimal(precision = 5, scale = 10) should
satisfy `0 < precision <= 38`, and `scale <= precision`.",
+ "Error during planning: Decimal(precision = 5, scale = 10) should
satisfy `0 < precision <= 76`, and `scale <= precision`.",
err.strip_backtrace()
);
}
diff --git a/datafusion/sqllogictest/test_files/decimal.slt
b/datafusion/sqllogictest/test_files/decimal.slt
index 570116b7a2..f968ffb90a 100644
--- a/datafusion/sqllogictest/test_files/decimal.slt
+++ b/datafusion/sqllogictest/test_files/decimal.slt
@@ -629,3 +629,67 @@ select AVG(column1) from t;
statement ok
drop table t;
+
+statement ok
+CREATE EXTERNAL TABLE decimal256_simple (
+c1 DECIMAL(50,6) NOT NULL,
+c2 DOUBLE NOT NULL,
+c3 BIGINT NOT NULL,
+c4 BOOLEAN NOT NULL,
+c5 DECIMAL(52,7) NOT NULL
+)
+STORED AS CSV
+WITH HEADER ROW
+LOCATION '../core/tests/data/decimal_data.csv';
+
+query TT
+select arrow_typeof(c1), arrow_typeof(c5) from decimal256_simple limit 1;
+----
+Decimal256(50, 6) Decimal256(52, 7)
+
+query R rowsort
+SELECT c1 from decimal256_simple;
+----
+0.00001
+0.00002
+0.00002
+0.00003
+0.00003
+0.00003
+0.00004
+0.00004
+0.00004
+0.00004
+0.00005
+0.00005
+0.00005
+0.00005
+0.00005
+
+query R rowsort
+select c1 from decimal256_simple where c1 > 0.000030;
+----
+0.00004
+0.00004
+0.00004
+0.00004
+0.00005
+0.00005
+0.00005
+0.00005
+0.00005
+
+query RRIBR rowsort
+select * from decimal256_simple where c1 > c5;
+----
+0.00002 0.000000000002 3 false 0.000019
+0.00003 0.000000000003 5 true 0.000011
+0.00005 0.000000000005 8 false 0.000033
+
+query TR
+select arrow_typeof(avg(c1)), avg(c1) from decimal256_simple;
+----
+Decimal256(54, 10) 0.0000366666
+
+statement ok
+drop table decimal256_simple;
