This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 69f0291910 chore: bump substrait version to `0.60.0` to use substrait
spec v0.75.0 (#17866)
69f0291910 is described below
commit 69f02919107109d3d707aeb3af9bf16e20f0b0a0
Author: Ben Bellick <[email protected]>
AuthorDate: Fri Oct 31 03:58:33 2025 -0400
chore: bump substrait version to `0.60.0` to use substrait spec v0.75.0
(#17866)
## Which issue does this PR close?
- Closes #17865.
## What changes are included in this PR?
Bump the `substrait` version to `v0.75.0` by bumping `substrait-rs` to
`v0.60.0`.
This PR was originally dependent on [this
PR](https://github.com/apache/datafusion/pull/17888) to update the
versions of some common dependencies, but that PR is now merged in.
## Are these changes tested?
There are no tests here, but there is no change to any logic within
datafusion. It is simply a bump in a dependency. Technically the public
API does change, but as noted in the issue description, there is no
change to internal logic because uri / urn from substrait plans are not
used.
## Are there any user-facing changes?
Yes. Previously substrait plans of spec version `v0.74.0` were accepted,
and now `v0.75.0` is accepted. However, this is a backwards compatible
change. The only difference is the inclusion of additional urn-based
fields in substrait plans. In a later PR, the old uri-based fields will
be dropped, which *will* be a breaking change.
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
Cargo.lock | 16 ++++++++--------
datafusion/substrait/Cargo.toml | 2 +-
datafusion/substrait/src/extensions.rs | 6 ++++++
.../substrait/src/logical_plan/producer/expr/mod.rs | 4 ++++
datafusion/substrait/src/logical_plan/producer/plan.rs | 5 +++++
5 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 120dc29db2..1c516277c3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6141,9 +6141,9 @@ dependencies = [
[[package]]
name = "substrait"
-version = "0.59.1"
+version = "0.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "540683f325ab9ab1a2008bc24588f3e76f63b6a3f52bc47e121122376a063639"
+checksum = "21f1cb6d0bcd097a39fc25f7236236be29881fe122e282e4173d6d007a929927"
dependencies = [
"heck 0.5.0",
"pbjson",
@@ -6736,9 +6736,9 @@ checksum =
"f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71"
[[package]]
name = "typify"
-version = "0.4.3"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7144144e97e987c94758a3017c920a027feac0799df325d6df4fc8f08d02068e"
+checksum = "e6d5bcc6f62eb1fa8aa4098f39b29f93dcb914e17158b76c50360911257aa629"
dependencies = [
"typify-impl",
"typify-macro",
@@ -6746,9 +6746,9 @@ dependencies = [
[[package]]
name = "typify-impl"
-version = "0.4.3"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "062879d46aa4c9dfe0d33b035bbaf512da192131645d05deacb7033ec8581a09"
+checksum = "a1eb359f7ffa4f9ebe947fa11a1b2da054564502968db5f317b7e37693cb2240"
dependencies = [
"heck 0.5.0",
"log",
@@ -6766,9 +6766,9 @@ dependencies = [
[[package]]
name = "typify-macro"
-version = "0.4.3"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9708a3ceb6660ba3f8d2b8f0567e7d4b8b198e2b94d093b8a6077a751425de9e"
+checksum = "911c32f3c8514b048c1b228361bebb5e6d73aeec01696e8cc0e82e2ffef8ab7a"
dependencies = [
"proc-macro2",
"quote",
diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.toml
index 8417bd5685..bff9a07a13 100644
--- a/datafusion/substrait/Cargo.toml
+++ b/datafusion/substrait/Cargo.toml
@@ -40,7 +40,7 @@ itertools = { workspace = true }
object_store = { workspace = true }
pbjson-types = { workspace = true }
prost = { workspace = true }
-substrait = { version = "0.59", features = ["serde"] }
+substrait = { version = "0.62", features = ["serde"] }
url = { workspace = true }
tokio = { workspace = true, features = ["fs"] }
uuid = { version = "1.17.0", features = ["v4"] }
diff --git a/datafusion/substrait/src/extensions.rs
b/datafusion/substrait/src/extensions.rs
index f9a2e0fb82..0792928982 100644
--- a/datafusion/substrait/src/extensions.rs
+++ b/datafusion/substrait/src/extensions.rs
@@ -113,11 +113,15 @@ impl TryFrom<&Vec<SimpleExtensionDeclaration>> for
Extensions {
}
impl From<Extensions> for Vec<SimpleExtensionDeclaration> {
+ // Silence deprecation warnings for `extension_uri_reference` during the
uri -> urn migration
+ // See: https://github.com/substrait-io/substrait/issues/856
+ #[allow(deprecated)]
fn from(val: Extensions) -> Vec<SimpleExtensionDeclaration> {
let mut extensions = vec![];
for (f_anchor, f_name) in val.functions {
let function_extension = ExtensionFunction {
extension_uri_reference: u32::MAX,
+ extension_urn_reference: u32::MAX,
function_anchor: f_anchor,
name: f_name,
};
@@ -130,6 +134,7 @@ impl From<Extensions> for Vec<SimpleExtensionDeclaration> {
for (t_anchor, t_name) in val.types {
let type_extension = ExtensionType {
extension_uri_reference: u32::MAX, //
https://github.com/apache/datafusion/issues/11545
+ extension_urn_reference: u32::MAX, //
https://github.com/apache/datafusion/issues/11545
type_anchor: t_anchor,
name: t_name,
};
@@ -142,6 +147,7 @@ impl From<Extensions> for Vec<SimpleExtensionDeclaration> {
for (tv_anchor, tv_name) in val.type_variations {
let type_variation_extension = ExtensionTypeVariation {
extension_uri_reference: u32::MAX, // We don't register proper
extension URIs yet
+ extension_urn_reference: u32::MAX, // We don't register proper
extension URIs yet
type_variation_anchor: tv_anchor,
name: tv_name,
};
diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs
b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs
index d37694ccea..f4e43fd586 100644
--- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs
+++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs
@@ -60,6 +60,9 @@ use substrait::version;
///
/// Substrait also requires the input schema of the expressions to be included
in the
/// message. The field names of the input schema will be serialized.
+// Silence deprecation warnings for `extension_uris` during the uri -> urn
migration
+// See: https://github.com/substrait-io/substrait/issues/856
+#[allow(deprecated)]
pub fn to_substrait_extended_expr(
exprs: &[(&Expr, &Field)],
schema: &DFSchemaRef,
@@ -85,6 +88,7 @@ pub fn to_substrait_extended_expr(
advanced_extensions: None,
expected_type_urls: vec![],
extension_uris: vec![],
+ extension_urns: vec![],
extensions: extensions.into(),
version: Some(version::version_with_producer("datafusion")),
referred_expr: substrait_exprs,
diff --git a/datafusion/substrait/src/logical_plan/producer/plan.rs
b/datafusion/substrait/src/logical_plan/producer/plan.rs
index 28f6acd089..ad8f45ec36 100644
--- a/datafusion/substrait/src/logical_plan/producer/plan.rs
+++ b/datafusion/substrait/src/logical_plan/producer/plan.rs
@@ -24,6 +24,9 @@ use substrait::proto::{plan_rel, Plan, PlanRel, Rel, RelRoot};
use substrait::version;
/// Convert DataFusion LogicalPlan to Substrait Plan
+// Silence deprecation warnings for `extension_uris` during the uri -> urn
migration
+// See: https://github.com/substrait-io/substrait/issues/856
+#[allow(deprecated)]
pub fn to_substrait_plan(
plan: &LogicalPlan,
state: &SessionState,
@@ -45,11 +48,13 @@ pub fn to_substrait_plan(
Ok(Box::new(Plan {
version: Some(version::version_with_producer("datafusion")),
extension_uris: vec![],
+ extension_urns: vec![],
extensions: extensions.into(),
relations: plan_rels,
advanced_extensions: None,
expected_type_urls: vec![],
parameter_bindings: vec![],
+ type_aliases: vec![],
}))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
