This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/41 in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
commit c76520caf482eacadf2b6a0fb59a22937b7e3633 Author: Chang chen <[email protected]> AuthorDate: Tue Dec 16 13:44:27 2025 +0800 [Fix] Adapt to DataSourceV2Relation interface change see https://github.com/apache/spark/pull/52599 --- .../gluten/extension/ArrowConvertorRule.scala | 29 ++++++----------- .../datasources/v2/DataSourceV2RelationShim.scala | 37 ++++++++++++++++++++++ .../datasources/v2/DataSourceV2RelationShim.scala | 37 ++++++++++++++++++++++ .../datasources/v2/DataSourceV2RelationShim.scala | 37 ++++++++++++++++++++++ .../datasources/v2/DataSourceV2RelationShim.scala | 37 ++++++++++++++++++++++ .../datasources/v2/DataSourceV2RelationShim.scala | 37 ++++++++++++++++++++++ .../datasources/v2/DataSourceV2RelationShim.scala | 37 ++++++++++++++++++++++ 7 files changed, 231 insertions(+), 20 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowConvertorRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowConvertorRule.scala index 25371be8d1..f5e0b8526a 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowConvertorRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowConvertorRule.scala @@ -29,8 +29,7 @@ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.PermissiveMode import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat -import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation -import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2RelationShim.CSVTableExtractor import org.apache.spark.sql.types.StructType import org.apache.spark.sql.utils.SparkArrowUtil @@ -56,25 +55,15 @@ case class ArrowConvertorRule(session: SparkSession) extends Rule[LogicalPlan] { l.copy(relation = r.copy(fileFormat = new ArrowCSVFileFormat(csvOptions))(session)) case _ => l } - case d @ DataSourceV2Relation( - t @ CSVTable( - name, - sparkSession, - options, - paths, - userSpecifiedSchema, - fallbackFileFormat), - _, - _, - _, - _) if validate(session, t.dataSchema, options.asCaseSensitiveMap().toMap) => + case CSVTableExtractor(d, t) + if validate(session, t.dataSchema, t.options.asCaseSensitiveMap().toMap) => d.copy(table = ArrowCSVTable( - "arrow" + name, - sparkSession, - options, - paths, - userSpecifiedSchema, - fallbackFileFormat)) + "arrow" + t.name, + t.sparkSession, + t.options, + t.paths, + t.userSpecifiedSchema, + t.fallbackFileFormat)) case r => r } diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala new file mode 100644 index 0000000000..a465f6cd53 --- /dev/null +++ b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable + +/** + * Shim layer for DataSourceV2Relation to maintain compatibility across different Spark versions. + */ +object DataSourceV2RelationShim { + + /** @since Spark 4.1 */ + object CSVTableExtractor { + def unapply(relation: DataSourceV2Relation): Option[(DataSourceV2Relation, CSVTable)] = { + relation match { + case d @ DataSourceV2Relation(t: CSVTable, _, _, _, _) => + Some((d, t)) + case _ => None + } + } + } + +} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala new file mode 100644 index 0000000000..a465f6cd53 --- /dev/null +++ b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable + +/** + * Shim layer for DataSourceV2Relation to maintain compatibility across different Spark versions. + */ +object DataSourceV2RelationShim { + + /** @since Spark 4.1 */ + object CSVTableExtractor { + def unapply(relation: DataSourceV2Relation): Option[(DataSourceV2Relation, CSVTable)] = { + relation match { + case d @ DataSourceV2Relation(t: CSVTable, _, _, _, _) => + Some((d, t)) + case _ => None + } + } + } + +} diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala new file mode 100644 index 0000000000..a465f6cd53 --- /dev/null +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable + +/** + * Shim layer for DataSourceV2Relation to maintain compatibility across different Spark versions. + */ +object DataSourceV2RelationShim { + + /** @since Spark 4.1 */ + object CSVTableExtractor { + def unapply(relation: DataSourceV2Relation): Option[(DataSourceV2Relation, CSVTable)] = { + relation match { + case d @ DataSourceV2Relation(t: CSVTable, _, _, _, _) => + Some((d, t)) + case _ => None + } + } + } + +} diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala new file mode 100644 index 0000000000..a465f6cd53 --- /dev/null +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable + +/** + * Shim layer for DataSourceV2Relation to maintain compatibility across different Spark versions. + */ +object DataSourceV2RelationShim { + + /** @since Spark 4.1 */ + object CSVTableExtractor { + def unapply(relation: DataSourceV2Relation): Option[(DataSourceV2Relation, CSVTable)] = { + relation match { + case d @ DataSourceV2Relation(t: CSVTable, _, _, _, _) => + Some((d, t)) + case _ => None + } + } + } + +} diff --git a/shims/spark40/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala b/shims/spark40/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala new file mode 100644 index 0000000000..a465f6cd53 --- /dev/null +++ b/shims/spark40/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable + +/** + * Shim layer for DataSourceV2Relation to maintain compatibility across different Spark versions. + */ +object DataSourceV2RelationShim { + + /** @since Spark 4.1 */ + object CSVTableExtractor { + def unapply(relation: DataSourceV2Relation): Option[(DataSourceV2Relation, CSVTable)] = { + relation match { + case d @ DataSourceV2Relation(t: CSVTable, _, _, _, _) => + Some((d, t)) + case _ => None + } + } + } + +} diff --git a/shims/spark41/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala b/shims/spark41/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala new file mode 100644 index 0000000000..cbbe42fbf6 --- /dev/null +++ b/shims/spark41/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationShim.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable + +/** + * Shim layer for DataSourceV2Relation to maintain compatibility across different Spark versions. + */ +object DataSourceV2RelationShim { + + /** @since Spark 4.1 */ + object CSVTableExtractor { + def unapply(relation: DataSourceV2Relation): Option[(DataSourceV2Relation, CSVTable)] = { + relation match { + case d @ DataSourceV2Relation(t: CSVTable, _, _, _, _, _) => + Some((d, t)) + case _ => None + } + } + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
