This is an automated email from the ASF dual-hosted git repository.
loneylee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 87657f63a4 [GLUTEN-7262][CH] Fix cache file commond run normal with
config disabled (#7263)
87657f63a4 is described below
commit 87657f63a44075ee4d3a0dcf5346ab5230d509ab
Author: Shuai li <[email protected]>
AuthorDate: Thu Sep 19 17:30:35 2024 +0800
[GLUTEN-7262][CH] Fix cache file commond run normal with config disabled
(#7263)
---
.../sql/execution/commands/GlutenCacheFilesCommand.scala | 11 +++++++++++
.../execution/GlutenClickHouseMergeTreeCacheDataSuite.scala | 13 +++++++++++++
2 files changed, 24 insertions(+)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/commands/GlutenCacheFilesCommand.scala
b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/commands/GlutenCacheFilesCommand.scala
index 0a08df7ceb..b2640df789 100644
---
a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/commands/GlutenCacheFilesCommand.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/commands/GlutenCacheFilesCommand.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.execution.commands
+import org.apache.gluten.GlutenConfig
+import org.apache.gluten.backendsapi.clickhouse.CHBackend
import org.apache.gluten.substrait.rel.LocalFilesBuilder
import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
@@ -50,6 +52,15 @@ case class GlutenCacheFilesCommand(
AttributeReference("reason", StringType, nullable = false)())
override def run(session: SparkSession): Seq[Row] = {
+ if (
+ !session.sparkContext.getConf.getBoolean(
+ s"${GlutenConfig.GLUTEN_CONFIG_PREFIX}${CHBackend.BACKEND_NAME}" +
+ s".runtime_config.gluten_cache.local.enabled",
+ defaultValue = false)
+ ) {
+ return Seq(Row(false, "Config `gluten_cache.local.enabled` is
disabled."))
+ }
+
val targetFile = new Path(filePath)
val hadoopConf: Configuration = session.sparkContext.hadoopConfiguration
val fs = targetFile.getFileSystem(hadoopConf)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeCacheDataSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeCacheDataSuite.scala
index 88bb00face..73dc5879ea 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeCacheDataSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeCacheDataSuite.scala
@@ -591,5 +591,18 @@ class GlutenClickHouseMergeTreeCacheDataSuite
})
spark.sql("drop table lineitem_mergetree_hdfs purge")
}
+
+ test("test disable cache files return") {
+ withSQLConf(s"$CH_CONFIG_PREFIX.gluten_cache.local.enabled" -> "false") {
+ runSql(
+ s"CACHE FILES select * from '$HDFS_URL_ENDPOINT/tpch-data/lineitem'",
+ noFallBack = false) {
+ df =>
+ val res = df.collect()
+ assert(res.length == 1)
+ assert(!res.apply(0).getBoolean(0))
+ }
+ }
+ }
}
// scalastyle:off line.size.limit
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
