comphead commented on PR #2801:
URL:
https://github.com/apache/datafusion-comet/pull/2801#issuecomment-3554538379
@andygrove it doesn't seem to work.
```
withSQLConf(
CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
"spark.comet.cast.allowIncompatible" -> "true",
"spark.sql.adaptive.enabled" -> "false",
"spark.comet.explain.native.enabled" -> "true",
"spark.comet.enabled" -> "true",
// "spark.comet.exec.replaceSortMergeJoin" -> "true",
"spark.comet.exec.shuffle.enableFastEncoding" -> "true",
"spark.comet.exec.shuffle.enabled" -> "true",
// "spark.comet.exec.shuffle.fallbackToColumnar" -> "true",
"spark.comet.explainFallback.enabled" -> "true",
CometConf.COMET_NATIVE_SCAN_IMPL.key -> "native_iceberg_compat",
"spark.shuffle.manager" ->
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
"spark.comet.logFallbackReasons.enabled" -> "true")
```
```
SQLConf.get
```
"spark.driver.port" -> "64746"
"spark.comet.exec.onHeap.enabled" -> "true"
"spark.app.startTime" -> "1763584407916"
"spark.memory.offHeap.size" -> "2g"
"spark.comet.scan.enabled" -> "true"
"spark.sql.adaptive.autoBroadcastJoinThreshold" -> "1g"
"spark.comet.exec.sortMergeJoinWithJoinFilter.enabled" -> "true"
"spark.shuffle.manager" ->
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
"spark.sql.shuffle.partitions" -> "10"
"spark.memory.offHeap.enabled" -> "true"
"spark.comet.explain.native.enabled" -> "true"
"spark.comet.memoryOverhead" -> "2g"
"spark.sql.adaptive.enabled" -> "false"
"spark.comet.parquet.respectFilterPushdown" -> "true"
"spark.comet.scan.impl" -> "native_iceberg_compat"
"spark.comet.explainFallback.enabled" -> "true"
"spark.hadoop.fs.file.impl" -> "org.apache.spark.DebugFilesystem"
"spark.comet.enabled" -> "true"
"spark.master" -> "local[5]"
"spark.comet.scan.allowIncompatible" -> "true"
"spark.executor.id" -> "driver"
"spark.comet.exec.enabled" -> "true"
"spark.app.name" -> "91e3ac4e-5f2d-4701-aa53-6d9390128af7"
"spark.comet.logFallbackReasons.enabled" -> "true"
"spark.comet.sparkToColumnar.enabled" -> "true"
"spark.comet.expression.SortOrder.allowIncompatible" -> "true"
"spark.comet.use.decimal128" -> "true"
"spark.comet.exec.shuffle.enableFastEncoding" -> "true"
"spark.app.id" -> "local-1763584408274"
"spark.sql.ansi.enabled" -> "false"
"spark.sql.autoBroadcastJoinThreshold" -> "1g"
"__fetch_continuous_blocks_in_batch_enabled" -> "true"
"spark.comet.cast.allowIncompatible" -> "true"
"spark.executor.extraJavaOptions" -> "-Djava.net.preferIPv6Addresses=false
-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
--add-opens=java.base/sun.security.action=ALL-UNNAMED
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
-Djdk.reflect.useDirectMethodHandle=false"
"spark.ui.enabled" -> "false"
"spark.driver.extraJavaOptions" -> "-Djava.net.preferIPv6Addresses=false
-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
--add-opens=java.base/sun.security.action=ALL-UNNAMED
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
-Djdk.reflect.useDirectMethodHandle=false"
"spark.comet.exec.shuffle.enabled" -> "true"
```
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
