[
https://issues.apache.org/jira/browse/SPARK-39448?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Dongjoon Hyun resolved SPARK-39448.
-----------------------------------
Fix Version/s: 3.3.0
Resolution: Fixed
Issue resolved by pull request 36847
[https://github.com/apache/spark/pull/36847]
> Add ReplaceCTERefWithRepartition into nonExcludableRules list
> -------------------------------------------------------------
>
> Key: SPARK-39448
> URL: https://issues.apache.org/jira/browse/SPARK-39448
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 3.3.0, 3.4.0
> Reporter: Yuming Wang
> Assignee: Yuming Wang
> Priority: Major
> Fix For: 3.3.0
>
>
> A unit test to test excluded rules:
> {code:scala}
> /*
> * Licensed to the Apache Software Foundation (ASF) under one or more
> * contributor license agreements. See the NOTICE file distributed with
> * this work for additional information regarding copyright ownership.
> * The ASF licenses this file to You under the Apache License, Version 2.0
> * (the "License"); you may not use this file except in compliance with
> * the License. You may obtain a copy of the License at
> *
> * http://www.apache.org/licenses/LICENSE-2.0
> *
> * Unless required by applicable law or agreed to in writing, software
> * distributed under the License is distributed on an "AS IS" BASIS,
> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> * See the License for the specific language governing permissions and
> * limitations under the License.
> */
> package org.apache.spark.sql
> import org.apache.spark.internal.config.Tests.IS_TESTING
> import org.apache.spark.launcher.SparkLauncher
> import org.apache.spark.sql.catalyst.util.resourceToString
> import org.apache.spark.sql.internal.SQLConf
> // scalastyle:off println
> object ExcludedRulesTestSuite extends TPCDSBase {
> override val spark = SparkSession.builder()
> .appName(this.getClass.getSimpleName)
> .master("local[40]")
> .config(SparkLauncher.DRIVER_MEMORY, "4g")
> .config(SQLConf.SHUFFLE_PARTITIONS.key, 2)
> .getOrCreate()
> override def injectStats: Boolean = true
> createTables()
> private lazy val excludableRules = spark.sessionState.optimizer.batches
> .flatMap(_.rules.map(_.ruleName))
> .distinct
> .filterNot(spark.sessionState.optimizer.nonExcludableRules.contains)
> def main(args: Array[String]): Unit = {
> System.setProperty(IS_TESTING.key, "true")
> tpcdsQueries.foreach { name =>
> val queryString = resourceToString(s"tpcds/$name.sql",
> classLoader = Thread.currentThread().getContextClassLoader)
> excludableRules.foreach { rule =>
> println(name + ": " + rule)
> try {
> withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> rule) {
> sql(queryString).collect()
> }
> } catch {
> case e: Exception =>
> println("Exception: " + e.getMessage)
> }
> }
> }
> tpcdsQueriesV2_7_0.foreach { name =>
> val queryString = resourceToString(s"tpcds-v2.7.0/$name.sql",
> classLoader = Thread.currentThread().getContextClassLoader)
> excludableRules.foreach { rule =>
> println("tpcds-v2.7.0 " + name + ": " + rule)
> try {
> withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> rule) {
> sql(queryString).collect()
> }
> } catch {
> case e: Exception =>
> println("Exception: " + e.getMessage)
> }
> }
> }
> modifiedTPCDSQueries.foreach { name =>
> val queryString = resourceToString(s"tpcds-modifiedQueries/$name.sql",
> classLoader = Thread.currentThread().getContextClassLoader)
> excludableRules.foreach { rule =>
> println("tpcds-modifiedQueries " + name + ": " + rule)
> try {
> withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> rule) {
> sql(queryString).collect()
> }
> } catch {
> case e: Exception =>
> println("Exception: " + e.getMessage)
> }
> }
> }
> // scalastyle:on println
> }
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
