This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8251f876fd6 [SPARK-40772][SQL] Improve
`spark.sql.adaptive.skewJoin.skewedPartitionFactor` to support `Double` values
8251f876fd6 is described below
commit 8251f876fd6a0300bbdac415091786dda3957532
Author: Dongjoon Hyun <dongj...@apache.org>
AuthorDate: Wed Oct 12 14:36:12 2022 -0700
[SPARK-40772][SQL] Improve
`spark.sql.adaptive.skewJoin.skewedPartitionFactor` to support `Double` values
### What changes were proposed in this pull request?
This PR aims to improve `spark.sql.adaptive.skewJoin.skewedPartitionFactor`
to support float values by converging to `doubleConf` from `intConf`.
### Why are the changes needed?
Like `spark.sql.adaptive.rebalancePartitionsSmallPartitionFactor`, this
allows users to use the configuration more flexibly.
### Does this PR introduce _any_ user-facing change?
Yes, but it will accept all previous Integer configuration values.
### How was this patch tested?
Pass the CIs with the changed default value, `5.0`.
Closes #38225 from dongjoon-hyun/SPARK-40772.
Authored-by: Dongjoon Hyun <dongj...@apache.org>
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
docs/sql-performance-tuning.md | 2 +-
.../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 4 ++--
.../org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index 9acfa5f2db4..d736ff8f83f 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -329,7 +329,7 @@ Data skew can severely downgrade the performance of join
queries. This feature d
</tr>
<tr>
<td><code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code></td>
- <td>5</td>
+ <td>5.0</td>
<td>
A partition is considered as skewed if its size is larger than this
factor multiplying the median partition size and also larger than
<code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code>.
</td>
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 161c4092f60..2f96209222b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -723,9 +723,9 @@ object SQLConf {
"multiplying the median partition size and also larger than " +
"'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'")
.version("3.0.0")
- .intConf
+ .doubleConf
.checkValue(_ >= 0, "The skew factor cannot be negative.")
- .createWithDefault(5)
+ .createWithDefault(5.0)
val SKEW_JOIN_SKEWED_PARTITION_THRESHOLD =
buildConf("spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes")
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index d4a173bb9cc..37cdea084d8 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -64,7 +64,7 @@ case class OptimizeSkewedJoin(ensureRequirements:
EnsureRequirements)
*/
def getSkewThreshold(medianSize: Long): Long = {
conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD).max(
- medianSize * conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_FACTOR))
+ (medianSize *
conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_FACTOR)).toLong)
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
