[spark] branch master updated: [SPARK-24783][SQL] spark.sql.shuffle.partitions=0 should throw exception

[gurwls223]

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 5ebb4b5  [SPARK-24783][SQL] spark.sql.shuffle.partitions=0 should 
throw exception
5ebb4b5 is described below

commit 5ebb4b572318abf1b483b5a8f41aa9b021eb5327
Author: Sean Owen <sean.o...@databricks.com>
AuthorDate: Fri Mar 8 14:09:53 2019 +0900

    [SPARK-24783][SQL] spark.sql.shuffle.partitions=0 should throw exception
    
    ## What changes were proposed in this pull request?
    
    Throw an exception if spark.sql.shuffle.partitions=0
    This takes over https://github.com/apache/spark/pull/23835
    
    ## How was this patch tested?
    
    Existing tests.
    
    Closes #24008 from srowen/SPARK-24783.2.
    
    Lead-authored-by: Sean Owen <sean.o...@databricks.com>
    Co-authored-by: WindCanDie <491237...@qq.com>
    Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala   |  1 +
 .../scala/org/apache/spark/sql/internal/SQLConfSuite.scala   | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index e6cfd9d..2271bac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -277,6 +277,7 @@ object SQLConf {
       "Note: For structured streaming, this configuration cannot be changed 
between query " +
       "restarts from the same checkpoint location.")
     .intConf
+    .checkValue(_ > 0, "The value of spark.sql.shuffle.partitions must be 
positive")
     .createWithDefault(200)
 
   val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE =
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index 62cf705..5ecb79b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.internal
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.sql._
-import org.apache.spark.sql.execution.WholeStageCodegenExec
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.sql.test.{SharedSQLContext, TestSQLContext}
 import org.apache.spark.util.Utils
@@ -310,4 +309,15 @@ class SQLConfSuite extends QueryTest with SharedSQLContext 
{
     SQLConf.unregister(fallback)
   }
 
+  test("SPARK-24783: spark.sql.shuffle.partitions=0 should throw exception ") {
+    val e = intercept[IllegalArgumentException] {
+      spark.conf.set(SQLConf.SHUFFLE_PARTITIONS.key, 0)
+    }
+    assert(e.getMessage.contains("spark.sql.shuffle.partitions"))
+    val e2 = intercept[IllegalArgumentException] {
+      spark.conf.set(SQLConf.SHUFFLE_PARTITIONS.key, -1)
+    }
+    assert(e2.getMessage.contains("spark.sql.shuffle.partitions"))
+  }
+
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org