commits@spark.apache.org
This is an automated email from the ASF dual-hosted git repository. yao pushed a commit to branch branch-3.5 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-3.5 by this push: new fbc150fbbb70 [SPARK-45847][SQL][TESTS] CliSuite flakiness due to non-sequential guarantee for stdout&stderr fbc150fbbb70 is described below commit fbc150fbbb702f18ca12c6e6dec3fe01dbe76612 Author: Kent Yao AuthorDate: Thu Nov 9 16:23:38 2023 +0800 [SPARK-45847][SQL][TESTS] CliSuite flakiness due to non-sequential guarantee for stdout&stderr ### What changes were proposed in this pull request? In CliSuite, This PR adds a retry for tests that write errors to STDERR. ### Why are the changes needed? To fix flakiness tests as below https://github.com/chenhao-db/apache-spark/actions/runs/6791437199/job/18463313766 https://github.com/dongjoon-hyun/spark/actions/runs/6753670527/job/18361206900 ```sql [info] Spark master: local, Application Id: local-1699402393189 [info] spark-sql> /* SELECT /*+ HINT() 4; */; [info] [info] [PARSE_SYNTAX_ERROR] Syntax error at or near ';'. SQLSTATE: 42601 (line 1, pos 26) [info] [info] == SQL == [info] /* SELECT /*+ HINT() 4; */; [info] --^^^ [info] [info] spark-sql> /* SELECT /*+ HINT() 4; */ SELECT 1; [info] 1 [info] Time taken: 1.499 seconds, Fetched 1 row(s) [info] [info] [UNCLOSED_BRACKETED_COMMENT] Found an unclosed bracketed comment. Please, append */ at the end of the comment. SQLSTATE: 42601 [info] == SQL == [info] /* Here is a unclosed bracketed comment SELECT 1; [info] spark-sql> /* Here is a unclosed bracketed comment SELECT 1; [info] spark-sql> /* SELECT /*+ HINT() */ 4; */; [info] spark-sql> ``` As you can see the fragment above, the query on the 3rd line from the bottom, came from STDOUT, was printed later than its error output, came from STDERR. In this scenario, the error output would not match anything and would simply go unnoticed. Finally, timed out and failed. ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests and CI ### Was this patch authored or co-authored using generative AI tooling? no Closes #43725 from yaooqinn/SPARK-45847. Authored-by: Kent Yao Signed-off-by: Kent Yao (cherry picked from commit 06d8cbe073499ff16bca3165e2de1192daad3984) Signed-off-by: Kent Yao --- .../apache/spark/sql/hive/thriftserver/CliSuite.scala | 19 +-- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 8ba9ea28a5a9..343b32e6227c 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -383,7 +383,7 @@ class CliSuite extends SparkFunSuite { ) } - test("SPARK-11188 Analysis error reporting") { + testRetry("SPARK-11188 Analysis error reporting") { runCliWithin(timeout = 2.minute, errorResponses = Seq("AnalysisException"))( "select * from nonexistent_table;" -> "nonexistent_table" @@ -551,7 +551,7 @@ class CliSuite extends SparkFunSuite { ) } - test("SparkException with root cause will be printStacktrace") { + testRetry("SparkException with root cause will be printStacktrace") { // If it is not in silent mode, will print the stacktrace runCliWithin( 1.minute, @@ -575,8 +575,8 @@ class CliSuite extends SparkFunSuite { runCliWithin(1.minute)("SELECT MAKE_DATE(-44, 3, 15);" -> "-0044-03-15") } - test("SPARK-33100: Ignore a semicolon inside a bracketed comment in spark-sql") { -runCliWithin(4.minute)( + testRetry("SPARK-33100: Ignore a semicolon inside a bracketed comment in spark-sql") { +runCliWithin(1.minute)( "/* SELECT 'test';*/ SELECT 'test';" -> "test", ";;/* SELECT 'test';*/ SELECT 'test';" -> "test", "/* SELECT 'test';*/;; SELECT 'test';" -> "test", @@ -623,8 +623,8 @@ class CliSuite extends SparkFunSuite { ) } - test("SPARK-37555: spark-sql should pass last unclosed comment to backend") { -runCliWithin(5.minute)( + testRetry("SPARK-37555: spark-sql should pass last unclosed comment to backend") { +runCliWithin(1.minute)( // Only unclosed comment. "/* SELECT /*+ HINT() 4; */;".stripMargin -> "Syntax error at or near ';'", // Unclosed nested bracketed comment. @@ -637,7 +637,7 @@ class CliSuite extends SparkFunSuite { ) } - test("SPARK-37694: delete [jar|file|ar
commits@spark.apache.org
This is an automated email from the ASF dual-hosted git repository. yao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new 06d8cbe07349 [SPARK-45847][SQL][TESTS] CliSuite flakiness due to non-sequential guarantee for stdout&stderr 06d8cbe07349 is described below commit 06d8cbe073499ff16bca3165e2de1192daad3984 Author: Kent Yao AuthorDate: Thu Nov 9 16:23:38 2023 +0800 [SPARK-45847][SQL][TESTS] CliSuite flakiness due to non-sequential guarantee for stdout&stderr ### What changes were proposed in this pull request? In CliSuite, This PR adds a retry for tests that write errors to STDERR. ### Why are the changes needed? To fix flakiness tests as below https://github.com/chenhao-db/apache-spark/actions/runs/6791437199/job/18463313766 https://github.com/dongjoon-hyun/spark/actions/runs/6753670527/job/18361206900 ```sql [info] Spark master: local, Application Id: local-1699402393189 [info] spark-sql> /* SELECT /*+ HINT() 4; */; [info] [info] [PARSE_SYNTAX_ERROR] Syntax error at or near ';'. SQLSTATE: 42601 (line 1, pos 26) [info] [info] == SQL == [info] /* SELECT /*+ HINT() 4; */; [info] --^^^ [info] [info] spark-sql> /* SELECT /*+ HINT() 4; */ SELECT 1; [info] 1 [info] Time taken: 1.499 seconds, Fetched 1 row(s) [info] [info] [UNCLOSED_BRACKETED_COMMENT] Found an unclosed bracketed comment. Please, append */ at the end of the comment. SQLSTATE: 42601 [info] == SQL == [info] /* Here is a unclosed bracketed comment SELECT 1; [info] spark-sql> /* Here is a unclosed bracketed comment SELECT 1; [info] spark-sql> /* SELECT /*+ HINT() */ 4; */; [info] spark-sql> ``` As you can see the fragment above, the query on the 3rd line from the bottom, came from STDOUT, was printed later than its error output, came from STDERR. In this scenario, the error output would not match anything and would simply go unnoticed. Finally, timed out and failed. ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests and CI ### Was this patch authored or co-authored using generative AI tooling? no Closes #43725 from yaooqinn/SPARK-45847. Authored-by: Kent Yao Signed-off-by: Kent Yao --- .../apache/spark/sql/hive/thriftserver/CliSuite.scala | 19 +-- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 5391965ded2e..4f0d4dff566c 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -383,7 +383,7 @@ class CliSuite extends SparkFunSuite { ) } - test("SPARK-11188 Analysis error reporting") { + testRetry("SPARK-11188 Analysis error reporting") { runCliWithin(timeout = 2.minute, errorResponses = Seq("AnalysisException"))( "select * from nonexistent_table;" -> "nonexistent_table" @@ -551,7 +551,7 @@ class CliSuite extends SparkFunSuite { ) } - test("SparkException with root cause will be printStacktrace") { + testRetry("SparkException with root cause will be printStacktrace") { // If it is not in silent mode, will print the stacktrace runCliWithin( 1.minute, @@ -575,8 +575,8 @@ class CliSuite extends SparkFunSuite { runCliWithin(1.minute)("SELECT MAKE_DATE(-44, 3, 15);" -> "-0044-03-15") } - test("SPARK-33100: Ignore a semicolon inside a bracketed comment in spark-sql") { -runCliWithin(4.minute)( + testRetry("SPARK-33100: Ignore a semicolon inside a bracketed comment in spark-sql") { +runCliWithin(1.minute)( "/* SELECT 'test';*/ SELECT 'test';" -> "test", ";;/* SELECT 'test';*/ SELECT 'test';" -> "test", "/* SELECT 'test';*/;; SELECT 'test';" -> "test", @@ -623,8 +623,8 @@ class CliSuite extends SparkFunSuite { ) } - test("SPARK-37555: spark-sql should pass last unclosed comment to backend") { -runCliWithin(5.minute)( + testRetry("SPARK-37555: spark-sql should pass last unclosed comment to backend") { +runCliWithin(1.minute)( // Only unclosed comment. "/* SELECT /*+ HINT() 4; */;".stripMargin -> "Syntax error at or near ';'", // Unclosed nested bracketed comment. @@ -637,7 +637,7 @@ class CliSuite extends SparkFunSuite { ) } - test("SPARK-37694: delete [jar|file|archive] shall use spark sql processor") { + testRetry("SPARK-37694: delete [jar|file|archive] shall use spark
(spark) branch master updated: [SPARK-44886][SQL] Introduce CLUSTER BY clause for CREATE/REPLACE TABLE
This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new 5ac88b12f86b [SPARK-44886][SQL] Introduce CLUSTER BY clause for CREATE/REPLACE TABLE 5ac88b12f86b is described below commit 5ac88b12f86b306e7612591154c26aebabb957a8 Author: Terry Kim AuthorDate: Thu Nov 9 19:30:12 2023 +0800 [SPARK-44886][SQL] Introduce CLUSTER BY clause for CREATE/REPLACE TABLE ### What changes were proposed in this pull request? This proposes to introduce `CLUSTER BY` SQL clause to CREATE/REPLACE SQL syntax: ``` CREATE TABLE tbl(a int, b string) CLUSTER BY (a, b) ``` This doesn't introduce a default implementation for clustering, but it's up to the catalog/datasource implementation to utilize the clustering information (e.g., Delta, Iceberg, etc.). ### Why are the changes needed? To introduce the concept of clustering to datasources. ### Does this PR introduce _any_ user-facing change? Yes, this introduces a new SQL keyword. ### How was this patch tested? Added extensive unit tests. ### Was this patch authored or co-authored using generative AI tooling? No Closes #42577 from imback82/cluster_by. Lead-authored-by: Terry Kim Co-authored-by: Terry Kim Signed-off-by: Wenchen Fan --- .../src/main/resources/error/error-classes.json| 12 +++ docs/sql-error-conditions.md | 12 +++ .../spark/sql/catalyst/parser/SqlBaseParser.g4 | 5 + .../spark/sql/errors/QueryParsingErrors.scala | 8 ++ .../spark/sql/catalyst/catalog/interface.scala | 63 +++- .../spark/sql/catalyst/parser/AstBuilder.scala | 46 +++-- .../sql/connector/catalog/CatalogV2Implicits.scala | 26 - .../sql/connector/expressions/expressions.scala| 36 +++ .../spark/sql/catalyst/parser/DDLParserSuite.scala | 110 - .../sql/connector/catalog/InMemoryBaseTable.scala | 1 + .../expressions/TransformExtractorSuite.scala | 43 +++- .../catalyst/analysis/ResolveSessionCatalog.scala | 8 +- .../spark/sql/execution/SparkSqlParser.scala | 3 +- .../datasources/v2/V2SessionCatalog.scala | 8 +- .../apache/spark/sql/internal/CatalogImpl.scala| 3 +- .../command/CreateTableClusterBySuiteBase.scala| 83 .../command/v1/CreateTableClusterBySuite.scala | 51 ++ .../command/v2/CreateTableClusterBySuite.scala | 50 ++ .../command/CreateTableClusterBySuite.scala| 39 19 files changed, 583 insertions(+), 24 deletions(-) diff --git a/common/utils/src/main/resources/error/error-classes.json b/common/utils/src/main/resources/error/error-classes.json index c38171c3d9e6..26f6c0240afb 100644 --- a/common/utils/src/main/resources/error/error-classes.json +++ b/common/utils/src/main/resources/error/error-classes.json @@ -2963,6 +2963,18 @@ ], "sqlState" : "42601" }, + "SPECIFY_CLUSTER_BY_WITH_BUCKETING_IS_NOT_ALLOWED" : { +"message" : [ + "Cannot specify both CLUSTER BY and CLUSTERED BY INTO BUCKETS." +], +"sqlState" : "42908" + }, + "SPECIFY_CLUSTER_BY_WITH_PARTITIONED_BY_IS_NOT_ALLOWED" : { +"message" : [ + "Cannot specify both CLUSTER BY and PARTITIONED BY." +], +"sqlState" : "42908" + }, "SPECIFY_PARTITION_IS_NOT_ALLOWED" : { "message" : [ "A CREATE TABLE without explicit column list cannot specify PARTITIONED BY.", diff --git a/docs/sql-error-conditions.md b/docs/sql-error-conditions.md index 8a5faa15dc9c..2cb433b19fa5 100644 --- a/docs/sql-error-conditions.md +++ b/docs/sql-error-conditions.md @@ -1852,6 +1852,18 @@ A CREATE TABLE without explicit column list cannot specify bucketing information Please use the form with explicit column list and specify bucketing information. Alternatively, allow bucketing information to be inferred by omitting the clause. +### SPECIFY_CLUSTER_BY_WITH_BUCKETING_IS_NOT_ALLOWED + +[SQLSTATE: 42908](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation) + +Cannot specify both CLUSTER BY and CLUSTERED BY INTO BUCKETS. + +### SPECIFY_CLUSTER_BY_WITH_PARTITIONED_BY_IS_NOT_ALLOWED + +[SQLSTATE: 42908](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation) + +Cannot specify both CLUSTER BY and PARTITIONED BY. + ### SPECIFY_PARTITION_IS_NOT_ALLOWED [SQLSTATE: 42601](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation) diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index 84a31dafed98..bd449a4e194e 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalys
(spark) branch master updated: [SPARK-45815][SQL][STREAMING] Provide an interface for other Streaming sources to add `_metadata` columns
This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new 1e93c408e19f [SPARK-45815][SQL][STREAMING] Provide an interface for other Streaming sources to add `_metadata` columns 1e93c408e19f is described below commit 1e93c408e19f4ce8cec8220fd5eb6c1cb76468ff Author: Yaohua Zhao AuthorDate: Thu Nov 9 19:35:51 2023 +0800 [SPARK-45815][SQL][STREAMING] Provide an interface for other Streaming sources to add `_metadata` columns ### What changes were proposed in this pull request? Currently, only the native V1 file-based streaming source can read the `_metadata` column: https://github.com/apache/spark/blob/370870b7a0303e4a2c4b3dea1b479b4fcbc93f8d/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala#L63 Our goal is to create an interface that allows other streaming sources to add `_metadata` columns. For instance, we would like the Delta Streaming source, which you can find here: https://github.com/delta-io/delta/blob/master/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaDataSource.scala#L49, to extend this interface and provide the `_metadata` column for its underlying storage format, such as Parquet. ### Why are the changes needed? A generic interface to enable other streaming sources to expose and add `_metadata` columns. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? N/A ### Was this patch authored or co-authored using generative AI tooling? No Closes #43692 from Yaohua628/spark-45815. Authored-by: Yaohua Zhao Signed-off-by: Wenchen Fan --- .../execution/streaming/StreamingRelation.scala| 11 .../org/apache/spark/sql/sources/interfaces.scala | 31 ++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala index 135d46c5291e..c5d5a79d3454 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala @@ -29,6 +29,7 @@ import org.apache.spark.sql.connector.read.streaming.SparkDataStream import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.LeafExecNode import org.apache.spark.sql.execution.datasources.{DataSource, FileFormat} +import org.apache.spark.sql.sources.SupportsStreamSourceMetadataColumns object StreamingRelation { def apply(dataSource: DataSource): StreamingRelation = { @@ -60,11 +61,11 @@ case class StreamingRelation(dataSource: DataSource, sourceName: String, output: override def newInstance(): LogicalPlan = this.copy(output = output.map(_.newInstance())) override lazy val metadataOutput: Seq[AttributeReference] = { -dataSource.providingClass match { - // If the dataSource provided class is a same or subclass of FileFormat class - case f if classOf[FileFormat].isAssignableFrom(f) => -metadataOutputWithOutConflicts( - Seq(dataSource.providingInstance().asInstanceOf[FileFormat].createFileMetadataCol())) +dataSource.providingInstance() match { + case f: FileFormat => metadataOutputWithOutConflicts(Seq(f.createFileMetadataCol())) + case s: SupportsStreamSourceMetadataColumns => +metadataOutputWithOutConflicts(s.getMetadataOutput( + dataSource.sparkSession, dataSource.options, dataSource.userSpecifiedSchema)) case _ => Nil } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala index 63e57c6804e1..d194ae77e968 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala @@ -309,3 +309,34 @@ trait InsertableRelation { trait CatalystScan { def buildScan(requiredColumns: Seq[Attribute], filters: Seq[Expression]): RDD[Row] } + +/** + * Implemented by StreamSourceProvider objects that can generate file metadata columns. + * This trait extends the basic StreamSourceProvider by allowing the addition of metadata + * columns to the schema of the Stream Data Source. + */ +trait SupportsStreamSourceMetadataColumns extends StreamSourceProvider { + + /** + * Returns the metadata columns that should be added to the schema of the Stream Source. + * These metadata columns supplement the columns + * defined in the sourceSchema() of the StreamSourceProvider. + * + * The final schema for the Stream Source, therefore,
(spark) branch master updated: [SPARK-45867][CORE] Support `spark.worker.idPattern`
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new 7bc96e8e3767 [SPARK-45867][CORE] Support `spark.worker.idPattern` 7bc96e8e3767 is described below commit 7bc96e8e37672483a07088dbbdcf3610a497af1d Author: Dongjoon Hyun AuthorDate: Thu Nov 9 13:59:44 2023 -0800 [SPARK-45867][CORE] Support `spark.worker.idPattern` ### What changes were proposed in this pull request? This PR aims to support `spark.worker.idPattern`. ### Why are the changes needed? To allow users to customize the worker IDs if they want. - From: `worker-20231109183042-[fe80::1%lo0]-39729` - To: `my-worker-20231109183042-[fe80::1%lo0]` For example, ``` $ cat conf/spark-defaults.conf spark.worker.idPattern worker-%2$s ``` https://github.com/apache/spark/assets/9700541/bc367e77-c19d-44f1-bbc5-3f4c5edec33d";> ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs with newly added test case. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43740 from dongjoon-hyun/SPARK-45867. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../scala/org/apache/spark/deploy/worker/Worker.scala | 3 ++- .../org/apache/spark/internal/config/Worker.scala | 11 +++ .../org/apache/spark/deploy/worker/WorkerSuite.scala | 18 -- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index 44082ae78794..ddbba55e00b4 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -62,6 +62,7 @@ private[deploy] class Worker( private val host = rpcEnv.address.host private val port = rpcEnv.address.port + private val workerIdPattern = conf.get(config.Worker.WORKER_ID_PATTERN) Utils.checkHost(host) assert (port > 0) @@ -813,7 +814,7 @@ private[deploy] class Worker( } private def generateWorkerId(): String = { -"worker-%s-%s-%d".format(createDateFormat.format(new Date), host, port) +workerIdPattern.format(createDateFormat.format(new Date), host, port) } override def onStop(): Unit = { diff --git a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala index fda3a57546b6..f160470edd8f 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala @@ -89,4 +89,15 @@ private[spark] object Worker { .version("3.2.0") .stringConf .createWithDefaultString("PWR") + + val WORKER_ID_PATTERN = ConfigBuilder("spark.worker.idPattern") +.internal() +.doc("The pattern for worker ID generation based on Java `String.format` method. The " + + "default value is `worker-%s-%s-%d` which represents the existing worker id string, e.g.," + + " `worker-20231109183042-[fe80::1%lo0]-39729`. Please be careful to generate unique IDs") +.version("4.0.0") +.stringConf +.checkValue(!_.format("2023110900", "host", 0).exists(_.isWhitespace), + "Whitespace is not allowed.") +.createWithDefaultString("worker-%s-%s-%d") } diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala index a07d4f76905a..1b2d92af4b02 100644 --- a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala @@ -29,7 +29,7 @@ import org.mockito.Answers.RETURNS_SMART_NULLS import org.mockito.ArgumentMatchers.any import org.mockito.Mockito._ import org.mockito.invocation.InvocationOnMock -import org.scalatest.BeforeAndAfter +import org.scalatest.{BeforeAndAfter, PrivateMethodTester} import org.scalatest.concurrent.Eventually.{eventually, interval, timeout} import org.scalatest.matchers.must.Matchers import org.scalatest.matchers.should.Matchers._ @@ -49,7 +49,7 @@ import org.apache.spark.resource.TestResourceIDs.{WORKER_FPGA_ID, WORKER_GPU_ID} import org.apache.spark.rpc.{RpcAddress, RpcEnv} import org.apache.spark.util.Utils -class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter { +class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter with PrivateMethodTester { import org.apache.spark.deploy.DeployTestUtils._ @@ -62,6 +62,8 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter { implicit val formats =
(spark) branch master updated: [SPARK-45731][SQL] Also update partition statistics with `ANALYZE TABLE` command
This is an automated email from the ASF dual-hosted git repository. sunchao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new ce818ba96953 [SPARK-45731][SQL] Also update partition statistics with `ANALYZE TABLE` command ce818ba96953 is described below commit ce818ba969537cf9eb16865a88148407a5992e98 Author: Chao Sun AuthorDate: Thu Nov 9 15:56:47 2023 -0800 [SPARK-45731][SQL] Also update partition statistics with `ANALYZE TABLE` command ### What changes were proposed in this pull request? Also update partition statistics (e.g., total size in bytes, row count) with `ANALYZE TABLE` command. ### Why are the changes needed? Currently when a `ANALYZE TABLE ` command is triggered against a partition table, only table stats are updated, but not partition stats. For Spark users who want to update the latter, they have to use a different syntax: `ANALYZE TABLE PARTITION()` which is more verbose. Given `ANALYZE TABLE` internally already calculates total size for all the partitions, it makes sense to also update partition stats using the result. In this way, Spark users do not need to remember two different syntaxes. In addition, when using `ANALYZE TABLE` with the "scan node", i.e., `NOSCAN` is NOT specified, we can also calculate row count for all the partitions and update the stats accordingly. The above behavior is controlled via a new flag `spark.sql.statistics.updatePartitionStatsInAnalyzeTable.enabled`, which by default is turned off. ### Does this PR introduce _any_ user-facing change? Not by default. When `spark.sql.statistics.updatePartitionStatsInAnalyzeTable.enabled`, Spark will now update partition stats as well with `ANALYZE TABLE` command, on a partitioned table. ### How was this patch tested? Added a unit test for this feature. ### Was this patch authored or co-authored using generative AI tooling? No Closes #43629 from sunchao/SPARK-45731. Authored-by: Chao Sun Signed-off-by: Chao Sun --- .../org/apache/spark/sql/internal/SQLConf.scala| 13 .../command/AnalyzePartitionCommand.scala | 50 ++--- .../spark/sql/execution/command/CommandUtils.scala | 87 ++ .../apache/spark/sql/hive/StatisticsSuite.scala| 78 +++ 4 files changed, 170 insertions(+), 58 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index ecc3e6e101fc..ff6ab7b541a3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2664,6 +2664,16 @@ object SQLConf { .booleanConf .createWithDefault(false) + val UPDATE_PART_STATS_IN_ANALYZE_TABLE_ENABLED = + buildConf("spark.sql.statistics.updatePartitionStatsInAnalyzeTable.enabled") + .doc("When this config is enabled, Spark will also update partition statistics in analyze " + +"table command (i.e., ANALYZE TABLE .. COMPUTE STATISTICS [NOSCAN]). Note the command " + +"will also become more expensive. When this config is disabled, Spark will only " + +"update table level statistics.") + .version("4.0.0") + .booleanConf + .createWithDefault(false) + val CBO_ENABLED = buildConf("spark.sql.cbo.enabled") .doc("Enables CBO for estimation of plan statistics when set true.") @@ -5113,6 +5123,9 @@ class SQLConf extends Serializable with Logging with SqlApiConf { def autoSizeUpdateEnabled: Boolean = getConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED) + def updatePartStatsInAnalyzeTableEnabled: Boolean = +getConf(SQLConf.UPDATE_PART_STATS_IN_ANALYZE_TABLE_ENABLED) + def joinReorderEnabled: Boolean = getConf(SQLConf.JOIN_REORDER_ENABLED) def joinReorderDPThreshold: Int = getConf(SQLConf.JOIN_REORDER_DP_THRESHOLD) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala index c2b227d6cad7..7fe4c73abf90 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala @@ -17,15 +17,12 @@ package org.apache.spark.sql.execution.command -import org.apache.spark.sql.{Column, Row, SparkSession} +import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, ExternalCatalogUti
(spark) branch master updated (ce818ba96953 -> d9c5f9d6d42a)
This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git from ce818ba96953 [SPARK-45731][SQL] Also update partition statistics with `ANALYZE TABLE` command add d9c5f9d6d42a [SPARK-45798][CONNECT] Assert server-side session ID No new revisions were added by this update. Summary of changes: .../spark/sql/connect/client/ArtifactSuite.scala | 6 +- .../src/main/protobuf/spark/connect/base.proto | 69 +- .../spark/sql/connect/client/ArtifactManager.scala | 10 +- .../client/CustomSparkConnectBlockingStub.scala| 34 ++- .../connect/client/CustomSparkConnectStub.scala| 6 +- .../ExecutePlanResponseReattachableIterator.scala | 19 +- .../connect/client/GrpcExceptionConverter.scala| 10 +- .../sql/connect/client/ResponseValidator.scala | 111 + .../sql/connect/client/SparkConnectClient.scala| 7 +- .../sql/connect/client/SparkConnectStubState.scala | 43 .../artifact/SparkConnectArtifactManager.scala | 2 +- .../execution/ExecuteResponseObserver.scala| 1 + .../execution/SparkConnectPlanExecution.scala | 11 +- .../sql/connect/planner/SparkConnectPlanner.scala | 7 +- .../spark/sql/connect/service/SessionHolder.scala | 7 + .../service/SparkConnectAddArtifactsHandler.scala | 2 + .../service/SparkConnectAnalyzeHandler.scala | 4 +- .../SparkConnectArtifactStatusesHandler.scala | 5 + .../service/SparkConnectConfigHandler.scala| 5 +- .../service/SparkConnectInterruptHandler.scala | 1 + .../SparkConnectReleaseExecuteHandler.scala| 1 + .../SparkConnectReleaseSessionHandler.scala| 3 + .../spark/sql/connect/utils/MetricGenerator.scala | 8 +- .../spark/sql/connect/SparkConnectServerTest.scala | 7 +- .../connect/artifact/ArtifactManagerSuite.scala| 18 +- .../service/ArtifactStatusesHandlerSuite.scala | 7 +- .../service/SparkConnectServiceE2ESuite.scala | 2 + python/pyspark/sql/connect/client/core.py | 92 python/pyspark/sql/connect/proto/base_pb2.py | 260 ++--- python/pyspark/sql/connect/proto/base_pb2.pyi | 165 +++-- 30 files changed, 671 insertions(+), 252 deletions(-) create mode 100644 connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/ResponseValidator.scala create mode 100644 connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/SparkConnectStubState.scala - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
(spark) branch master updated: [SPARK-45756][CORE] Support `spark.master.useAppNameAsAppId.enabled`
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new a566099133ff [SPARK-45756][CORE] Support `spark.master.useAppNameAsAppId.enabled` a566099133ff is described below commit a566099133ff38cd1b2cd2fe64879bf0ba75fa9b Author: Dongjoon Hyun AuthorDate: Thu Nov 9 18:34:43 2023 -0800 [SPARK-45756][CORE] Support `spark.master.useAppNameAsAppId.enabled` ### What changes were proposed in this pull request? This PR aims to support `spark.master.useAppNameAsAppId.enabled` as an experimental feature in Spark Standalone cluster. ### Why are the changes needed? This allows the users to control the appID completely. https://github.com/apache/spark/assets/9700541/ad2b89ce-9d7d-4144-bd52-f29b94051103";> ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual tests with the following procedure. ``` $ SPARK_MASTER_OPTS="-Dspark.master.useAppNameAsAppId.enabled=true" sbin/start-master.sh $ bin/spark-shell --master spark://max.local:7077 ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43743 from dongjoon-hyun/SPARK-45756. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../scala/org/apache/spark/deploy/master/Master.scala| 7 ++- .../scala/org/apache/spark/internal/config/package.scala | 8 .../org/apache/spark/deploy/master/MasterSuite.scala | 16 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index dbb647252c5f..b3fbec1830e4 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -120,6 +120,7 @@ private[deploy] class Master( private val defaultCores = conf.get(DEFAULT_CORES) val reverseProxy = conf.get(UI_REVERSE_PROXY) val historyServerUrl = conf.get(MASTER_UI_HISTORY_SERVER_URL) + val useAppNameAsAppId = conf.get(MASTER_USE_APP_NAME_AS_APP_ID) // Alternative application submission gateway that is stable across Spark versions private val restServerEnabled = conf.get(MASTER_REST_SERVER_ENABLED) @@ -1041,7 +1042,11 @@ private[deploy] class Master( ApplicationInfo = { val now = System.currentTimeMillis() val date = new Date(now) -val appId = newApplicationId(date) +val appId = if (useAppNameAsAppId) { + desc.name.toLowerCase().replaceAll("\\s+", "") +} else { + newApplicationId(date) +} new ApplicationInfo(now, appId, desc, date, driver, defaultCores) } diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index bbadf91fc41c..b2bf30863a91 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -1846,6 +1846,14 @@ package object config { .stringConf .createOptional + private[spark] val MASTER_USE_APP_NAME_AS_APP_ID = +ConfigBuilder("spark.master.useAppNameAsAppId.enabled") + .internal() + .doc("(Experimental) If true, Spark master uses the user-provided appName for appId.") + .version("4.0.0") + .booleanConf + .createWithDefault(false) + private[spark] val IO_COMPRESSION_SNAPPY_BLOCKSIZE = ConfigBuilder("spark.io.compression.snappy.blockSize") .doc("Block size in bytes used in Snappy compression, in the case when " + diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala index 4f8457f930e4..2e54673649c7 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala @@ -804,6 +804,7 @@ class MasterSuite extends SparkFunSuite private val _state = PrivateMethod[RecoveryState.Value](Symbol("state")) private val _newDriverId = PrivateMethod[String](Symbol("newDriverId")) private val _newApplicationId = PrivateMethod[String](Symbol("newApplicationId")) + private val _createApplication = PrivateMethod[ApplicationInfo](Symbol("createApplication")) private val workerInfo = makeWorkerInfo(4096, 10) private val workerInfos = Array(workerInfo, workerInfo, workerInfo) @@ -1275,6 +1276,21 @@ class MasterSuite extends SparkFunSuite assert(master.invokePrivate(_newApplicationId(submitDate)) === s"${i % 1000}") } } + + test("SPARK-45756: Use appName for appId") { +val conf = new Sp
(spark) branch master updated (a566099133f -> 2085ac52175)
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git from a566099133f [SPARK-45756][CORE] Support `spark.master.useAppNameAsAppId.enabled` add 2085ac52175 [SPARK-45850][BUILD] Upgrade oracle jdbc driver to 23.3.0.23.09 No new revisions were added by this update. Summary of changes: connector/docker-integration-tests/pom.xml | 2 +- pom.xml| 4 ++-- sql/core/pom.xml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
(spark-docker) branch master updated: Add support for java 17 from spark 3.5.0
This is an automated email from the ASF dual-hosted git repository. yikun pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark-docker.git The following commit(s) were added to refs/heads/master by this push: new 6f68fe0 Add support for java 17 from spark 3.5.0 6f68fe0 is described below commit 6f68fe0f7051c10f2bf43a50a7decfce2e97baf0 Author: vakarisbk AuthorDate: Fri Nov 10 11:33:39 2023 +0800 Add support for java 17 from spark 3.5.0 ### What changes were proposed in this pull request? 1. Create Java17 base images alongside Java11 images starting from spark 3.5.0 2. Change ubuntu version to 22.04 for `scala2.12-java17-*` ### Why are the changes needed? Spark supports multiple Java versions, but the images are currently built only with Java 11. ### Does this PR introduce _any_ user-facing change? New images would be available in the repositories. ### How was this patch tested? Closes #56 from vakarisbk/master. Authored-by: vakarisbk Signed-off-by: Yikun Jiang --- .github/workflows/build_3.5.0.yaml | 3 +- .github/workflows/main.yml | 20 +++- .github/workflows/publish.yml | 4 +- .github/workflows/test.yml | 3 + 3.5.0/scala2.12-java17-python3-r-ubuntu/Dockerfile | 29 + 3.5.0/scala2.12-java17-python3-ubuntu/Dockerfile | 26 + 3.5.0/scala2.12-java17-r-ubuntu/Dockerfile | 28 + 3.5.0/scala2.12-java17-ubuntu/Dockerfile | 79 + 3.5.0/scala2.12-java17-ubuntu/entrypoint.sh| 130 + add-dockerfiles.sh | 23 +++- tools/ci_runner_cleaner/free_disk_space.sh | 53 + .../ci_runner_cleaner/free_disk_space_container.sh | 33 ++ tools/template.py | 2 +- versions.json | 29 + 14 files changed, 454 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_3.5.0.yaml b/.github/workflows/build_3.5.0.yaml index 6eb3ad6..9f2b2d6 100644 --- a/.github/workflows/build_3.5.0.yaml +++ b/.github/workflows/build_3.5.0.yaml @@ -31,11 +31,12 @@ jobs: strategy: matrix: image-type: ["all", "python", "scala", "r"] +java: [11, 17] name: Run secrets: inherit uses: ./.github/workflows/main.yml with: spark: 3.5.0 scala: 2.12 - java: 11 + java: ${{ matrix.java }} image-type: ${{ matrix.image-type }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fe755ed..145b529 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,6 +79,14 @@ jobs: - name: Checkout Spark Docker repository uses: actions/checkout@v3 + - name: Free up disk space +shell: 'script -q -e -c "bash {0}"' +run: | + chmod +x tools/ci_runner_cleaner/free_disk_space_container.sh + tools/ci_runner_cleaner/free_disk_space_container.sh + chmod +x tools/ci_runner_cleaner/free_disk_space.sh + tools/ci_runner_cleaner/free_disk_space.sh + - name: Prepare - Generate tags run: | case "${{ inputs.image-type }}" in @@ -195,7 +203,8 @@ jobs: - name : Test - Run spark application for standalone cluster on docker run: testing/run_tests.sh --image-url $IMAGE_URL --scala-version ${{ inputs.scala }} --spark-version ${{ inputs.spark }} - - name: Test - Checkout Spark repository + - name: Test - Checkout Spark repository for Spark 3.3.0 (with fetch-depth 0) +if: inputs.spark == '3.3.0' uses: actions/checkout@v3 with: fetch-depth: 0 @@ -203,6 +212,14 @@ jobs: ref: v${{ inputs.spark }} path: ${{ github.workspace }}/spark + - name: Test - Checkout Spark repository +if: inputs.spark != '3.3.0' +uses: actions/checkout@v3 +with: + repository: apache/spark + ref: v${{ inputs.spark }} + path: ${{ github.workspace }}/spark + - name: Test - Cherry pick commits # Apache Spark enable resource limited k8s IT since v3.3.1, cherry-pick patches for old release # https://github.com/apache/spark/pull/36087#issuecomment-1251756266 @@ -247,6 +264,7 @@ jobs: # TODO(SPARK-44495): Resume to use the latest minikube for k8s-integration-tests. curl -LO https://storage.googleapis.com/minikube/releases/v1.30.1/minikube-linux-amd64 sudo install minikube-linux-amd64 /usr/local/bin/minikube + rm minikube-linux-amd64 # Github Action limit cpu:2, memory: 6947MB, limit to 2U6G for better resource statistic minikube start --cpus 2 --memory 6144 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8cf
(spark) branch branch-3.4 updated: [SPARK-45814][CONNECT][SQL][3.4] Make ArrowConverters.createEmptyArrowBatch call close() to avoid memory leak
This is an automated email from the ASF dual-hosted git repository. yangjie01 pushed a commit to branch branch-3.4 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-3.4 by this push: new 259ac250017 [SPARK-45814][CONNECT][SQL][3.4] Make ArrowConverters.createEmptyArrowBatch call close() to avoid memory leak 259ac250017 is described below commit 259ac250017bcc1805f6cb44a5e7eedf9e552a98 Author: xieshuaihu AuthorDate: Fri Nov 10 12:33:24 2023 +0800 [SPARK-45814][CONNECT][SQL][3.4] Make ArrowConverters.createEmptyArrowBatch call close() to avoid memory leak ### What changes were proposed in this pull request? Make `ArrowBatchIterator` implement `AutoCloseable` and `ArrowConverters.createEmptyArrowBatch()` call close() to avoid memory leak. ### Why are the changes needed? `ArrowConverters.createEmptyArrowBatch` don't call `super.hasNext`, if `TaskContext.get` returns `None`, then memory allocated in `ArrowBatchIterator` is leaked. In spark connect, `createEmptyArrowBatch` is called in [SparkConnectPlanner](https://github.com/apache/spark/blob/master/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala#L2558) and [SparkConnectPlanExecution](https://github.com/apache/spark/blob/master/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/execution/SparkConnectPlanExecution.scala#L224), which cause a long running driver consume all off-heap memory sp [...] This is the exception stack: ``` org.apache.arrow.memory.OutOfMemoryException: Failure allocating buffer. at io.netty.buffer.PooledByteBufAllocatorL.allocate(PooledByteBufAllocatorL.java:67) at org.apache.arrow.memory.NettyAllocationManager.(NettyAllocationManager.java:77) at org.apache.arrow.memory.NettyAllocationManager.(NettyAllocationManager.java:84) at org.apache.arrow.memory.NettyAllocationManager$1.create(NettyAllocationManager.java:34) at org.apache.arrow.memory.BaseAllocator.newAllocationManager(BaseAllocator.java:354) at org.apache.arrow.memory.BaseAllocator.newAllocationManager(BaseAllocator.java:349) at org.apache.arrow.memory.BaseAllocator.bufferWithoutReservation(BaseAllocator.java:337) at org.apache.arrow.memory.BaseAllocator.buffer(BaseAllocator.java:315) at org.apache.arrow.memory.BaseAllocator.buffer(BaseAllocator.java:279) at org.apache.arrow.vector.BaseValueVector.allocFixedDataAndValidityBufs(BaseValueVector.java:192) at org.apache.arrow.vector.BaseFixedWidthVector.allocateBytes(BaseFixedWidthVector.java:338) at org.apache.arrow.vector.BaseFixedWidthVector.allocateNew(BaseFixedWidthVector.java:308) at org.apache.arrow.vector.BaseFixedWidthVector.allocateNew(BaseFixedWidthVector.java:273) at org.apache.spark.sql.execution.arrow.ArrowWriter$.$anonfun$create$1(ArrowWriter.scala:44) at scala.collection.StrictOptimizedIterableOps.map(StrictOptimizedIterableOps.scala:100) at scala.collection.StrictOptimizedIterableOps.map$(StrictOptimizedIterableOps.scala:87) at scala.collection.convert.JavaCollectionWrappers$JListWrapper.map(JavaCollectionWrappers.scala:103) at org.apache.spark.sql.execution.arrow.ArrowWriter$.create(ArrowWriter.scala:43) at org.apache.spark.sql.execution.arrow.ArrowConverters$ArrowBatchIterator.(ArrowConverters.scala:93) at org.apache.spark.sql.execution.arrow.ArrowConverters$ArrowBatchWithSchemaIterator.(ArrowConverters.scala:138) at org.apache.spark.sql.execution.arrow.ArrowConverters$$anon$1.(ArrowConverters.scala:231) at org.apache.spark.sql.execution.arrow.ArrowConverters$.createEmptyArrowBatch(ArrowConverters.scala:229) at org.apache.spark.sql.connect.planner.SparkConnectPlanner.handleSqlCommand(SparkConnectPlanner.scala:2481) at org.apache.spark.sql.connect.planner.SparkConnectPlanner.process(SparkConnectPlanner.scala:2426) at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.handleCommand(ExecuteThreadRunner.scala:202) at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:158) at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:132) at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:189) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900) at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:189) at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifa
commits@spark.apache.org
This is an automated email from the ASF dual-hosted git repository. yao pushed a commit to branch branch-3.4 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-3.4 by this push: new c53caddb6b1 [SPARK-45847][SQL][TESTS] CliSuite flakiness due to non-sequential guarantee for stdout&stderr c53caddb6b1 is described below commit c53caddb6b16390c969b4d0f8ee1ce8bc280ba8a Author: Kent Yao AuthorDate: Thu Nov 9 16:23:38 2023 +0800 [SPARK-45847][SQL][TESTS] CliSuite flakiness due to non-sequential guarantee for stdout&stderr ### What changes were proposed in this pull request? In CliSuite, This PR adds a retry for tests that write errors to STDERR. ### Why are the changes needed? To fix flakiness tests as below https://github.com/chenhao-db/apache-spark/actions/runs/6791437199/job/18463313766 https://github.com/dongjoon-hyun/spark/actions/runs/6753670527/job/18361206900 ```sql [info] Spark master: local, Application Id: local-1699402393189 [info] spark-sql> /* SELECT /*+ HINT() 4; */; [info] [info] [PARSE_SYNTAX_ERROR] Syntax error at or near ';'. SQLSTATE: 42601 (line 1, pos 26) [info] [info] == SQL == [info] /* SELECT /*+ HINT() 4; */; [info] --^^^ [info] [info] spark-sql> /* SELECT /*+ HINT() 4; */ SELECT 1; [info] 1 [info] Time taken: 1.499 seconds, Fetched 1 row(s) [info] [info] [UNCLOSED_BRACKETED_COMMENT] Found an unclosed bracketed comment. Please, append */ at the end of the comment. SQLSTATE: 42601 [info] == SQL == [info] /* Here is a unclosed bracketed comment SELECT 1; [info] spark-sql> /* Here is a unclosed bracketed comment SELECT 1; [info] spark-sql> /* SELECT /*+ HINT() */ 4; */; [info] spark-sql> ``` As you can see the fragment above, the query on the 3rd line from the bottom, came from STDOUT, was printed later than its error output, came from STDERR. In this scenario, the error output would not match anything and would simply go unnoticed. Finally, timed out and failed. ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing tests and CI ### Was this patch authored or co-authored using generative AI tooling? no Closes #43725 from yaooqinn/SPARK-45847. Authored-by: Kent Yao Signed-off-by: Kent Yao (cherry picked from commit 06d8cbe073499ff16bca3165e2de1192daad3984) Signed-off-by: Kent Yao --- .../apache/spark/sql/hive/thriftserver/CliSuite.scala | 19 +-- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 651c6b7aafb..1b91442c228 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -388,7 +388,7 @@ class CliSuite extends SparkFunSuite { ) } - test("SPARK-11188 Analysis error reporting") { + testRetry("SPARK-11188 Analysis error reporting") { runCliWithin(timeout = 2.minute, errorResponses = Seq("AnalysisException"))( "select * from nonexistent_table;" -> "nonexistent_table" @@ -556,7 +556,7 @@ class CliSuite extends SparkFunSuite { ) } - test("SparkException with root cause will be printStacktrace") { + testRetry("SparkException with root cause will be printStacktrace") { // If it is not in silent mode, will print the stacktrace runCliWithin( 1.minute, @@ -580,8 +580,8 @@ class CliSuite extends SparkFunSuite { runCliWithin(1.minute)("SELECT MAKE_DATE(-44, 3, 15);" -> "-0044-03-15") } - test("SPARK-33100: Ignore a semicolon inside a bracketed comment in spark-sql") { -runCliWithin(4.minute)( + testRetry("SPARK-33100: Ignore a semicolon inside a bracketed comment in spark-sql") { +runCliWithin(1.minute)( "/* SELECT 'test';*/ SELECT 'test';" -> "test", ";;/* SELECT 'test';*/ SELECT 'test';" -> "test", "/* SELECT 'test';*/;; SELECT 'test';" -> "test", @@ -628,8 +628,8 @@ class CliSuite extends SparkFunSuite { ) } - test("SPARK-37555: spark-sql should pass last unclosed comment to backend") { -runCliWithin(5.minute)( + testRetry("SPARK-37555: spark-sql should pass last unclosed comment to backend") { +runCliWithin(1.minute)( // Only unclosed comment. "/* SELECT /*+ HINT() 4; */;".stripMargin -> "Syntax error at or near ';'", // Unclosed nested bracketed comment. @@ -642,7 +642,7 @@ class CliSuite extends SparkFunSuite { ) } - test("SPARK-37694: delete [jar|file|archiv