AnishMahto commented on code in PR #52119:
URL: https://github.com/apache/spark/pull/52119#discussion_r2316654811
##########
sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/SqlPipelineSuite.scala:
##########
@@ -266,6 +267,44 @@ class SqlPipelineSuite extends PipelineTest with
SharedSparkSession {
)
}
+ test("MV/ST with partition columns works") {
+ val unresolvedDataflowGraph = unresolvedDataflowGraphFromSql(
+ sqlText = """
+ |CREATE MATERIALIZED VIEW mv
+ |PARTITIONED BY (id_mod)
+ |AS
+ |SELECT
+ | id,
+ | id % 2 AS id_mod
+ |FROM range(3);
+ |
+ |CREATE STREAMING TABLE st
+ |PARTITIONED BY (id_mod)
+ |AS
+ |SELECT * FROM STREAM(mv);
+ |""".stripMargin
+ )
+ startPipelineAndWaitForCompletion(unresolvedDataflowGraph)
+ val expected = Seq(
+ Row(0, 0),
+ Row(1, 1),
+ Row(2, 0)
+ )
+ val catalog =
spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog]
+
+ Seq("mv", "st").foreach { tableName =>
+ // check table partition columns
+ val table = catalog.loadTable(Identifier.of(Array("test_db"), tableName))
Review Comment:
We should probably do this for PythonPipelineSuite though
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
