Github user gatorsmile commented on a diff in the pull request: https://github.com/apache/spark/pull/19083#discussion_r142020457 --- Diff: sql/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala --- @@ -95,50 +96,62 @@ class DataFrameTimeWindowingSuite extends QueryTest with SharedSQLContext with B } test("sliding window grouping") { - val df = Seq( - ("2016-03-27 19:39:34", 1, "a"), - ("2016-03-27 19:39:56", 2, "a"), - ("2016-03-27 19:39:27", 4, "b")).toDF("time", "value", "id") - - checkAnswer( - df.groupBy(window($"time", "10 seconds", "3 seconds", "0 second")) - .agg(count("*").as("counts")) - .orderBy($"window.start".asc) - .select($"window.start".cast("string"), $"window.end".cast("string"), $"counts"), - // 2016-03-27 19:39:27 UTC -> 4 bins - // 2016-03-27 19:39:34 UTC -> 3 bins - // 2016-03-27 19:39:56 UTC -> 3 bins - Seq( - Row("2016-03-27 19:39:18", "2016-03-27 19:39:28", 1), - Row("2016-03-27 19:39:21", "2016-03-27 19:39:31", 1), - Row("2016-03-27 19:39:24", "2016-03-27 19:39:34", 1), - Row("2016-03-27 19:39:27", "2016-03-27 19:39:37", 2), - Row("2016-03-27 19:39:30", "2016-03-27 19:39:40", 1), - Row("2016-03-27 19:39:33", "2016-03-27 19:39:43", 1), - Row("2016-03-27 19:39:48", "2016-03-27 19:39:58", 1), - Row("2016-03-27 19:39:51", "2016-03-27 19:40:01", 1), - Row("2016-03-27 19:39:54", "2016-03-27 19:40:04", 1)) - ) - } - - test("sliding window projection") { - val df = Seq( + // In SPARK-21871, we added code to check the actual bytecode size of gen'd methods. If the size + // goes over `hugeMethodLimit`, Spark fails to compile the methods and the execution also fails + // in a test mode. So, we explicitly turn off whole-stage codegen here. + // This guard can be removed if this issue fixed. + withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") { + val df = Seq( ("2016-03-27 19:39:34", 1, "a"), ("2016-03-27 19:39:56", 2, "a"), ("2016-03-27 19:39:27", 4, "b")).toDF("time", "value", "id") - .select(window($"time", "10 seconds", "3 seconds", "0 second"), $"value") - .orderBy($"window.start".asc, $"value".desc).select("value") - val expands = df.queryExecution.optimizedPlan.find(_.isInstanceOf[Expand]) - assert(expands.nonEmpty, "Sliding windows require expand") + checkAnswer( + df.groupBy(window($"time", "10 seconds", "3 seconds", "0 second")) + .agg(count("*").as("counts")) + .orderBy($"window.start".asc) + .select($"window.start".cast("string"), $"window.end".cast("string"), $"counts"), + // 2016-03-27 19:39:27 UTC -> 4 bins + // 2016-03-27 19:39:34 UTC -> 3 bins + // 2016-03-27 19:39:56 UTC -> 3 bins + Seq( + Row("2016-03-27 19:39:18", "2016-03-27 19:39:28", 1), + Row("2016-03-27 19:39:21", "2016-03-27 19:39:31", 1), + Row("2016-03-27 19:39:24", "2016-03-27 19:39:34", 1), + Row("2016-03-27 19:39:27", "2016-03-27 19:39:37", 2), + Row("2016-03-27 19:39:30", "2016-03-27 19:39:40", 1), + Row("2016-03-27 19:39:33", "2016-03-27 19:39:43", 1), + Row("2016-03-27 19:39:48", "2016-03-27 19:39:58", 1), + Row("2016-03-27 19:39:51", "2016-03-27 19:40:01", 1), + Row("2016-03-27 19:39:54", "2016-03-27 19:40:04", 1)) + ) + } + } - checkAnswer( - df, - // 2016-03-27 19:39:27 UTC -> 4 bins - // 2016-03-27 19:39:34 UTC -> 3 bins - // 2016-03-27 19:39:56 UTC -> 3 bins - Seq(Row(4), Row(4), Row(4), Row(4), Row(1), Row(1), Row(1), Row(2), Row(2), Row(2)) - ) + test("sliding window projection") { + // In SPARK-21871, we added code to check the actual bytecode size of gen'd methods. If the size + // goes over `hugeMethodLimit`, Spark fails to compile the methods and the execution also fails + // in a test mode. So, we explicitly turn off whole-stage codegen here. + // This guard can be removed if this issue fixed. + withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") { --- End diff -- The same here.
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org