[ https://issues.apache.org/jira/browse/SPARK-17405?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Josh Rosen updated SPARK-17405: ------------------------------- Assignee: Eric Liang > Simple aggregation query OOMing after SPARK-16525 > ------------------------------------------------- > > Key: SPARK-17405 > URL: https://issues.apache.org/jira/browse/SPARK-17405 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 2.1.0 > Reporter: Josh Rosen > Assignee: Eric Liang > Priority: Blocker > > Prior to SPARK-16525 / https://github.com/apache/spark/pull/14176, the > following query ran fine via Beeline / Thrift Server and the Spark shell, but > after that patch it is consistently OOMING: > {code} > CREATE TEMPORARY VIEW table_1(double_col_1, boolean_col_2, timestamp_col_3, > smallint_col_4, boolean_col_5, int_col_6, timestamp_col_7, varchar0008_col_8, > int_col_9, string_col_10) AS ( > SELECT * FROM (VALUES > (CAST(-147.818640624 AS DOUBLE), CAST(NULL AS BOOLEAN), > TIMESTAMP('2012-10-19 00:00:00.0'), CAST(9 AS SMALLINT), false, 77, > TIMESTAMP('2014-07-01 00:00:00.0'), '-945', -646, '722'), > (CAST(594.195125271 AS DOUBLE), false, TIMESTAMP('2016-12-04 > 00:00:00.0'), CAST(NULL AS SMALLINT), CAST(NULL AS BOOLEAN), CAST(NULL AS > INT), TIMESTAMP('1999-12-26 00:00:00.0'), '250', -861, '55'), > (CAST(-454.171126363 AS DOUBLE), false, TIMESTAMP('2008-12-13 > 00:00:00.0'), CAST(NULL AS SMALLINT), false, -783, TIMESTAMP('2010-05-28 > 00:00:00.0'), '211', -959, CAST(NULL AS STRING)), > (CAST(437.670945524 AS DOUBLE), true, TIMESTAMP('2011-10-16 00:00:00.0'), > CAST(952 AS SMALLINT), true, 297, TIMESTAMP('2013-01-13 00:00:00.0'), '262', > CAST(NULL AS INT), '936'), > (CAST(-387.226759334 AS DOUBLE), false, TIMESTAMP('2019-10-03 > 00:00:00.0'), CAST(-496 AS SMALLINT), CAST(NULL AS BOOLEAN), -925, > TIMESTAMP('2028-06-27 00:00:00.0'), '-657', 948, '18'), > (CAST(-306.138230875 AS DOUBLE), true, TIMESTAMP('1997-10-07 > 00:00:00.0'), CAST(332 AS SMALLINT), false, 744, TIMESTAMP('1990-09-22 > 00:00:00.0'), '-345', 566, '-574'), > (CAST(675.402140308 AS DOUBLE), false, TIMESTAMP('2017-06-26 > 00:00:00.0'), CAST(972 AS SMALLINT), true, CAST(NULL AS INT), > TIMESTAMP('2026-06-10 00:00:00.0'), '518', 683, '-320'), > (CAST(734.839647174 AS DOUBLE), true, TIMESTAMP('1995-06-01 00:00:00.0'), > CAST(-792 AS SMALLINT), CAST(NULL AS BOOLEAN), CAST(NULL AS INT), > TIMESTAMP('2021-07-11 00:00:00.0'), '-318', 564, '142') > ) as t); > CREATE TEMPORARY VIEW table_3(string_col_1, float_col_2, timestamp_col_3, > boolean_col_4, timestamp_col_5, decimal3317_col_6) AS ( > SELECT * FROM (VALUES > ('88', CAST(191.92508 AS FLOAT), TIMESTAMP('1990-10-25 00:00:00.0'), > false, TIMESTAMP('1992-11-02 00:00:00.0'), CAST(NULL AS DECIMAL(33,17))), > ('-419', CAST(-13.477915 AS FLOAT), TIMESTAMP('1996-03-02 00:00:00.0'), > true, CAST(NULL AS TIMESTAMP), -653.51000000000000000BD), > ('970', CAST(-360.432 AS FLOAT), TIMESTAMP('2010-07-29 00:00:00.0'), > false, TIMESTAMP('1995-09-01 00:00:00.0'), -936.48000000000000000BD), > ('807', CAST(814.30756 AS FLOAT), TIMESTAMP('2019-11-06 00:00:00.0'), > false, TIMESTAMP('1996-04-25 00:00:00.0'), 335.56000000000000000BD), > ('-872', CAST(616.50525 AS FLOAT), TIMESTAMP('2011-08-28 00:00:00.0'), > false, TIMESTAMP('2003-07-19 00:00:00.0'), -951.18000000000000000BD), > ('-167', CAST(-875.35675 AS FLOAT), TIMESTAMP('1995-07-14 00:00:00.0'), > false, TIMESTAMP('2005-11-29 00:00:00.0'), 224.89000000000000000BD) > ) as t); > SELECT > CAST(MIN(t2.smallint_col_4) AS STRING) AS char_col, > LEAD(MAX((-387) + (727.64)), 90) OVER (PARTITION BY COALESCE(t2.int_col_9, > t2.smallint_col_4, t2.int_col_9) ORDER BY COALESCE(t2.int_col_9, > t2.smallint_col_4, t2.int_col_9) DESC, CAST(MIN(t2.smallint_col_4) AS > STRING)) AS decimal_col, > COALESCE(t2.int_col_9, t2.smallint_col_4, t2.int_col_9) AS int_col > FROM table_3 t1 > INNER JOIN table_1 t2 ON (((t2.timestamp_col_3) = (t1.timestamp_col_5)) AND > ((t2.string_col_10) = (t1.string_col_1))) AND ((t2.string_col_10) = > (t1.string_col_1)) > WHERE > (t2.smallint_col_4) IN (t2.int_col_9, t2.int_col_9) > GROUP BY > COALESCE(t2.int_col_9, t2.smallint_col_4, t2.int_col_9); > {code} > Here's the OOM: > {code} > org.apache.hive.service.cli.HiveSQLException: > org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in > stage 1.0 failed 1 times, most recent failure: Lost task 1.0 in stage 1.0 > (TID 9, localhost): java.lang.OutOfMemoryError: Unable to acquire 262144 > bytes of memory, got 0 > at > org.apache.spark.memory.MemoryConsumer.allocateArray(MemoryConsumer.java:100) > at > org.apache.spark.unsafe.map.BytesToBytesMap.allocate(BytesToBytesMap.java:783) > at > org.apache.spark.unsafe.map.BytesToBytesMap.<init>(BytesToBytesMap.java:204) > at > org.apache.spark.unsafe.map.BytesToBytesMap.<init>(BytesToBytesMap.java:219) > at > org.apache.spark.sql.execution.UnsafeFixedWidthAggregationMap.<init>(UnsafeFixedWidthAggregationMap.java:104) > at > org.apache.spark.sql.execution.aggregate.HashAggregateExec.createHashMap(HashAggregateExec.scala:305) > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.agg_doAggregateWithKeys$(Unknown > Source) > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.processNext(Unknown > Source) > at > org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) > at > org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$8$$anon$1.hasNext(WholeStageCodegenExec.scala:370) > at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) > at > org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:126) > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:79) > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:47) > at org.apache.spark.scheduler.Task.run(Task.scala:86) > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org