Romi Kuntsman created SPARK-11229:
-------------------------------------
Summary: NPE in JoinedRow.isNullAt when
spark.shuffle.memoryFraction=0
Key: SPARK-11229
URL: https://issues.apache.org/jira/browse/SPARK-11229
Project: Spark
Issue Type: Bug
Components: SQL
Affects Versions: 1.5.1
Environment: 14.04.1-Ubuntu SMP x86_64 GNU/Linux
Reporter: Romi Kuntsman
Steps to reproduce:
1. set spark.shuffle.memoryFraction=0
2. load dataframe from parquet file
3. see it's read correctly by calling dataframe.show()
4. call dataframe.count()
Expected behaviour:
get count of rows in dataframe
OR, if memoryFraction=0 is an invalid setting, get notified about it
Actual behaviour:
CatalystReadSupport doesn't read the schema (even thought there is one) and
then there's a NullPointerException.
Driver stacktrace:
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270)
at
scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at
org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697)
at scala.Option.foreach(Option.scala:236)
at
org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
at
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1822)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1835)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1848)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1919)
at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:905)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
at org.apache.spark.rdd.RDD.collect(RDD.scala:904)
at
org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:177)
at
org.apache.spark.sql.DataFrame$$anonfun$collect$1.apply(DataFrame.scala:1385)
at
org.apache.spark.sql.DataFrame$$anonfun$collect$1.apply(DataFrame.scala:1385)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:56)
at
org.apache.spark.sql.DataFrame.withNewExecutionId(DataFrame.scala:1903)
at org.apache.spark.sql.DataFrame.collect(DataFrame.scala:1384)
at org.apache.spark.sql.DataFrame.count(DataFrame.scala:1402)
... 14 more
Caused by: java.lang.NullPointerException
at
org.apache.spark.sql.catalyst.expressions.JoinedRow.isNullAt(JoinedRow.scala:70)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificMutableProjection.apply(Unknown
Source)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator$$anonfun$generateProcessRow$1.apply(TungstenAggregationIterator.scala:194)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator$$anonfun$generateProcessRow$1.apply(TungstenAggregationIterator.scala:192)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator.processInputs(TungstenAggregationIterator.scala:368)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator.start(TungstenAggregationIterator.scala:622)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregate$$anonfun$doExecute$1.org$apache$spark$sql$execution$aggregate$TungstenAggregate$$anonfun$$executePartition$1(TungstenAggregate.scala:110)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregate$$anonfun$doExecute$1$$anonfun$2.apply(TungstenAggregate.scala:119)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregate$$anonfun$doExecute$1$$anonfun$2.apply(TungstenAggregate.scala:119)
at
org.apache.spark.rdd.MapPartitionsWithPreparationRDD.compute(MapPartitionsWithPreparationRDD.scala:64)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
at
org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
at
org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
at org.apache.spark.scheduler.Task.run(Task.scala:88)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Worker log:
15/10/21 12:06:36 INFO CoarseGrainedExecutorBackend: Got assigned task 2
15/10/21 12:06:36 INFO Executor: Running task 0.0 in stage 1.0 (TID 2)
15/10/21 12:06:36 INFO TorrentBroadcast: Started reading broadcast variable 2
15/10/21 12:06:36 INFO MemoryStore: ensureFreeSpace(5275) called with
curMem=69383, maxMem=2793500835
15/10/21 12:06:36 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in
memory (estimated size 5.2 KB, free 2.6 GB)
15/10/21 12:06:36 INFO TorrentBroadcast: Reading broadcast variable 2 took 9 ms
15/10/21 12:06:36 INFO MemoryStore: ensureFreeSpace(10432) called with
curMem=74658, maxMem=2793500835
15/10/21 12:06:36 INFO MemoryStore: Block broadcast_2 stored as values in
memory (estimated size 10.2 KB, free 2.6 GB)
15/10/21 12:06:36 INFO GenerateMutableProjection: Code generated in 70.404364 ms
15/10/21 12:06:36 INFO GenerateUnsafeProjection: Code generated in 6.235261 ms
15/10/21 12:06:36 INFO GenerateMutableProjection: Code generated in 10.861097 ms
15/10/21 12:06:36 INFO GenerateUnsafeRowJoiner: Code generated in 5.404177 ms
15/10/21 12:06:36 INFO GenerateUnsafeProjection: Code generated in 4.892669 ms
15/10/21 12:06:36 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1: Input
split: ParquetInputSplit{part: file:/home/user/parquet/part-r-00001.gz.parquet
start: 0 end: 178913 length: 178913 hosts: []}
15/10/21 12:06:36 INFO TorrentBroadcast: Started reading broadcast variable 1
15/10/21 12:06:36 INFO MemoryStore: ensureFreeSpace(15856) called with
curMem=85090, maxMem=2793500835
15/10/21 12:06:36 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in
memory (estimated size 15.5 KB, free 2.6 GB)
15/10/21 12:06:36 INFO TorrentBroadcast: Reading broadcast variable 1 took 9 ms
15/10/21 12:06:36 INFO MemoryStore: ensureFreeSpace(196360) called with
curMem=100946, maxMem=2793500835
15/10/21 12:06:36 INFO MemoryStore: Block broadcast_1 stored as values in
memory (estimated size 191.8 KB, free 2.6 GB)
15/10/21 12:06:36 INFO deprecation: mapred.min.split.size is deprecated.
Instead, use mapreduce.input.fileinputformat.split.minsize
15/10/21 12:06:36 WARN ParquetRecordReader: Can not initialize counter due to
context is not a instance of TaskInputOutputContext, but is
org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
15/10/21 12:06:36 INFO CatalystReadSupport: Going to read the following fields
from the Parquet file:
Parquet form:
message root {
}
Catalyst form:
StructType()
15/10/21 12:06:36 INFO InternalParquetRecordReader: RecordReader initialized
will read a total of 36833 records.
15/10/21 12:06:36 INFO InternalParquetRecordReader: at row 0. reading next block
15/10/21 12:06:36 INFO InternalParquetRecordReader: block read in memory in 2
ms. row count = 36833
15/10/21 12:06:36 ERROR Executor: Exception in task 0.0 in stage 1.0 (TID 2)
java.lang.NullPointerException
at
org.apache.spark.sql.catalyst.expressions.JoinedRow.isNullAt(JoinedRow.scala:70)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificMutableProjection.apply(Unknown
Source)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator$$anonfun$generateProcessRow$1.apply(TungstenAggregationIterator.scala:194)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator$$anonfun$generateProcessRow$1.apply(TungstenAggregationIterator.scala:192)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator.processInputs(TungstenAggregationIterator.scala:368)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator.start(TungstenAggregationIterator.scala:622)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregate$$anonfun$doExecute$1.org$apache$spark$sql$execution$aggregate$TungstenAggregate$$anonfun$$executePartition$1(TungstenAggregate.scala:110)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregate$$anonfun$doExecute$1$$anonfun$2.apply(TungstenAggregate.scala:119)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregate$$anonfun$doExecute$1$$anonfun$2.apply(TungstenAggregate.scala:119)
at
org.apache.spark.rdd.MapPartitionsWithPreparationRDD.compute(MapPartitionsWithPreparationRDD.scala:64)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
at
org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
at
org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
at org.apache.spark.scheduler.Task.run(Task.scala:88)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
