Weichen Xu created SPARK-18051:
----------------------------------
Summary: Custom PartitionCoalescer cause serialization exception
Key: SPARK-18051
URL: https://issues.apache.org/jira/browse/SPARK-18051
Project: Spark
Issue Type: Bug
Components: Spark Core
Affects Versions: 2.1.0
Reporter: Weichen Xu
for example, the following code cause exception:
{code: title=code}
class MyCoalescer extends PartitionCoalescer{
override def coalesce(maxPartitions: Int, parent: RDD[_]):
Array[PartitionGroup] = {
val pglist = Array.fill(2)(new PartitionGroup())
pglist(0).partitions.append(parent.partitions(0), parent.partitions(1),
parent.partitions(2))
pglist(1).partitions.append(parent.partitions(3), parent.partitions(4),
parent.partitions(5))
pglist
}
}
object Test1 {
def main(args: Array[String]) = {
val spark = SparkSession.builder().appName("test").getOrCreate()
val sc = spark.sparkContext
val rdd = sc.parallelize(1 to 6, 6)
rdd.coalesce(2, false, Some(new MyCoalescer)).count()
spark.stop()
}
}
{code}
it throws exception:
Exception in thread "dag-scheduler-event-loop" java.lang.StackOverflowError
at java.lang.Exception.<init>(Exception.java:102)
....
at
org.apache.spark.serializer.SerializationDebugger$SerializationDebugger.visitSerializableWithWriteObjectMethod(SerializationDebugger.scala:230)
at
org.apache.spark.serializer.SerializationDebugger$SerializationDebugger.visitSerializable(SerializationDebugger.scala:189)
at
org.apache.spark.serializer.SerializationDebugger$SerializationDebugger.visit(SerializationDebugger.scala:108)
at
org.apache.spark.serializer.SerializationDebugger$SerializationDebugger.visitSerializableWithWriteObjectMethod(SerializationDebugger.scala:243)
at
org.apache.spark.serializer.SerializationDebugger$SerializationDebugger.visitSerializable(SerializationDebugger.scala:189)
....
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
