[
https://issues.apache.org/jira/browse/SPARK-15059?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15267894#comment-15267894
]
Apache Spark commented on SPARK-15059:
--------------------------------------
User 'tankkyo' has created a pull request for this issue:
https://github.com/apache/spark/pull/12857
> Fine-grained class loader lock in ChildFirstURLClassLoader caused dead locks
> ----------------------------------------------------------------------------
>
> Key: SPARK-15059
> URL: https://issues.apache.org/jira/browse/SPARK-15059
> Project: Spark
> Issue Type: Bug
> Components: Spark Core
> Affects Versions: 1.5.1, 1.5.2
> Reporter: Zheng Tan
>
> Two threads in executors are locked by each other when loading the same class
> concurrently using ChildFirstURLClassLoader. This bug can be reproduced in
> our product environment(1.5.1).
> Here is our java thread dump information:
> Found one Java-level deadlock:
> =============================
> "Timer-3":
> waiting to lock monitor 0x00007f5a0c01f598 (object 0x00000005947cead0, a
> org.apache.spark.util.ChildFirstURLClassLoader),
> which is held by "pool-15-thread-7"
> "pool-15-thread-7":
> waiting to lock monitor 0x00007f5a0c01f4e8 ({color:red}object
> 0x000000059b5fe8d8, a java.lang.Object{color}),
> which is held by "pool-16-thread-5"
> "pool-16-thread-5":
> waiting to lock monitor 0x00007f5a0c01f598 ({color:red}object
> 0x00000005947cead0, a org.apache.spark.util.ChildFirstURLClassLoader{color}),
> which is held by "pool-15-thread-7"
> Java stack information for the threads listed above:
> ===================================================
> "Timer-3":
> at
> com.mchange.v2.resourcepool.BasicResourcePool.destroyResource(BasicResourcePool.java:980)
> at
> com.mchange.v2.resourcepool.BasicResourcePool.removeResource(BasicResourcePool.java:1406)
> at
> com.mchange.v2.resourcepool.BasicResourcePool.removeResource(BasicResourcePool.java:1378)
> at
> com.mchange.v2.resourcepool.BasicResourcePool.cullExpired(BasicResourcePool.java:1462)
> at
> com.mchange.v2.resourcepool.BasicResourcePool.access$1900(BasicResourcePool.java:32)
> at
> com.mchange.v2.resourcepool.BasicResourcePool$CullTask.run(BasicResourcePool.java:1937)
> - locked <0x000000059b5dd938> (a
> com.mchange.v2.resourcepool.BasicResourcePool)
> at java.util.TimerThread.mainLoop(Timer.java:555)
> at java.util.TimerThread.run(Timer.java:505)
> "pool-15-thread-7":
> at
> org.apache.spark.util.ChildFirstURLClassLoader.loadClass(MutableURLClassLoader.scala:71)
> - waiting to lock <0x000000059b5fe8d8> (a java.lang.Object)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> at java.lang.Class.forName0(Native Method)
> at java.lang.Class.forName(Class.java:270)
> at com.sun.beans.finder.ClassFinder.findClass(ClassFinder.java:67)
> at com.sun.beans.finder.ClassFinder.findClass(ClassFinder.java:110)
> at
> com.sun.beans.finder.InstanceFinder.instantiate(InstanceFinder.java:93)
> at com.sun.beans.finder.InstanceFinder.find(InstanceFinder.java:66)
> at java.beans.Introspector.findExplicitBeanInfo(Introspector.java:438)
> at java.beans.Introspector.<init>(Introspector.java:388)
> at java.beans.Introspector.getBeanInfo(Introspector.java:163)
> at java.beans.Introspector.getBeanInfo(Introspector.java:250)
> at java.beans.Introspector.<init>(Introspector.java:397)
> at java.beans.Introspector.getBeanInfo(Introspector.java:163)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getPropertyDescriptors(PropertyUtilsBean.java:971)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getPropertyDescriptors(PropertyUtilsBean.java:1075)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getPropertyDescriptor(PropertyUtilsBean.java:903)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getSimpleProperty(PropertyUtilsBean.java:1310)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getNestedProperty(PropertyUtilsBean.java:762)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getProperty(PropertyUtilsBean.java:837)
> at
> org.apache.commons.beanutils.PropertyUtils.getProperty(PropertyUtils.java:426)
> at
> com.hulu.recommendation.datastore.impl.table.Indexing.getKey(Indexing.java:255)
> at
> com.hulu.recommendation.datastore.impl.table.Indexing.indexIntoMap(Indexing.java:183)
> at
> com.hulu.recommendation.datastore.impl.table.Indexing.indexIntoMap(Indexing.java:179)
> at
> com.hulu.recommendation.datastore.impl.table.Indexing.createIndexInBatch(Indexing.java:86)
> at
> com.hulu.recommendation.datastore.impl.table.TableHolder.loadItemInBatch(TableHolder.java:268)
> at
> com.hulu.recommendation.datastore.impl.table.TableHolder.loadDataIncrementallyByFieldOrPk(TableHolder.java:214)
> at
> com.hulu.recommendation.datastore.impl.table.TableHolder.loadDataIncrementallyByFieldOrPk(TableHolder.java:153)
> at
> com.hulu.recommendation.datastore.impl.table.TableHolder.loadDataInitially(TableHolder.java:121)
> - locked <0x000000059b6864b8> (a
> com.hulu.recommendation.datastore.impl.table.TableHolder)
> at
> com.hulu.recommendation.datastore.impl.table.TableHolder.loadDataInitially(TableHolder.java:106)
> at
> com.hulu.recommendation.datastore.impl.table.TableHolder.loadData(TableHolder.java:369)
> at
> com.hulu.recommendation.datastore.BaseDataHolder.confirmDataLoaded(BaseDataHolder.java:25)
> - locked <0x000000059b6864b8> (a
> com.hulu.recommendation.datastore.impl.table.TableHolder)
> at
> com.hulu.recommendation.datastore.BaseDataHolder.initialize(BaseDataHolder.java:39)
> at
> com.hulu.recommendation.datastore.DataComponentUtils$1.call(DataComponentUtils.java:17)
> at
> com.hulu.recommendation.datastore.DataComponentUtils$1.call(DataComponentUtils.java:14)
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> "pool-16-thread-5":
> at java.lang.ClassLoader.loadClass(ClassLoader.java:405)
> - waiting to lock <0x00000005947cead0> (a
> org.apache.spark.util.ChildFirstURLClassLoader)
> at
> org.apache.spark.util.ChildFirstURLClassLoader.liftedTree1$1(MutableURLClassLoader.scala:72)
> at
> org.apache.spark.util.ChildFirstURLClassLoader.loadClass(MutableURLClassLoader.scala:71)
> - locked <0x000000059b5fe8d8> (a java.lang.Object)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:412)
> - locked <0x00000005948408d8> (a org.openejb.util.MemoryClassLoader)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> at java.lang.Class.forName0(Native Method)
> at java.lang.Class.forName(Class.java:270)
> at com.sun.beans.finder.ClassFinder.findClass(ClassFinder.java:103)
> at
> com.sun.beans.finder.InstanceFinder.instantiate(InstanceFinder.java:93)
> at com.sun.beans.finder.InstanceFinder.find(InstanceFinder.java:66)
> at java.beans.Introspector.findExplicitBeanInfo(Introspector.java:438)
> at java.beans.Introspector.<init>(Introspector.java:388)
> at java.beans.Introspector.getBeanInfo(Introspector.java:163)
> at java.beans.Introspector.getBeanInfo(Introspector.java:250)
> at java.beans.Introspector.<init>(Introspector.java:397)
> at java.beans.Introspector.getBeanInfo(Introspector.java:163)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getPropertyDescriptors(PropertyUtilsBean.java:971)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getPropertyDescriptors(PropertyUtilsBean.java:1075)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getPropertyDescriptor(PropertyUtilsBean.java:903)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getSimpleProperty(PropertyUtilsBean.java:1310)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getNestedProperty(PropertyUtilsBean.java:762)
> at
> org.apache.commons.beanutils.PropertyUtilsBean.getProperty(PropertyUtilsBean.java:837)
> at
> org.apache.commons.beanutils.PropertyUtils.getProperty(PropertyUtils.java:426)
> ......
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> Found 1 deadlock.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
