[jira] [Updated] (SPARK-20666) Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError
[ https://issues.apache.org/jira/browse/SPARK-20666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Shixiong Zhu updated SPARK-20666: - Fix Version/s: (was: 2.2.1) (was: 2.3.0) 2.2.0 > Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError > --- > > Key: SPARK-20666 > URL: https://issues.apache.org/jira/browse/SPARK-20666 > Project: Spark > Issue Type: Bug > Components: ML, Spark Core, SparkR >Affects Versions: 2.2.0 >Reporter: Felix Cheung >Assignee: Wenchen Fan >Priority: Critical > Fix For: 2.2.0 > > > seeing quite a bit of this on AppVeyor, aka Windows only,-> seems like in > other test runs too, always only when running ML tests, it seems > {code} > Exception in thread "SparkListenerBus" java.lang.IllegalAccessError: > Attempted to access garbage collected accumulator 159454 > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:265) > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:261) > at scala.Option.map(Option.scala:146) > at > org.apache.spark.util.AccumulatorContext$.get(AccumulatorV2.scala:261) > at org.apache.spark.util.AccumulatorV2.name(AccumulatorV2.scala:88) > at > org.apache.spark.sql.execution.metric.SQLMetric.toInfo(SQLMetrics.scala:67) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.AbstractTraversable.map(Traversable.scala:104) > at > org.apache.spark.sql.execution.ui.SQLListener.onTaskEnd(SQLListener.scala:216) > at > org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:45) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:63) > at > org.apache.spark.scheduler.LiveListenerBus.postToAll(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(LiveListenerBus.scala:94) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(LiveListenerBus.scala:78) > at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1268) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1.run(LiveListenerBus.scala:77) > 1 > MLlib recommendation algorithms: Spark package found in SPARK_HOME: > C:\projects\spark\bin\.. > {code} > {code} > java.lang.IllegalStateException: SparkContext has been shutdown > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2015) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2044) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2063) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2088) > at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) > at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) > at org.apache.spark.rdd.RDD.collect(RDD.scala:935) > at > org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275) > at > org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2923) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at
[jira] [Updated] (SPARK-20666) Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError
[ https://issues.apache.org/jira/browse/SPARK-20666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Shixiong Zhu updated SPARK-20666: - Affects Version/s: (was: 2.3.0) 2.2.0 > Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError > --- > > Key: SPARK-20666 > URL: https://issues.apache.org/jira/browse/SPARK-20666 > Project: Spark > Issue Type: Bug > Components: ML, Spark Core, SparkR >Affects Versions: 2.2.0 >Reporter: Felix Cheung >Assignee: Wenchen Fan >Priority: Critical > Fix For: 2.2.0 > > > seeing quite a bit of this on AppVeyor, aka Windows only,-> seems like in > other test runs too, always only when running ML tests, it seems > {code} > Exception in thread "SparkListenerBus" java.lang.IllegalAccessError: > Attempted to access garbage collected accumulator 159454 > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:265) > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:261) > at scala.Option.map(Option.scala:146) > at > org.apache.spark.util.AccumulatorContext$.get(AccumulatorV2.scala:261) > at org.apache.spark.util.AccumulatorV2.name(AccumulatorV2.scala:88) > at > org.apache.spark.sql.execution.metric.SQLMetric.toInfo(SQLMetrics.scala:67) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.AbstractTraversable.map(Traversable.scala:104) > at > org.apache.spark.sql.execution.ui.SQLListener.onTaskEnd(SQLListener.scala:216) > at > org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:45) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:63) > at > org.apache.spark.scheduler.LiveListenerBus.postToAll(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(LiveListenerBus.scala:94) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(LiveListenerBus.scala:78) > at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1268) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1.run(LiveListenerBus.scala:77) > 1 > MLlib recommendation algorithms: Spark package found in SPARK_HOME: > C:\projects\spark\bin\.. > {code} > {code} > java.lang.IllegalStateException: SparkContext has been shutdown > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2015) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2044) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2063) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2088) > at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) > at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) > at org.apache.spark.rdd.RDD.collect(RDD.scala:935) > at > org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275) > at > org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2923) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at org.apache.spark.sql.Dataset$$anonfun$57.apply(Dataset.scala:2907) > at >
[jira] [Updated] (SPARK-20666) Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError
[ https://issues.apache.org/jira/browse/SPARK-20666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Michael Armbrust updated SPARK-20666: - Target Version/s: 2.2.0 > Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError > --- > > Key: SPARK-20666 > URL: https://issues.apache.org/jira/browse/SPARK-20666 > Project: Spark > Issue Type: Bug > Components: ML, Spark Core, SparkR >Affects Versions: 2.3.0 >Reporter: Felix Cheung >Priority: Critical > > seeing quite a bit of this on AppVeyor, aka Windows only,-> seems like in > other test runs too, always only when running ML tests, it seems > {code} > Exception in thread "SparkListenerBus" java.lang.IllegalAccessError: > Attempted to access garbage collected accumulator 159454 > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:265) > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:261) > at scala.Option.map(Option.scala:146) > at > org.apache.spark.util.AccumulatorContext$.get(AccumulatorV2.scala:261) > at org.apache.spark.util.AccumulatorV2.name(AccumulatorV2.scala:88) > at > org.apache.spark.sql.execution.metric.SQLMetric.toInfo(SQLMetrics.scala:67) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.AbstractTraversable.map(Traversable.scala:104) > at > org.apache.spark.sql.execution.ui.SQLListener.onTaskEnd(SQLListener.scala:216) > at > org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:45) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:63) > at > org.apache.spark.scheduler.LiveListenerBus.postToAll(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(LiveListenerBus.scala:94) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(LiveListenerBus.scala:78) > at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1268) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1.run(LiveListenerBus.scala:77) > 1 > MLlib recommendation algorithms: Spark package found in SPARK_HOME: > C:\projects\spark\bin\.. > {code} > {code} > java.lang.IllegalStateException: SparkContext has been shutdown > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2015) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2044) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2063) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2088) > at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) > at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) > at org.apache.spark.rdd.RDD.collect(RDD.scala:935) > at > org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275) > at > org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2923) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at org.apache.spark.sql.Dataset$$anonfun$57.apply(Dataset.scala:2907) > at > org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:65) > at
[jira] [Updated] (SPARK-20666) Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError
[ https://issues.apache.org/jira/browse/SPARK-20666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Felix Cheung updated SPARK-20666: - Description: seeing quite a bit of this on AppVeyor, aka Windows only,-> seems like in other test runs too, always only when running ML tests, it seems {code} Exception in thread "SparkListenerBus" java.lang.IllegalAccessError: Attempted to access garbage collected accumulator 159454 at org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:265) at org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:261) at scala.Option.map(Option.scala:146) at org.apache.spark.util.AccumulatorContext$.get(AccumulatorV2.scala:261) at org.apache.spark.util.AccumulatorV2.name(AccumulatorV2.scala:88) at org.apache.spark.sql.execution.metric.SQLMetric.toInfo(SQLMetrics.scala:67) at org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) at org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at org.apache.spark.sql.execution.ui.SQLListener.onTaskEnd(SQLListener.scala:216) at org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:45) at org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) at org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) at org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:63) at org.apache.spark.scheduler.LiveListenerBus.postToAll(LiveListenerBus.scala:36) at org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(LiveListenerBus.scala:94) at org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) at org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58) at org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(LiveListenerBus.scala:78) at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1268) at org.apache.spark.scheduler.LiveListenerBus$$anon$1.run(LiveListenerBus.scala:77) 1 MLlib recommendation algorithms: Spark package found in SPARK_HOME: C:\projects\spark\bin\.. {code} {code} java.lang.IllegalStateException: SparkContext has been shutdown at org.apache.spark.SparkContext.runJob(SparkContext.scala:2015) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2044) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2063) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2088) at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) at org.apache.spark.rdd.RDD.collect(RDD.scala:935) at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275) at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2923) at org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) at org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) at org.apache.spark.sql.Dataset$$anonfun$57.apply(Dataset.scala:2907) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:65) at org.apache.spark.sql.Dataset.withAction(Dataset.scala:2906) at org.apache.spark.sql.Dataset.collect(Dataset.scala:2474) at org.apache.spark.sql.api.r.SQLUtils$.dfToCols(SQLUtils.scala:173) at org.apache.spark.sql.api.r.SQLUtils.dfToCols(SQLUtils.scala) at sun.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at
[jira] [Updated] (SPARK-20666) Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError
[ https://issues.apache.org/jira/browse/SPARK-20666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin updated SPARK-20666: --- Summary: Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError (was: Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError on Windows) > Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError > --- > > Key: SPARK-20666 > URL: https://issues.apache.org/jira/browse/SPARK-20666 > Project: Spark > Issue Type: Bug > Components: ML, Spark Core, SparkR >Affects Versions: 2.3.0 >Reporter: Felix Cheung >Priority: Critical > > seeing quite a bit of this on AppVeyor, aka Windows only, always only when > running ML tests, it seems > {code} > Exception in thread "SparkListenerBus" java.lang.IllegalAccessError: > Attempted to access garbage collected accumulator 159454 > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:265) > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:261) > at scala.Option.map(Option.scala:146) > at > org.apache.spark.util.AccumulatorContext$.get(AccumulatorV2.scala:261) > at org.apache.spark.util.AccumulatorV2.name(AccumulatorV2.scala:88) > at > org.apache.spark.sql.execution.metric.SQLMetric.toInfo(SQLMetrics.scala:67) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.AbstractTraversable.map(Traversable.scala:104) > at > org.apache.spark.sql.execution.ui.SQLListener.onTaskEnd(SQLListener.scala:216) > at > org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:45) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:63) > at > org.apache.spark.scheduler.LiveListenerBus.postToAll(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(LiveListenerBus.scala:94) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(LiveListenerBus.scala:78) > at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1268) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1.run(LiveListenerBus.scala:77) > 1 > MLlib recommendation algorithms: Spark package found in SPARK_HOME: > C:\projects\spark\bin\.. > {code} > {code} > java.lang.IllegalStateException: SparkContext has been shutdown > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2015) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2044) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2063) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2088) > at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) > at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) > at org.apache.spark.rdd.RDD.collect(RDD.scala:935) > at > org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275) > at > org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2923) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at
[jira] [Updated] (SPARK-20666) Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError on Windows
[ https://issues.apache.org/jira/browse/SPARK-20666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin updated SPARK-20666: --- Priority: Critical (was: Major) > Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError > on Windows > -- > > Key: SPARK-20666 > URL: https://issues.apache.org/jira/browse/SPARK-20666 > Project: Spark > Issue Type: Bug > Components: ML, Spark Core, SparkR >Affects Versions: 2.3.0 >Reporter: Felix Cheung >Priority: Critical > > seeing quite a bit of this on AppVeyor, aka Windows only, always only when > running ML tests, it seems > {code} > Exception in thread "SparkListenerBus" java.lang.IllegalAccessError: > Attempted to access garbage collected accumulator 159454 > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:265) > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:261) > at scala.Option.map(Option.scala:146) > at > org.apache.spark.util.AccumulatorContext$.get(AccumulatorV2.scala:261) > at org.apache.spark.util.AccumulatorV2.name(AccumulatorV2.scala:88) > at > org.apache.spark.sql.execution.metric.SQLMetric.toInfo(SQLMetrics.scala:67) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.AbstractTraversable.map(Traversable.scala:104) > at > org.apache.spark.sql.execution.ui.SQLListener.onTaskEnd(SQLListener.scala:216) > at > org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:45) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:63) > at > org.apache.spark.scheduler.LiveListenerBus.postToAll(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(LiveListenerBus.scala:94) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(LiveListenerBus.scala:78) > at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1268) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1.run(LiveListenerBus.scala:77) > 1 > MLlib recommendation algorithms: Spark package found in SPARK_HOME: > C:\projects\spark\bin\.. > {code} > {code} > java.lang.IllegalStateException: SparkContext has been shutdown > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2015) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2044) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2063) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2088) > at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) > at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) > at org.apache.spark.rdd.RDD.collect(RDD.scala:935) > at > org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275) > at > org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2923) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at org.apache.spark.sql.Dataset$$anonfun$57.apply(Dataset.scala:2907) > at > org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:65) > at
[jira] [Updated] (SPARK-20666) Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError on Windows
[ https://issues.apache.org/jira/browse/SPARK-20666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Felix Cheung updated SPARK-20666: - Summary: Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError on Windows (was: Flaky test - SparkListenerBus randomly failing on Windows) > Flaky test - SparkListenerBus randomly failing java.lang.IllegalAccessError > on Windows > -- > > Key: SPARK-20666 > URL: https://issues.apache.org/jira/browse/SPARK-20666 > Project: Spark > Issue Type: Bug > Components: ML, Spark Core, SparkR >Affects Versions: 2.3.0 >Reporter: Felix Cheung > > seeing quite a bit of this on AppVeyor, aka Windows only, always only when > running ML tests, it seems > {code} > Exception in thread "SparkListenerBus" java.lang.IllegalAccessError: > Attempted to access garbage collected accumulator 159454 > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:265) > at > org.apache.spark.util.AccumulatorContext$$anonfun$get$1.apply(AccumulatorV2.scala:261) > at scala.Option.map(Option.scala:146) > at > org.apache.spark.util.AccumulatorContext$.get(AccumulatorV2.scala:261) > at org.apache.spark.util.AccumulatorV2.name(AccumulatorV2.scala:88) > at > org.apache.spark.sql.execution.metric.SQLMetric.toInfo(SQLMetrics.scala:67) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > org.apache.spark.sql.execution.ui.SQLListener$$anonfun$onTaskEnd$1.apply(SQLListener.scala:216) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.AbstractTraversable.map(Traversable.scala:104) > at > org.apache.spark.sql.execution.ui.SQLListener.onTaskEnd(SQLListener.scala:216) > at > org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:45) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus.doPostEvent(LiveListenerBus.scala:36) > at > org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:63) > at > org.apache.spark.scheduler.LiveListenerBus.postToAll(LiveListenerBus.scala:36) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(LiveListenerBus.scala:94) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(LiveListenerBus.scala:79) > at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(LiveListenerBus.scala:78) > at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1268) > at > org.apache.spark.scheduler.LiveListenerBus$$anon$1.run(LiveListenerBus.scala:77) > 1 > MLlib recommendation algorithms: Spark package found in SPARK_HOME: > C:\projects\spark\bin\.. > {code} > {code} > java.lang.IllegalStateException: SparkContext has been shutdown > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2015) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2044) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2063) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2088) > at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) > at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) > at org.apache.spark.rdd.RDD.collect(RDD.scala:935) > at > org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275) > at > org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2923) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at > org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2474) > at org.apache.spark.sql.Dataset$$anonfun$57.apply(Dataset.scala:2907) > at >