This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 4a54107 [SPARK-26961][CORE] Enable parallel classloading capability
4a54107 is described below
commit 4a54107811f59ff345dec8729d282c3fa3ef94a2
Author: Ajith <ajith2...@gmail.com>
AuthorDate: Mon Mar 25 19:07:30 2019 -0500
[SPARK-26961][CORE] Enable parallel classloading capability
## What changes were proposed in this pull request?
As per https://docs.oracle.com/javase/8/docs/api/java/lang/ClassLoader.html
``Class loaders that support concurrent loading of classes are known as
parallel capable class loaders and are required to register themselves at their
class initialization time by invoking the ClassLoader.registerAsParallelCapable
method. Note that the ClassLoader class is registered as parallel capable by
default. However, its subclasses still need to register themselves if they are
parallel capable. ``
i.e we can have finer class loading locks by registering classloaders as
parallel capable. (Refer to deadlock due to macro lock
https://issues.apache.org/jira/browse/SPARK-26961).
All the classloaders we have are wrapper of URLClassLoader which by itself
is parallel capable.
But this cannot be achieved by scala code due to static registration Refer
https://github.com/scala/bug/issues/11429
## How was this patch tested?
All Existing UT must pass
Closes #24126 from ajithme/driverlock.
Authored-by: Ajith <ajith2...@gmail.com>
Signed-off-by: Sean Owen <sean.o...@databricks.com>
(cherry picked from commit b61dce23d2ee7ca95770bc7c390029aae8c65f7e)
Signed-off-by: Sean Owen <sean.o...@databricks.com>
---
.../spark/util/ChildFirstURLClassLoader.java | 68 ++++++++++++++++++++
.../apache/spark/util/MutableURLClassLoader.java} | 23 ++++---
.../org/apache/spark/util/ParentClassLoader.java} | 21 +++---
.../apache/spark/util/MutableURLClassLoader.scala | 75 ----------------------
.../internal/NonClosableMutableURLClassLoader.java | 24 +++----
.../apache/spark/sql/internal/SharedState.scala | 11 ----
6 files changed, 107 insertions(+), 115 deletions(-)
diff --git
a/core/src/main/java/org/apache/spark/util/ChildFirstURLClassLoader.java
b/core/src/main/java/org/apache/spark/util/ChildFirstURLClassLoader.java
new file mode 100644
index 0000000..57d9675
--- /dev/null
+++ b/core/src/main/java/org/apache/spark/util/ChildFirstURLClassLoader.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+
+/**
+ * A mutable class loader that gives preference to its own URLs over the
parent class loader
+ * when loading classes and resources.
+ */
+public class ChildFirstURLClassLoader extends MutableURLClassLoader {
+
+ static {
+ ClassLoader.registerAsParallelCapable();
+ }
+
+ private ParentClassLoader parent;
+
+ public ChildFirstURLClassLoader(URL[] urls, ClassLoader parent) {
+ super(urls, null);
+ this.parent = new ParentClassLoader(parent);
+ }
+
+ @Override
+ public Class<?> loadClass(String name, boolean resolve) throws
ClassNotFoundException {
+ try {
+ return super.loadClass(name, resolve);
+ } catch (ClassNotFoundException cnf) {
+ return parent.loadClass(name, resolve);
+ }
+ }
+
+ @Override
+ public Enumeration<URL> getResources(String name) throws IOException {
+ ArrayList<URL> urls = Collections.list(super.getResources(name));
+ urls.addAll(Collections.list(parent.getResources(name)));
+ return Collections.enumeration(urls);
+ }
+
+ @Override
+ public URL getResource(String name) {
+ URL url = super.getResource(name);
+ if (url != null) {
+ return url;
+ } else {
+ return parent.getResource(name);
+ }
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
b/core/src/main/java/org/apache/spark/util/MutableURLClassLoader.java
similarity index 64%
copy from core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
copy to core/src/main/java/org/apache/spark/util/MutableURLClassLoader.java
index c9b7493..a7c775d 100644
--- a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
+++ b/core/src/main/java/org/apache/spark/util/MutableURLClassLoader.java
@@ -15,23 +15,26 @@
* limitations under the License.
*/
-package org.apache.spark.util
+package org.apache.spark.util;
+
+import java.net.URL;
+import java.net.URLClassLoader;
/**
- * A class loader which makes some protected methods in ClassLoader accessible.
+ * URL class loader that exposes the `addURL` method in URLClassLoader.
*/
-private[spark] class ParentClassLoader(parent: ClassLoader) extends
ClassLoader(parent) {
+public class MutableURLClassLoader extends URLClassLoader {
- override def findClass(name: String): Class[_] = {
- super.findClass(name)
+ static {
+ ClassLoader.registerAsParallelCapable();
}
- override def loadClass(name: String): Class[_] = {
- super.loadClass(name)
+ public MutableURLClassLoader(URL[] urls, ClassLoader parent) {
+ super(urls, parent);
}
- override def loadClass(name: String, resolve: Boolean): Class[_] = {
- super.loadClass(name, resolve)
+ @Override
+ public void addURL(URL url) {
+ super.addURL(url);
}
-
}
diff --git a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
b/core/src/main/java/org/apache/spark/util/ParentClassLoader.java
similarity index 65%
copy from core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
copy to core/src/main/java/org/apache/spark/util/ParentClassLoader.java
index c9b7493..094005c 100644
--- a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
+++ b/core/src/main/java/org/apache/spark/util/ParentClassLoader.java
@@ -15,23 +15,28 @@
* limitations under the License.
*/
-package org.apache.spark.util
+package org.apache.spark.util;
/**
* A class loader which makes some protected methods in ClassLoader accessible.
*/
-private[spark] class ParentClassLoader(parent: ClassLoader) extends
ClassLoader(parent) {
+public class ParentClassLoader extends ClassLoader {
- override def findClass(name: String): Class[_] = {
- super.findClass(name)
+ static {
+ ClassLoader.registerAsParallelCapable();
}
- override def loadClass(name: String): Class[_] = {
- super.loadClass(name)
+ public ParentClassLoader(ClassLoader parent) {
+ super(parent);
}
- override def loadClass(name: String, resolve: Boolean): Class[_] = {
- super.loadClass(name, resolve)
+ @Override
+ public Class<?> findClass(String name) throws ClassNotFoundException {
+ return super.findClass(name);
}
+ @Override
+ public Class<?> loadClass(String name, boolean resolve) throws
ClassNotFoundException {
+ return super.loadClass(name, resolve);
+ }
}
diff --git
a/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala
b/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala
deleted file mode 100644
index 034826c..0000000
--- a/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.util
-
-import java.net.{URL, URLClassLoader}
-import java.util.Enumeration
-
-import scala.collection.JavaConverters._
-
-/**
- * URL class loader that exposes the `addURL` and `getURLs` methods in
URLClassLoader.
- */
-private[spark] class MutableURLClassLoader(urls: Array[URL], parent:
ClassLoader)
- extends URLClassLoader(urls, parent) {
-
- override def addURL(url: URL): Unit = {
- super.addURL(url)
- }
-
- override def getURLs(): Array[URL] = {
- super.getURLs()
- }
-
-}
-
-/**
- * A mutable class loader that gives preference to its own URLs over the
parent class loader
- * when loading classes and resources.
- */
-private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent:
ClassLoader)
- extends MutableURLClassLoader(urls, null) {
-
- private val parentClassLoader = new ParentClassLoader(parent)
-
- override def loadClass(name: String, resolve: Boolean): Class[_] = {
- try {
- super.loadClass(name, resolve)
- } catch {
- case e: ClassNotFoundException =>
- parentClassLoader.loadClass(name, resolve)
- }
- }
-
- override def getResource(name: String): URL = {
- val url = super.findResource(name)
- val res = if (url != null) url else parentClassLoader.getResource(name)
- res
- }
-
- override def getResources(name: String): Enumeration[URL] = {
- val childUrls = super.findResources(name).asScala
- val parentUrls = parentClassLoader.getResources(name).asScala
- (childUrls ++ parentUrls).asJavaEnumeration
- }
-
- override def addURL(url: URL) {
- super.addURL(url)
- }
-
-}
diff --git a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
b/sql/core/src/main/java/org/apache/spark/sql/internal/NonClosableMutableURLClassLoader.java
similarity index 63%
rename from core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
rename to
sql/core/src/main/java/org/apache/spark/sql/internal/NonClosableMutableURLClassLoader.java
index c9b7493..db77d91 100644
--- a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
+++
b/sql/core/src/main/java/org/apache/spark/sql/internal/NonClosableMutableURLClassLoader.java
@@ -15,23 +15,25 @@
* limitations under the License.
*/
-package org.apache.spark.util
+package org.apache.spark.sql.internal;
+
+import java.net.URL;
+
+import org.apache.spark.util.MutableURLClassLoader;
/**
- * A class loader which makes some protected methods in ClassLoader accessible.
+ * This class loader cannot be closed (its `close` method is a no-op).
*/
-private[spark] class ParentClassLoader(parent: ClassLoader) extends
ClassLoader(parent) {
-
- override def findClass(name: String): Class[_] = {
- super.findClass(name)
- }
+public class NonClosableMutableURLClassLoader extends MutableURLClassLoader {
- override def loadClass(name: String): Class[_] = {
- super.loadClass(name)
+ static {
+ ClassLoader.registerAsParallelCapable();
}
- override def loadClass(name: String, resolve: Boolean): Class[_] = {
- super.loadClass(name, resolve)
+ public NonClosableMutableURLClassLoader(ClassLoader parent) {
+ super(new URL[]{}, parent);
}
+ @Override
+ public void close() {}
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index baea4ce..4afcb25 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -190,14 +190,3 @@ object SharedState extends Logging {
}
}
}
-
-
-/**
- * URL class loader that exposes the `addURL` and `getURLs` methods in
URLClassLoader.
- * This class loader cannot be closed (its `close` method is a no-op).
- */
-private[sql] class NonClosableMutableURLClassLoader(parent: ClassLoader)
- extends MutableURLClassLoader(Array.empty, parent) {
-
- override def close(): Unit = {}
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
