This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new dc382136b1 [VL] Promote SharedLibraryLoader to SPI (#10774)
dc382136b1 is described below
commit dc382136b1059a26ead4b820a16b3b762f27961c
Author: Kent Yao <[email protected]>
AuthorDate: Wed Sep 24 16:01:56 2025 +0800
[VL] Promote SharedLibraryLoader to SPI (#10774)
* [VL] Promote SharedLibrayLoader to SPI
* throw
* style
---
.../org.apache.gluten.spi.SharedLibraryLoader | 26 ++++
.../backendsapi/velox/VeloxListenerApi.scala | 2 +-
.../apache/gluten/spi/SharedLibraryLoader.scala | 47 ++++++++
.../SharedLibraryLoaderCentos7.scala | 12 +-
.../SharedLibraryLoaderCentos8.scala | 12 +-
.../SharedLibraryLoaderCentos9.scala | 8 +-
.../SharedLibraryLoaderDebian11.scala | 8 +-
.../SharedLibraryLoaderDebian12.scala | 8 +-
.../gluten/spi/SharedLibraryLoaderMacOS.scala | 29 +++++
.../SharedLibraryLoaderOpenEuler2403.scala | 7 +-
.../SharedLibraryLoaderUbuntu2004.scala | 7 +-
.../SharedLibraryLoaderUbuntu2204.scala | 7 +-
.../apache/gluten/utils/SharedLibraryLoader.scala | 134 ---------------------
.../gluten/utils/SharedLibraryLoaderUtils.scala | 92 ++++++++++++++
.../org.apache.gluten.spi.SharedLibraryLoader | 18 +++
.../gluten/utils/MySharedLibraryLoader.scala | 43 +++++++
.../utils/SharedLibraryLoaderUtilsSuite.scala | 36 ++++++
17 files changed, 353 insertions(+), 143 deletions(-)
diff --git
a/backends-velox/src/main/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
b/backends-velox/src/main/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
new file mode 100644
index 0000000000..5e0a0fe939
--- /dev/null
+++
b/backends-velox/src/main/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.gluten.spi.SharedLibraryLoaderCentos7
+org.apache.gluten.spi.SharedLibraryLoaderCentos8
+org.apache.gluten.spi.SharedLibraryLoaderCentos9
+org.apache.gluten.spi.SharedLibraryLoaderDebian11
+org.apache.gluten.spi.SharedLibraryLoaderDebian12
+org.apache.gluten.spi.SharedLibraryLoaderMacOS
+org.apache.gluten.spi.SharedLibraryLoaderOpenEuler2403
+org.apache.gluten.spi.SharedLibraryLoaderUbuntu2004
+org.apache.gluten.spi.SharedLibraryLoaderUbuntu2204
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 5468ad2c56..3488ef5653 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -210,7 +210,7 @@ class VeloxListenerApi extends ListenerApi with Logging {
val loader = JniWorkspace.getDefault.libLoader
// Load shared native libraries the backend libraries depend on.
- SharedLibraryLoader.load(conf, loader)
+ SharedLibraryLoaderUtils.load(conf, loader)
// Load backend libraries.
val libPath = conf.get(GlutenConfig.GLUTEN_LIB_PATH)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoader.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoader.scala
new file mode 100755
index 0000000000..7841e67ed4
--- /dev/null
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoader.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.spi
+
+import org.apache.gluten.jni.JniLibLoader
+
+/**
+ * :: DeveloperApi ::
+ *
+ * Interface for loading shared libraries based on the operating system name
and version.
+ */
+trait SharedLibraryLoader {
+
+ /**
+ * Check if this loader can load libraries for the given OS name and version.
+ *
+ * @param osName
+ * OS name
+ * @param osVersion
+ * OS version
+ * @return
+ * true if this loader can load libraries for the given OS name and
version, false otherwise
+ */
+ def accepts(osName: String, osVersion: String): Boolean
+
+ /**
+ * Load the required shared libraries using the given JniLibLoader.
+ *
+ * @param loader
+ * JniLibLoader to load the shared libraries
+ */
+ def loadLib(loader: JniLibLoader): Unit
+}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
old mode 100755
new mode 100644
similarity index 80%
rename from
backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
rename to
backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
index 0ae9bbd5eb..2ee087d185
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
@@ -14,11 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.spi
import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ (osName.contains("CentOS") && osVersion.startsWith("7")) ||
+ (osName.contains("Oracle") && osVersion.startsWith("7")) ||
+ (osName.contains("Anolis") && osVersion.startsWith("7")) ||
+ (osName.contains("Red Hat") && osVersion.startsWith("7")) ||
+ (osName.contains("Alibaba Cloud Linux") && osVersion.startsWith("2")) ||
+ (osName.contains("tencentos") && osVersion.contains("2.4"))
+ }
+
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
@@ -37,4 +46,5 @@ class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
loader.loadAndCreateLink("liblz4.so.1", "liblz4.so")
loader.loadAndCreateLink("libgeos.so.3.10.7", "libgeos.so")
}
+
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
similarity index 82%
rename from
backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
rename to
backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
index f0882ea8f1..854777d774 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
@@ -14,11 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.spi
import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderCentos8 extends SharedLibraryLoader {
+
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ (osName.contains("CentOS") && osVersion.startsWith("8")) ||
+ (osName.contains("Oracle") && osVersion.startsWith("8")) ||
+ (osName.contains("Anolis") && osVersion.startsWith("8")) ||
+ (osName.contains("Red Hat") && osVersion.startsWith("8")) ||
+ (osName.contains("Alibaba Cloud Linux") && osVersion.startsWith("3")) ||
+ (osName.contains("tencentos") && osVersion.contains("3.2"))
+ }
+
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
similarity index 91%
rename from
backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
rename to
backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
index dea336cb36..f8268066c7 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
@@ -14,11 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.spi
import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderCentos9 extends SharedLibraryLoader {
+
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ osName.contains("CentOS") && osVersion.startsWith("9") ||
+ osName.contains("Red Hat") && osVersion.startsWith("9")
+ }
+
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
similarity index 93%
rename from
backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
rename to
backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
index 611ca9091f..9e51ddd34a 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
@@ -14,11 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.spi
import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderDebian11 extends SharedLibraryLoader {
+
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ osName.contains("Debian") && osVersion.startsWith("11")
+ }
+
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libicudata.so.67", "libicudata.so")
loader.loadAndCreateLink("libre2.so.9", "libre2.so")
@@ -45,4 +50,5 @@ class SharedLibraryLoaderDebian11 extends SharedLibraryLoader
{
loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
}
+
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
similarity index 94%
rename from
backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
rename to
backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
index 9f7650ed1f..d6b36ae9f3 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
@@ -14,11 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.spi
import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderDebian12 extends SharedLibraryLoader {
+
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ osName.contains("Debian") && osVersion.startsWith("12")
+ }
+
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libcrypto.so.3", "libcrypto.so")
loader.loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so")
@@ -51,4 +56,5 @@ class SharedLibraryLoaderDebian12 extends SharedLibraryLoader
{
loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
}
+
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderMacOS.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderMacOS.scala
new file mode 100644
index 0000000000..0305957905
--- /dev/null
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderMacOS.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.spi
+import org.apache.gluten.jni.JniLibLoader
+
+class SharedLibraryLoaderMacOS extends SharedLibraryLoader {
+
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ osName.startsWith("Mac OS X") || osName.startsWith("macOS")
+ }
+
+ override def loadLib(loader: JniLibLoader): Unit = {
+ // Placeholder for loading shared libs on MacOS if user needs.
+ }
+}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderOpenEuler2403.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
similarity index 93%
rename from
backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderOpenEuler2403.scala
rename to
backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
index ebf5c95a7a..ca0db50d48 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderOpenEuler2403.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
@@ -14,11 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.spi
import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderOpenEuler2403 extends SharedLibraryLoader {
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ osName.contains("openEuler") && osVersion.startsWith("24.03")
+ }
+
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
@@ -42,4 +46,5 @@ class SharedLibraryLoaderOpenEuler2403 extends
SharedLibraryLoader {
loader.loadAndCreateLink("libre2.so.11", "libre2.so")
loader.loadAndCreateLink("libsodium.so.26", "libsodium.so")
}
+
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2004.scala
similarity index 95%
rename from
backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
rename to
backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2004.scala
index 6f463f3427..7aaa07853e 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2004.scala
@@ -14,11 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.spi
import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader {
+
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ osName.contains("Ubuntu") && osVersion.startsWith("20.04")
+ }
+
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libroken.so.18", "libroken.so")
loader.loadAndCreateLink("libasn1.so.8", "libasn1.so")
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
similarity index 93%
rename from
backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
rename to
backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
index 029ddf45fa..a0535f188b 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
@@ -14,11 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.spi
import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader {
+
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ osName.contains("Ubuntu") && osVersion.startsWith("22.04")
+ }
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
loader.loadAndCreateLink("libicudata.so.70", "libicudata.so")
@@ -46,4 +50,5 @@ class SharedLibraryLoaderUbuntu2204 extends
SharedLibraryLoader {
loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so")
loader.loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so")
}
+
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala
deleted file mode 100755
index cb0c9cd891..0000000000
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.utils
-
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.exception.GlutenException
-import org.apache.gluten.jni.JniLibLoader
-
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.internal.SparkConfigUtil._
-
-import scala.sys.process._
-
-trait SharedLibraryLoader {
- def loadLib(loader: JniLibLoader): Unit
-}
-
-object SharedLibraryLoader {
- def load(conf: SparkConf, jni: JniLibLoader): Unit = {
- val shouldLoad = conf.get(GlutenConfig.GLUTEN_LOAD_LIB_FROM_JAR)
- if (!shouldLoad) {
- return
- }
- val osName = System.getProperty("os.name")
- if (osName.startsWith("Mac OS X") || osName.startsWith("macOS")) {
- loadLibWithMacOS(jni)
- } else {
- loadLibWithLinux(conf, jni)
- }
- }
-
- private def loadLibWithLinux(conf: SparkConf, jni: JniLibLoader): Unit = {
- val loader = find(conf)
- loader.loadLib(jni)
- }
-
- private def loadLibWithMacOS(jni: JniLibLoader): Unit = {
- // Placeholder for loading shared libs on MacOS if user needs.
- }
-
- private def find(conf: SparkConf): SharedLibraryLoader = {
- val systemName = conf.get(GlutenConfig.GLUTEN_LOAD_LIB_OS)
- val loader = if (systemName.isDefined) {
- val systemVersion = conf.get(GlutenConfig.GLUTEN_LOAD_LIB_OS_VERSION)
- if (systemVersion.isEmpty) {
- throw new GlutenException(
- s"${GlutenConfig.GLUTEN_LOAD_LIB_OS_VERSION.key} must be specified
when specifies the " +
- s"${GlutenConfig.GLUTEN_LOAD_LIB_OS.key}")
- }
- getForOS(systemName.get, systemVersion.get, "")
- } else {
- val system = "cat /etc/os-release".!!
- val systemNamePattern = "^NAME=\"?(.*)\"?".r
- val systemVersionPattern = "^VERSION=\"?(.*)\"?".r
- val systemInfoLines = system.stripMargin.split("\n")
- val systemNamePattern(systemName) =
- systemInfoLines.find(_.startsWith("NAME=")).getOrElse("")
- val systemVersionPattern(systemVersion) =
- systemInfoLines.find(_.startsWith("VERSION=")).getOrElse("")
- if (systemName.isEmpty || systemVersion.isEmpty) {
- throw new GlutenException("Failed to get OS name and version info.")
- }
- getForOS(systemName, systemVersion, system)
- }
- loader
- }
-
- private def getForOS(
- systemName: String,
- systemVersion: String,
- system: String): SharedLibraryLoader = {
- if (systemName.contains("Ubuntu") && systemVersion.startsWith("20.04")) {
- new SharedLibraryLoaderUbuntu2004
- } else if (systemName.contains("Ubuntu") &&
systemVersion.startsWith("22.04")) {
- new SharedLibraryLoaderUbuntu2204
- } else if (systemName.contains("CentOS") && systemVersion.startsWith("9"))
{
- new SharedLibraryLoaderCentos9
- } else if (
- (systemName.contains("CentOS") || systemName.contains("Oracle"))
- && systemVersion.startsWith("8")
- ) {
- new SharedLibraryLoaderCentos8
- } else if (
- (systemName.contains("CentOS") || systemName.contains("Oracle"))
- && systemVersion.startsWith("7")
- ) {
- new SharedLibraryLoaderCentos7
- } else if (systemName.contains("openEuler") &&
systemVersion.startsWith("24.03")) {
- new SharedLibraryLoaderOpenEuler2403
- } else if (systemName.contains("Alibaba Cloud Linux") &&
systemVersion.startsWith("3")) {
- new SharedLibraryLoaderCentos8
- } else if (systemName.contains("Alibaba Cloud Linux") &&
systemVersion.startsWith("2")) {
- new SharedLibraryLoaderCentos7
- } else if (systemName.contains("Anolis") && systemVersion.startsWith("8"))
{
- new SharedLibraryLoaderCentos8
- } else if (systemName.contains("Anolis") && systemVersion.startsWith("7"))
{
- new SharedLibraryLoaderCentos7
- } else if (system.contains("tencentos") && system.contains("2.4")) {
- new SharedLibraryLoaderCentos7
- } else if (system.contains("tencentos") && system.contains("3.2")) {
- new SharedLibraryLoaderCentos8
- } else if (systemName.contains("Red Hat") &&
systemVersion.startsWith("9")) {
- new SharedLibraryLoaderCentos9
- } else if (systemName.contains("Red Hat") &&
systemVersion.startsWith("8")) {
- new SharedLibraryLoaderCentos8
- } else if (systemName.contains("Red Hat") &&
systemVersion.startsWith("7")) {
- new SharedLibraryLoaderCentos7
- } else if (systemName.contains("Debian") &&
systemVersion.startsWith("11")) {
- new SharedLibraryLoaderDebian11
- } else if (systemName.contains("Debian") &&
systemVersion.startsWith("12")) {
- new SharedLibraryLoaderDebian12
- } else {
- throw new GlutenException(
- s"Found unsupported OS($systemName, $systemVersion)! Currently,
Gluten's Velox backend" +
- " only supports Ubuntu 20.04/22.04, CentOS 7/8, Oracle 7/8,
openEuler 24.03, " +
- "Alibaba Cloud Linux 2/3 & Anolis 7/8, tencentos 2.4/3.2, RedHat
7/8, " +
- "Debian 11/12.")
- }
- }
-}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUtils.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUtils.scala
new file mode 100755
index 0000000000..a66a351d9e
--- /dev/null
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUtils.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils
+
+import org.apache.gluten.config.GlutenConfig._
+import org.apache.gluten.exception.GlutenException
+import org.apache.gluten.jni.JniLibLoader
+import org.apache.gluten.spi.SharedLibraryLoader
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.internal.SparkConfigUtil._
+
+import java.io.FileInputStream
+import java.util.{Properties, ServiceLoader}
+
+import scala.collection.JavaConverters._
+
+object SharedLibraryLoaderUtils {
+ private def isMacOS: Boolean = {
+ val osName = System.getProperty("os.name")
+ osName.startsWith("Mac OS X") || osName.startsWith("macOS")
+ }
+
+ def load(conf: SparkConf, jni: JniLibLoader): Unit = {
+ val shouldLoad = conf.get(GLUTEN_LOAD_LIB_FROM_JAR)
+ if (!shouldLoad) {
+ return
+ }
+
+ val (osName, osVersion) = conf.get(GLUTEN_LOAD_LIB_OS) match {
+ case Some(os) =>
+ (
+ os,
+ conf
+ .get(GLUTEN_LOAD_LIB_OS_VERSION)
+ .getOrElse(
+ throw new GlutenException(
+ s"${GLUTEN_LOAD_LIB_OS_VERSION.key} must be specified when
specifies the " +
+ s"${GLUTEN_LOAD_LIB_OS.key}")))
+ case None if isMacOS =>
+ (System.getProperty("os.name"), System.getProperty("os.version"))
+ case None =>
+ val props = new Properties()
+ val in = new FileInputStream("/etc/os-release")
+ props.load(in)
+ (props.getProperty("NAME"), props.getProperty("VERSION"))
+ }
+
+ val loaders = ServiceLoader
+ .load(classOf[SharedLibraryLoader])
+ .asScala
+ .filter(loader => loader.accepts(osName, osVersion))
+ .toSeq
+
+ if (loaders.isEmpty) {
+ throw new GlutenException(
+ s"Cannot find SharedLibraryLoader for $osName $osVersion, please" +
+ "check whether your custom SharedLibraryLoader is implemented and
loadable.")
+ }
+
+ if (loaders.size > 1) {
+ throw new GlutenException(
+ s"Found more than one SharedLibraryLoader for $osName $osVersion:" +
+ s" ${loaders.mkString(",")}, " +
+ "please check whether your custom SharedLibraryLoader is implemented
correctly.")
+ }
+
+ val loader = loaders.head
+ try {
+ loader.loadLib(jni)
+ } catch {
+ case e: Throwable =>
+ throw new GlutenException(
+ s"Failed to load shared libraries for $osName $osVersion using
$loader",
+ e)
+ }
+ }
+}
diff --git
a/backends-velox/src/test/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
b/backends-velox/src/test/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
new file mode 100644
index 0000000000..0b3ef55dda
--- /dev/null
+++
b/backends-velox/src/test/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.gluten.utils.MySharedLibraryLoader
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/utils/MySharedLibraryLoader.scala
b/backends-velox/src/test/scala/org/apache/gluten/utils/MySharedLibraryLoader.scala
new file mode 100644
index 0000000000..c2902d652c
--- /dev/null
+++
b/backends-velox/src/test/scala/org/apache/gluten/utils/MySharedLibraryLoader.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils
+
+import org.apache.gluten.jni.JniLibLoader
+import org.apache.gluten.spi.SharedLibraryLoader
+import org.apache.gluten.utils.MySharedLibraryLoader.LOADED
+
+import java.util.concurrent.atomic.AtomicBoolean
+
+class MySharedLibraryLoader extends SharedLibraryLoader {
+ override def accepts(osName: String, osVersion: String): Boolean = {
+ osName == "My OS" && osVersion == "1.0"
+ }
+
+ /**
+ * Load the required shared libraries using the given JniLibLoader.
+ *
+ * @param loader
+ * JniLibLoader to load the shared libraries
+ */
+ override def loadLib(loader: JniLibLoader): Unit = {
+ LOADED.set(true)
+ }
+}
+
+object MySharedLibraryLoader {
+ val LOADED = new AtomicBoolean(false)
+}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/utils/SharedLibraryLoaderUtilsSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/utils/SharedLibraryLoaderUtilsSuite.scala
new file mode 100644
index 0000000000..c0c16eff74
--- /dev/null
+++
b/backends-velox/src/test/scala/org/apache/gluten/utils/SharedLibraryLoaderUtilsSuite.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils
+
+import org.apache.gluten.config.GlutenConfig.{GLUTEN_LOAD_LIB_FROM_JAR,
GLUTEN_LOAD_LIB_OS, GLUTEN_LOAD_LIB_OS_VERSION}
+
+import org.apache.spark.SparkConf
+
+import org.scalatest.funsuite.AnyFunSuite
+
+class SharedLibraryLoaderUtilsSuite extends AnyFunSuite {
+
+ test("Load SharedLibraryLoader with SPI") {
+ val sparkConf = new SparkConf()
+ .set(GLUTEN_LOAD_LIB_FROM_JAR.key, "true")
+ .set(GLUTEN_LOAD_LIB_OS_VERSION.key, "1.0")
+ .set(GLUTEN_LOAD_LIB_OS.key, "My OS")
+
+ SharedLibraryLoaderUtils.load(sparkConf, null)
+ assert(MySharedLibraryLoader.LOADED.get())
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
