This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ef5f01cba [GLUTEN-6148][CORE] Simplify JniLibLoader loading mechanism
for native libraries (#6791)
ef5f01cba is described below
commit ef5f01cba5c6c7ddf57734ca685a458174fc9064
Author: Arnav Balyan <[email protected]>
AuthorDate: Wed Aug 14 06:15:49 2024 +0530
[GLUTEN-6148][CORE] Simplify JniLibLoader loading mechanism for native
libraries (#6791)
Closes #6148
---
.../gluten/utils/SharedLibraryLoaderCentos7.scala | 42 +--
.../gluten/utils/SharedLibraryLoaderCentos8.scala | 50 ++--
.../gluten/utils/SharedLibraryLoaderCentos9.scala | 50 ++--
.../gluten/utils/SharedLibraryLoaderDebian11.scala | 53 ++--
.../gluten/utils/SharedLibraryLoaderDebian12.scala | 65 +++--
.../utils/SharedLibraryLoaderUbuntu2004.scala | 85 +++---
.../utils/SharedLibraryLoaderUbuntu2204.scala | 55 ++--
.../org/apache/gluten/vectorized/JniLibLoader.java | 300 +++++++--------------
8 files changed, 294 insertions(+), 406 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
index 47ed2c47c..a7750dcb3 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
@@ -20,26 +20,26 @@ import org.apache.gluten.vectorized.JniLibLoader
class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader
- .newTransaction()
- .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so",
false)
- .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so",
false)
- .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so",
false)
- .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- .loadAndCreateLink("libboost_program_options.so.1.84.0",
"libboost_program_options.so", false)
- .loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
- .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so",
false)
- .loadAndCreateLink("libdouble-conversion.so.1",
"libdouble-conversion.so", false)
- .loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so", false)
- .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- .loadAndCreateLink("libglog.so.0", "libglog.so", false)
- .loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
- .loadAndCreateLink("libre2.so.10", "libre2.so", false)
- .loadAndCreateLink("libzstd.so.1", "libzstd.so", false)
- .loadAndCreateLink("liblz4.so.1", "liblz4.so", false)
- .commit()
+ loader.loadAndCreateLink("libboost_atomic.so.1.84.0",
"libboost_atomic.so", false)
+ loader.loadAndCreateLink("libboost_thread.so.1.84.0",
"libboost_thread.so", false)
+ loader.loadAndCreateLink("libboost_system.so.1.84.0",
"libboost_system.so", false)
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
+ loader.loadAndCreateLink(
+ "libboost_program_options.so.1.84.0",
+ "libboost_program_options.so",
+ false)
+ loader.loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
+ loader.loadAndCreateLink("libdouble-conversion.so.1",
"libdouble-conversion.so", false)
+ loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so", false)
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
+ loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+ loader.loadAndCreateLink("libre2.so.10", "libre2.so", false)
+ loader.loadAndCreateLink("libzstd.so.1", "libzstd.so", false)
+ loader.loadAndCreateLink("liblz4.so.1", "liblz4.so", false)
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
index c1d3bf2e2..bd8bf15be 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
@@ -20,30 +20,30 @@ import org.apache.gluten.vectorized.JniLibLoader
class SharedLibraryLoaderCentos8 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader
- .newTransaction()
- .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so",
false)
- .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so",
false)
- .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so",
false)
- .loadAndCreateLink("libicudata.so.60", "libicudata.so", false)
- .loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false)
- .loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false)
- .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- .loadAndCreateLink("libboost_program_options.so.1.84.0",
"libboost_program_options.so", false)
- .loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
- .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so",
false)
- .loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- .loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so", false)
- .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- .loadAndCreateLink("libglog.so.1", "libglog.so", false)
- .loadAndCreateLink("libdwarf.so.1", "libdwarf.so", false)
- .loadAndCreateLink("libidn.so.11", "libidn.so", false)
- .loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
- .loadAndCreateLink("libre2.so.0", "libre2.so", false)
- .loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
- .commit()
+ loader.loadAndCreateLink("libboost_atomic.so.1.84.0",
"libboost_atomic.so", false)
+ loader.loadAndCreateLink("libboost_thread.so.1.84.0",
"libboost_thread.so", false)
+ loader.loadAndCreateLink("libboost_system.so.1.84.0",
"libboost_system.so", false)
+ loader.loadAndCreateLink("libicudata.so.60", "libicudata.so", false)
+ loader.loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false)
+ loader.loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false)
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
+ loader.loadAndCreateLink(
+ "libboost_program_options.so.1.84.0",
+ "libboost_program_options.so",
+ false)
+ loader.loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
+ loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so", false)
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
+ loader.loadAndCreateLink("libglog.so.1", "libglog.so", false)
+ loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so", false)
+ loader.loadAndCreateLink("libidn.so.11", "libidn.so", false)
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+ loader.loadAndCreateLink("libre2.so.0", "libre2.so", false)
+ loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
index 2d9ececb3..06fb25c8b 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
@@ -20,30 +20,30 @@ import org.apache.gluten.vectorized.JniLibLoader
class SharedLibraryLoaderCentos9 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader
- .newTransaction()
- .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so",
false)
- .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so",
false)
- .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so",
false)
- .loadAndCreateLink("libicudata.so.67", "libicudata.so", false)
- .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false)
- .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false)
- .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- .loadAndCreateLink("libboost_program_options.so.1.84.0",
"libboost_program_options.so", false)
- .loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
- .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so",
false)
- .loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- .loadAndCreateLink("libglog.so.1", "libglog.so", false)
- .loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false)
- .loadAndCreateLink("libidn.so.12", "libidn.so", false)
- .loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
- .loadAndCreateLink("libre2.so.9", "libre2.so", false)
- .loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
- .commit()
+ loader.loadAndCreateLink("libboost_atomic.so.1.84.0",
"libboost_atomic.so", false)
+ loader.loadAndCreateLink("libboost_thread.so.1.84.0",
"libboost_thread.so", false)
+ loader.loadAndCreateLink("libboost_system.so.1.84.0",
"libboost_system.so", false)
+ loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false)
+ loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false)
+ loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false)
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
+ loader.loadAndCreateLink(
+ "libboost_program_options.so.1.84.0",
+ "libboost_program_options.so",
+ false)
+ loader.loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
+ loader.loadAndCreateLink("libglog.so.1", "libglog.so", false)
+ loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false)
+ loader.loadAndCreateLink("libidn.so.12", "libidn.so", false)
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+ loader.loadAndCreateLink("libre2.so.9", "libre2.so", false)
+ loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
index ca7d1d22d..a300cc5b9 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
@@ -20,33 +20,30 @@ import org.apache.gluten.vectorized.JniLibLoader
class SharedLibraryLoaderDebian11 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader
- .newTransaction()
- .loadAndCreateLink("libicudata.so.67", "libicudata.so", false)
- .loadAndCreateLink("libre2.so.9", "libre2.so", false)
- .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false)
- .loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false)
- .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
- .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false)
- .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false)
- .loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
- .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
- .loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
- .loadAndCreateLink("libssh2.so.1", "libssh2.so", false)
- .loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
- .loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false)
- .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
- .loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false)
- .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so",
false)
- .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- .loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- .loadAndCreateLink("libglog.so.0", "libglog.so", false)
- .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
- .loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
- .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
- .commit()
+ loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false)
+ loader.loadAndCreateLink("libre2.so.9", "libre2.so", false)
+ loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false)
+ loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false)
+ loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
+ loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so",
false)
+ loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false)
+ loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
+ loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
+ loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
+ loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false)
+ loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
+ loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false)
+ loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
+ loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false)
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
+ loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
+ loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
+ loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
index 128c8eaa2..8e24ef0ba 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
@@ -20,39 +20,36 @@ import org.apache.gluten.vectorized.JniLibLoader
class SharedLibraryLoaderDebian12 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader
- .newTransaction()
- .loadAndCreateLink("libcrypto.so.3", "libcrypto.so", false)
- .loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so", false)
- .loadAndCreateLink("libssl.so.3", "libssl.so", false)
- .loadAndCreateLink("libicudata.so.72", "libicudata.so", false)
- .loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so", false)
- .loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so", false)
- .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
- .loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so", false)
- .loadAndCreateLink("libicuuc.so.72", "libicuuc.so", false)
- .loadAndCreateLink("libkrb5.so.3", "libkrb5.so", false)
- .loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so", false)
- .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
- .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false)
- .loadAndCreateLink("libicui18n.so.72", "libicui18n.so", false)
- .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- .loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
- .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
- .loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
- .loadAndCreateLink("libssh2.so.1", "libssh2.so", false)
- .loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
- .loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so", false)
- .loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so", false)
- .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
- .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so",
false)
- .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- .loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- .loadAndCreateLink("libglog.so.1", "libglog.so", false)
- .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- .loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
- .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
- .commit()
+ loader.loadAndCreateLink("libcrypto.so.3", "libcrypto.so", false)
+ loader.loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so", false)
+ loader.loadAndCreateLink("libssl.so.3", "libssl.so", false)
+ loader.loadAndCreateLink("libicudata.so.72", "libicudata.so", false)
+ loader.loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so", false)
+ loader.loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so", false)
+ loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
+ loader.loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so", false)
+ loader.loadAndCreateLink("libicuuc.so.72", "libicuuc.so", false)
+ loader.loadAndCreateLink("libkrb5.so.3", "libkrb5.so", false)
+ loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so", false)
+ loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
+ loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so",
false)
+ loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so", false)
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
+ loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
+ loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
+ loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
+ loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false)
+ loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
+ loader.loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so", false)
+ loader.loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so", false)
+ loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
+ loader.loadAndCreateLink("libglog.so.1", "libglog.so", false)
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
+ loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
index 18f2e6cfb..a03a0a7e8 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
@@ -20,49 +20,46 @@ import org.apache.gluten.vectorized.JniLibLoader
class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader
- .newTransaction()
- .loadAndCreateLink("libroken.so.18", "libroken.so", false)
- .loadAndCreateLink("libasn1.so.8", "libasn1.so", false)
- .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so",
false)
- .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false)
- .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
- .loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
- .loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so", false)
- .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
- .loadAndCreateLink("libnettle.so.7", "libnettle.so", false)
- .loadAndCreateLink("libhogweed.so.5", "libhogweed.so", false)
- .loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
- .loadAndCreateLink("libssh.so.4", "libssh.so", false)
- .loadAndCreateLink("libssl.so.1.1", "libssl.so", false)
- .loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false)
- .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
- .loadAndCreateLink("libwind.so.0", "libwind.so", false)
- .loadAndCreateLink("libheimbase.so.1", "libheimbase.so", false)
- .loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so", false)
- .loadAndCreateLink("libhx509.so.5", "libhx509.so", false)
- .loadAndCreateLink("libkrb5.so.26", "libkrb5.so", false)
- .loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so", false)
- .loadAndCreateLink("libgssapi.so.3", "libgssapi.so", false)
- .loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false)
- .loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
- .loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- .loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
- .loadAndCreateLink("libglog.so.0", "libglog.so", false)
- .loadAndCreateLink("libidn.so.11", "libidn.so", false)
- .loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- .loadAndCreateLink("libicudata.so.66", "libicudata.so", false)
- .loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false)
- .loadAndCreateLink("libxml2.so.2", "libxml2.so", false)
- .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
- .loadAndCreateLink("libre2.so.5", "libre2.so", false)
- .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
- .loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false)
- .commit()
+ loader.loadAndCreateLink("libroken.so.18", "libroken.so", false)
+ loader.loadAndCreateLink("libasn1.so.8", "libasn1.so", false)
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
+ loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so",
false)
+ loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
+ loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
+ loader.loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so", false)
+ loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
+ loader.loadAndCreateLink("libnettle.so.7", "libnettle.so", false)
+ loader.loadAndCreateLink("libhogweed.so.5", "libhogweed.so", false)
+ loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
+ loader.loadAndCreateLink("libssh.so.4", "libssh.so", false)
+ loader.loadAndCreateLink("libssl.so.1.1", "libssl.so", false)
+ loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false)
+ loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
+ loader.loadAndCreateLink("libwind.so.0", "libwind.so", false)
+ loader.loadAndCreateLink("libheimbase.so.1", "libheimbase.so", false)
+ loader.loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so", false)
+ loader.loadAndCreateLink("libhx509.so.5", "libhx509.so", false)
+ loader.loadAndCreateLink("libkrb5.so.26", "libkrb5.so", false)
+ loader.loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so", false)
+ loader.loadAndCreateLink("libgssapi.so.3", "libgssapi.so", false)
+ loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false)
+ loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
+ loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
+ loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
+ loader.loadAndCreateLink("libidn.so.11", "libidn.so", false)
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ loader.loadAndCreateLink("libicudata.so.66", "libicudata.so", false)
+ loader.loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false)
+ loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false)
+ loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+ loader.loadAndCreateLink("libre2.so.5", "libre2.so", false)
+ loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
+ loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false)
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
index b23105b7d..4bb2b59b3 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
@@ -20,34 +20,31 @@ import org.apache.gluten.vectorized.JniLibLoader
class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader
- .newTransaction()
- .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so",
false)
- .loadAndCreateLink("libicudata.so.70", "libicudata.so", false)
- .loadAndCreateLink("libicuuc.so.70", "libicuuc.so", false)
- .loadAndCreateLink("libicui18n.so.70", "libicui18n.so", false)
- .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
- .loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
- .loadAndCreateLink("libssh.so.4", "libssh.so", false)
- .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
- .loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so", false)
- .loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so", false)
- .loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
- .loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- .loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
- .loadAndCreateLink("libglog.so.0", "libglog.so", false)
- .loadAndCreateLink("libidn.so.12", "libidn.so", false)
- .loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- .loadAndCreateLink("libxml2.so.2", "libxml2.so", false)
- .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
- .loadAndCreateLink("libre2.so.9", "libre2.so", false)
- .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
- .loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false)
- .commit()
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
+ loader.loadAndCreateLink("libicudata.so.70", "libicudata.so", false)
+ loader.loadAndCreateLink("libicuuc.so.70", "libicuuc.so", false)
+ loader.loadAndCreateLink("libicui18n.so.70", "libicui18n.so", false)
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
+ loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
+ loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
+ loader.loadAndCreateLink("libssh.so.4", "libssh.so", false)
+ loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
+ loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so", false)
+ loader.loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so", false)
+ loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
+ loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
+ loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
+ loader.loadAndCreateLink("libidn.so.12", "libidn.so", false)
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false)
+ loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+ loader.loadAndCreateLink("libre2.so.9", "libre2.so", false)
+ loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
+ loader.loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false)
}
}
diff --git
a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java
b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java
index b1feb1e5b..37cd29649 100644
--- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java
+++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java
@@ -35,30 +35,20 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Objects;
import java.util.Set;
import java.util.Vector;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
import scala.runtime.BoxedUnit;
-/**
- * LoadXXX methods in the utility prevents reloading of a library internally.
It's not necessary for
- * caller to manage a loaded library list.
- */
public class JniLibLoader {
private static final Logger LOG =
LoggerFactory.getLogger(JniLibLoader.class);
- private static final Set<String> LOADED_LIBRARY_PATHS = new HashSet<>();
- private static final Set<String> REQUIRE_UNLOAD_LIBRARY_PATHS = new
LinkedHashSet<>();
+ private static final Set<String> LOADED_LIBRARY_PATHS =
+ Collections.synchronizedSet(new HashSet<>());
+ private static final Set<String> REQUIRE_UNLOAD_LIBRARY_PATHS =
+ Collections.synchronizedSet(new LinkedHashSet<>());
static {
GlutenShutdownManager.addHookForLibUnloading(
@@ -69,15 +59,17 @@ public class JniLibLoader {
}
private final String workDir;
- private final Set<String> loadedLibraries = new HashSet<>();
- private final Lock sync = new ReentrantLock();
+ private final Set<String> loadedLibraries = Collections.synchronizedSet(new
HashSet<>());
JniLibLoader(String workDir) {
this.workDir = workDir;
}
- public static synchronized void forceUnloadAll() {
- List<String> loaded = new ArrayList<>(REQUIRE_UNLOAD_LIBRARY_PATHS);
+ public static void forceUnloadAll() {
+ List<String> loaded;
+ synchronized (REQUIRE_UNLOAD_LIBRARY_PATHS) {
+ loaded = new ArrayList<>(REQUIRE_UNLOAD_LIBRARY_PATHS);
+ }
Collections.reverse(loaded); // use reversed order to unload
loaded.forEach(JniLibLoader::unloadFromPath);
}
@@ -95,21 +87,25 @@ public class JniLibLoader {
}
}
- private static synchronized void loadFromPath0(String libPath, boolean
requireUnload) {
+ private static void loadFromPath0(String libPath, boolean requireUnload) {
libPath = toRealPath(libPath);
- if (LOADED_LIBRARY_PATHS.contains(libPath)) {
- LOG.debug("Library in path {} has already been loaded, skipping",
libPath);
- } else {
- System.load(libPath);
- LOADED_LIBRARY_PATHS.add(libPath);
- LOG.info("Library {} has been loaded using path-loading method",
libPath);
+ synchronized (LOADED_LIBRARY_PATHS) {
+ if (LOADED_LIBRARY_PATHS.contains(libPath)) {
+ LOG.debug("Library in path {} has already been loaded, skipping",
libPath);
+ } else {
+ System.load(libPath);
+ LOADED_LIBRARY_PATHS.add(libPath);
+ LOG.info("Library {} has been loaded using path-loading method",
libPath);
+ }
}
if (requireUnload) {
- REQUIRE_UNLOAD_LIBRARY_PATHS.add(libPath);
+ synchronized (REQUIRE_UNLOAD_LIBRARY_PATHS) {
+ REQUIRE_UNLOAD_LIBRARY_PATHS.add(libPath);
+ }
}
}
- public static void loadFromPath(String libPath, boolean requireUnload) {
+ public static synchronized void loadFromPath(String libPath, boolean
requireUnload) {
final File file = new File(libPath);
if (!file.isFile() || !file.exists()) {
throw new GlutenException("library at path: " + libPath + " is not a
file or does not exist");
@@ -117,55 +113,42 @@ public class JniLibLoader {
loadFromPath0(file.getAbsolutePath(), requireUnload);
}
- public void mapAndLoad(String unmappedLibName, boolean requireUnload) {
- newTransaction().mapAndLoad(unmappedLibName, requireUnload).commit();
- }
-
- public void load(String libName, boolean requireUnload) {
- newTransaction().load(libName, requireUnload).commit();
- }
-
- public void loadAndCreateLink(String libName, String linkName, boolean
requireUnload) {
- newTransaction().loadAndCreateLink(libName, linkName,
requireUnload).commit();
- }
-
- public JniLoadTransaction newTransaction() {
- return new JniLoadTransaction();
- }
-
- public static synchronized void unloadFromPath(String libPath) {
- if (!LOADED_LIBRARY_PATHS.remove(libPath)) {
- LOG.warn("Library {} was not loaded or alreay unloaded:", libPath);
- return;
+ public static void unloadFromPath(String libPath) {
+ synchronized (LOADED_LIBRARY_PATHS) {
+ if (!LOADED_LIBRARY_PATHS.remove(libPath)) {
+ LOG.warn("Library {} was not loaded or already unloaded:", libPath);
+ return;
+ }
}
-
LOG.info("Starting unload library path: {} ", libPath);
- REQUIRE_UNLOAD_LIBRARY_PATHS.remove(libPath);
-
+ synchronized (REQUIRE_UNLOAD_LIBRARY_PATHS) {
+ REQUIRE_UNLOAD_LIBRARY_PATHS.remove(libPath);
+ }
try {
ClassLoader classLoader = JniLibLoader.class.getClassLoader();
Field field = ClassLoader.class.getDeclaredField("nativeLibraries");
field.setAccessible(true);
Vector<Object> libs = (Vector<Object>) field.get(classLoader);
- Iterator it = libs.iterator();
- while (it.hasNext()) {
- Object object = it.next();
- Field[] fs = object.getClass().getDeclaredFields();
- for (int k = 0; k < fs.length; k++) {
- if (fs[k].getName().equals("name")) {
- fs[k].setAccessible(true);
-
- String verbosePath = fs[k].get(object).toString();
- File verboseFile = new File(verbosePath);
- String verboseFileName = verboseFile.getName();
- File libFile = new File(libPath);
- String libFileName = libFile.getName();
-
- if (verboseFileName.equals(libFileName)) {
- LOG.info("Finalizing library file: {}", libFileName);
- Method finalize =
object.getClass().getDeclaredMethod("finalize");
- finalize.setAccessible(true);
- finalize.invoke(object);
+ synchronized (libs) {
+ Iterator<Object> it = libs.iterator();
+ while (it.hasNext()) {
+ Object object = it.next();
+ Field[] fs = object.getClass().getDeclaredFields();
+ for (int k = 0; k < fs.length; k++) {
+ if (fs[k].getName().equals("name")) {
+ fs[k].setAccessible(true);
+ String verbosePath = fs[k].get(object).toString();
+ File verboseFile = new File(verbosePath);
+ String verboseFileName = verboseFile.getName();
+ File libFile = new File(libPath);
+ String libFileName = libFile.getName();
+
+ if (verboseFileName.equals(libFileName)) {
+ LOG.info("Finalizing library file: {}", libFileName);
+ Method finalize =
object.getClass().getDeclaredMethod("finalize");
+ finalize.setAccessible(true);
+ finalize.invoke(object);
+ }
}
}
}
@@ -175,160 +158,77 @@ public class JniLibLoader {
}
}
- private static final class LoadRequest {
- final String libName;
- final String linkName;
- final boolean requireUnload;
-
- private LoadRequest(String libName, String linkName, boolean
requireUnload) {
- this.libName = libName;
- this.linkName = linkName;
- this.requireUnload = requireUnload;
- }
- }
-
- private static final class LoadAction {
- final String libName;
- final String linkName;
- final boolean requireUnload;
- final File file;
-
- private LoadAction(String libName, String linkName, boolean requireUnload,
File file) {
- this.libName = libName;
- this.linkName = linkName;
- this.requireUnload = requireUnload;
- this.file = file;
- }
-
- public boolean requireLinking() {
- return !Objects.isNull(linkName);
- }
- }
-
- public class JniLoadTransaction {
- private final AtomicBoolean finished = new AtomicBoolean(false);
- private final Map<String, LoadRequest> toLoad = new LinkedHashMap<>(); //
ordered
-
- private JniLoadTransaction() {
- JniLibLoader.this.sync.lock();
- }
-
- public JniLoadTransaction mapAndLoad(String unmappedLibName, boolean
requireUnload) {
+ public void mapAndLoad(String unmappedLibName, boolean requireUnload) {
+ synchronized (loadedLibraries) {
try {
final String mappedLibName = System.mapLibraryName(unmappedLibName);
load(mappedLibName, requireUnload);
- return this;
} catch (Exception e) {
- abort();
throw new GlutenException(e);
}
}
+ }
- public JniLoadTransaction load(String libName, boolean requireUnload) {
+ public void load(String libName, boolean requireUnload) {
+ synchronized (loadedLibraries) {
try {
- toLoad.put(libName, new LoadRequest(libName, null, requireUnload));
- return this;
- } catch (Exception e) {
- abort();
+ if (loadedLibraries.contains(libName)) {
+ LOG.debug("Library {} has already been loaded, skipping", libName);
+ return;
+ }
+ File file = moveToWorkDir(workDir, libName);
+ loadWithLink(file.getAbsolutePath(), null, requireUnload);
+ loadedLibraries.add(libName);
+ LOG.info("Successfully loaded library {}", libName);
+ } catch (IOException e) {
throw new GlutenException(e);
}
}
+ }
- public JniLoadTransaction loadAndCreateLink(
- String libName, String linkName, boolean requireUnload) {
+ public void loadAndCreateLink(String libName, String linkName, boolean
requireUnload) {
+ synchronized (loadedLibraries) {
try {
- toLoad.put(libName, new LoadRequest(libName, linkName, requireUnload));
- return this;
- } catch (Exception e) {
- abort();
+ if (loadedLibraries.contains(libName)) {
+ LOG.debug("Library {} has already been loaded, skipping", libName);
+ }
+ File file = moveToWorkDir(workDir, System.mapLibraryName(libName));
+ loadWithLink(file.getAbsolutePath(), linkName, requireUnload);
+ loadedLibraries.add(libName);
+ LOG.info("Successfully loaded library {}", libName);
+ } catch (IOException e) {
throw new GlutenException(e);
}
}
+ }
- public void commit() {
- try {
- terminate();
- toLoad.entrySet().stream()
- .flatMap(
- e -> {
- try {
- final LoadRequest req = e.getValue();
- if (loadedLibraries.contains(req.libName)) {
- LOG.debug("Library {} has already been loaded,
skipping", req.libName);
- return Stream.empty();
- }
- // load only libraries not loaded yet
- final File file = moveToWorkDir(workDir, req.libName);
- return Stream.of(
- new LoadAction(req.libName, req.linkName,
req.requireUnload, file));
- } catch (IOException ex) {
- throw new GlutenException(ex);
- }
- })
- .collect(Collectors.toList())
- .forEach(
- e -> {
- try {
- LOG.info("Trying to load library {}", e.libName);
- loadWithLink(workDir, e);
- loadedLibraries.add(e.libName);
- LOG.info("Successfully loaded library {}", e.libName);
- } catch (Exception ex) {
- throw new GlutenException(ex);
- }
- });
- } finally {
- JniLibLoader.this.sync.unlock();
- }
- }
-
- public void abort() {
- try {
- terminate();
- // do nothing as of now
- } finally {
- JniLibLoader.this.sync.unlock();
- }
- }
-
- private void terminate() {
- if (!finished.compareAndSet(false, true)) {
- throw new IllegalStateException();
- }
+ private File moveToWorkDir(String workDir, String libraryToLoad) throws
IOException {
+ // final File temp = File.createTempFile(workDir, libraryToLoad);
+ final Path libPath = Paths.get(workDir + "/" + libraryToLoad);
+ if (Files.exists(libPath)) {
+ Files.delete(libPath);
}
-
- private File moveToWorkDir(String workDir, String libraryToLoad) throws
IOException {
- // final File temp = File.createTempFile(workDir, libraryToLoad);
- final Path libPath = Paths.get(workDir + "/" + libraryToLoad);
- if (Files.exists(libPath)) {
- Files.delete(libPath);
+ final File temp = new File(workDir + "/" + libraryToLoad);
+ try (InputStream is =
JniLibLoader.class.getClassLoader().getResourceAsStream(libraryToLoad)) {
+ if (is == null) {
+ throw new FileNotFoundException(libraryToLoad);
}
- final File temp = new File(workDir + "/" + libraryToLoad);
- try (InputStream is =
-
JniLibLoader.class.getClassLoader().getResourceAsStream(libraryToLoad)) {
- if (is == null) {
- throw new FileNotFoundException(libraryToLoad);
- }
- try {
- Files.copy(is, temp.toPath());
- } catch (Exception e) {
- throw new GlutenException(e);
- }
+ try {
+ Files.copy(is, temp.toPath());
+ } catch (Exception e) {
+ throw new GlutenException(e);
}
- return temp;
}
+ return temp;
+ }
- private void loadWithLink(String workDir, LoadAction req) throws
IOException {
- String libPath = req.file.getAbsolutePath();
- loadFromPath0(libPath, req.requireUnload);
- LOG.info("Library {} has been loaded", libPath);
- if (!req.requireLinking()) {
- LOG.debug("Symbolic link not required for library {}, skipping",
libPath);
- return;
- }
- // create link
- Path target = Paths.get(req.file.getPath());
- Path link = Paths.get(workDir, req.linkName);
+ private void loadWithLink(String libPath, String linkName, boolean
requireUnload)
+ throws IOException {
+ loadFromPath0(libPath, requireUnload);
+ LOG.info("Library {} has been loaded", libPath);
+ if (linkName != null) {
+ Path target = Paths.get(libPath);
+ Path link = Paths.get(workDir, linkName);
if (Files.exists(link)) {
LOG.info("Symbolic link already exists for library {}, deleting",
libPath);
Files.delete(link);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
