This is an automated email from the ASF dual-hosted git repository. lanking pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push: new b242b0c [MXNET-1224]: improve scala maven jni build and packing. (#13493) b242b0c is described below commit b242b0c1fb71da43fba8d6208ee8ca282e735474 Author: Frank Liu <frankfliu2...@gmail.com> AuthorDate: Tue Dec 11 15:21:05 2018 -0800 [MXNET-1224]: improve scala maven jni build and packing. (#13493) Major JNI feature changes. Please find more info here: https://cwiki.apache.org/confluence/display/MXNET/Scala+maven+build+improvement --- scala-package/assembly/linux-x86_64-cpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 +++ scala-package/assembly/linux-x86_64-gpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 +++ .../osx-x86_64-cpu/main/assembly/assembly.xml | 30 ----------- scala-package/assembly/osx-x86_64-cpu/pom.xml | 4 ++ .../osx-x86_64-cpu/src/main/assembly/assembly.xml | 6 +++ scala-package/core/pom.xml | 8 +++ .../apache/mxnet/util/NativeLibraryLoader.scala | 55 ++++++++++++------- scala-package/examples/pom.xml | 4 ++ scala-package/infer/pom.xml | 4 ++ scala-package/init-native/linux-x86_64/pom.xml | 42 +++++++++++---- scala-package/init-native/osx-x86_64/pom.xml | 49 ++++++++++++++--- scala-package/native/README.md | 63 ++++++++++++++++++++++ scala-package/native/linux-x86_64-cpu/pom.xml | 25 ++++----- scala-package/native/linux-x86_64-gpu/pom.xml | 25 ++++----- scala-package/native/osx-x86_64-cpu/pom.xml | 50 ++++++++++++++--- scala-package/pom.xml | 2 + 18 files changed, 291 insertions(+), 96 deletions(-) diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index abefead..1658f36 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -14,6 +14,10 @@ <name>MXNet Scala Package - Full Linux-x86_64 CPU-only</name> <packaging>jar</packaging> + <properties> + <MXNET_DIR>${project.parent.parent.basedir}/..</MXNET_DIR> + </properties> + <dependencies> <dependency> <groupId>org.apache.mxnet</groupId> diff --git a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml index a574f8a..f4c2017 100644 --- a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml @@ -25,4 +25,10 @@ </includes> </dependencySet> </dependencySets> + <files> + <file> + <source>${MXNET_DIR}/lib/libmxnet.so</source> + <outputDirectory>lib/native</outputDirectory> + </file> + </files> </assembly> diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index 96ffa38..c80515e 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -14,6 +14,10 @@ <name>MXNet Scala Package - Full Linux-x86_64 GPU</name> <packaging>jar</packaging> + <properties> + <MXNET_DIR>${project.parent.parent.basedir}/..</MXNET_DIR> + </properties> + <dependencies> <dependency> <groupId>org.apache.mxnet</groupId> diff --git a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml index 3a064bf..2aca64b 100644 --- a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml @@ -25,4 +25,10 @@ </includes> </dependencySet> </dependencySets> + <files> + <file> + <source>${MXNET_DIR}/lib/libmxnet.so</source> + <outputDirectory>lib/native</outputDirectory> + </file> + </files> </assembly> diff --git a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml deleted file mode 100644 index fecafec..0000000 --- a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml +++ /dev/null @@ -1,30 +0,0 @@ -<assembly> - <id>full</id> - <formats> - <format>jar</format> - </formats> - <includeBaseDirectory>false</includeBaseDirectory> - <dependencySets> - <dependencySet> - <includes> - <include>*:*:jar</include> - </includes> - <outputDirectory>/</outputDirectory> - <useProjectArtifact>true</useProjectArtifact> - <unpack>true</unpack> - <scope>runtime</scope> - </dependencySet> - <dependencySet> - <outputDirectory>lib/native</outputDirectory> - <outputFileNameMapping>${artifact.artifactId}${dashClassifier?}.${artifact.extension}</outputFileNameMapping> - <unpack>false</unpack> - <useProjectArtifact>false</useProjectArtifact> - <useStrictFiltering>false</useStrictFiltering> - <includes> - <include>*:*:dll:*</include> - <include>*:*:so:*</include> - <include>*:*:jnilib:*</include> - </includes> - </dependencySet> - </dependencySets> -</assembly> diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index 5c5733a..62979a1 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -14,6 +14,10 @@ <name>MXNet Scala Package - Full OSX-x86_64 CPU-only</name> <packaging>jar</packaging> + <properties> + <MXNET_DIR>${project.parent.parent.basedir}/..</MXNET_DIR> + </properties> + <dependencies> <dependency> <groupId>org.apache.mxnet</groupId> diff --git a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml index bdbd09f..e9bc372 100644 --- a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml @@ -25,4 +25,10 @@ </includes> </dependencySet> </dependencySets> + <files> + <file> + <source>${MXNET_DIR}/lib/libmxnet.so</source> + <outputDirectory>lib/native</outputDirectory> + </file> + </files> </assembly> diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 484fbbd..976383f 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -12,6 +12,7 @@ <properties> <skipTests>true</skipTests> + <MXNET_DIR>${project.parent.basedir}/..</MXNET_DIR> </properties> <artifactId>mxnet-core_2.11</artifactId> @@ -77,6 +78,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties </argLine> + <environmentVariables> + <LD_LIBRARY_PATH>${MXNET_DIR}/lib</LD_LIBRARY_PATH> + </environmentVariables> </configuration> </plugin> <plugin> @@ -88,6 +92,10 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target </argLine> <skipTests>${skipTests}</skipTests> + <forkMode>always</forkMode> + <environmentVariables> + <LD_LIBRARY_PATH>${MXNET_DIR}/lib</LD_LIBRARY_PATH> + </environmentVariables> </configuration> </plugin> <plugin> diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala index e94d320..2ce893b 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala @@ -85,12 +85,10 @@ private[mxnet] object NativeLibraryLoader { } logger.debug(s"Attempting to load $loadLibname") val libFileInJar = libPathInJar + loadLibname - val is: InputStream = getClass.getResourceAsStream(libFileInJar) - if (is == null) { - throw new UnsatisfiedLinkError(s"Couldn't find the resource $loadLibname") - } - logger.info(s"Loading $loadLibname from $libPathInJar copying to $libname") - loadLibraryFromStream(libname, is) + saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so") + val tempfile: File = saveLibraryToTemp(libname, libFileInJar) + + loadLibraryFromFile(libname, tempfile) } /** @@ -109,7 +107,7 @@ private[mxnet] object NativeLibraryLoader { @throws(classOf[IOException]) private def createTempFile(name: String): File = { - new File(_tempDir + File.separator + name) + new File(_tempDir, name) } /** @@ -117,11 +115,34 @@ private[mxnet] object NativeLibraryLoader { * and loads from there. * * @param libname name of the library (just used in constructing the library name) - * @param is InputStream pointing to the library + * @param tempfile File pointing to the library */ - private def loadLibraryFromStream(libname: String, is: InputStream) { + private def loadLibraryFromFile(libname: String, tempfile: File) { + try { + logger.debug("Loading library from {}", tempfile.getPath) + System.load(tempfile.getPath) + } catch { + case ule: UnsatisfiedLinkError => + logger.error("Couldn't load copied link file: {}", ule.toString) + throw ule + } + } + + /** + * Load a system library from a stream. Copies the library to a temp file + * and loads from there. + * + * @param libname name of the library (just used in constructing the library name) + * @param resource String resource path in the jar file + */ + private def saveLibraryToTemp(libname: String, resource: String): File = { try { - val tempfile: File = createTempFile(libname) + val is: InputStream = getClass.getResourceAsStream(resource) + if (is == null) { + throw new UnsatisfiedLinkError(s"Couldn't find the resource $resource") + } + + val tempfile: File = new File(_tempDir, libname) val os: OutputStream = new FileOutputStream(tempfile) logger.debug("tempfile.getPath() = {}", tempfile.getPath) val savedTime: Long = System.currentTimeMillis @@ -131,20 +152,14 @@ private[mxnet] object NativeLibraryLoader { os.write(buf, 0, len) len = is.read(buf) } - os.flush() - val lock: InputStream = new FileInputStream(tempfile) os.close() + is.close() val seconds: Double = (System.currentTimeMillis - savedTime).toDouble / 1e3 - logger.debug(s"Copying took $seconds seconds.") - logger.debug("Loading library from {}", tempfile.getPath) - System.load(tempfile.getPath) - lock.close() + logger.debug(s"Copying $libname took $seconds seconds.") + tempfile } catch { case io: IOException => - logger.error("Could not create the temp file: {}", io.toString) - case ule: UnsatisfiedLinkError => - logger.error("Couldn't load copied link file: {}", ule.toString) - throw ule + throw new UnsatisfiedLinkError(s"Could not create temp file for $libname") } } } diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index bc53339..3ebb39b 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -15,6 +15,7 @@ <properties> <skipTests>true</skipTests> + <MXNET_DIR>${project.parent.basedir}/..</MXNET_DIR> </properties> <profiles> @@ -151,6 +152,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties </argLine> + <environmentVariables> + <LD_LIBRARY_PATH>${MXNET_DIR}/lib</LD_LIBRARY_PATH> + </environmentVariables> </configuration> </plugin> <plugin> diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index ac76cdd..fb5cf37 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -15,6 +15,7 @@ <properties> <skipTests>true</skipTests> + <MXNET_DIR>${project.parent.basedir}/..</MXNET_DIR> </properties> <profiles> @@ -77,6 +78,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties </argLine> + <environmentVariables> + <LD_LIBRARY_PATH>${MXNET_DIR}/lib</LD_LIBRARY_PATH> + </environmentVariables> </configuration> </plugin> <plugin> diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml index b71d7cf..242f2f3 100644 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ b/scala-package/init-native/linux-x86_64/pom.xml @@ -16,6 +16,10 @@ <packaging>so</packaging> + <properties> + <MXNET_DIR>${project.parent.parent.basedir}/..</MXNET_DIR> + </properties> + <dependencies> <dependency> <groupId>org.apache.mxnet</groupId> @@ -62,22 +66,24 @@ <compilerStartOption>-std=c++0x</compilerStartOption> </compilerStartOptions> <compilerEndOptions> - <compilerEndOption>-I${project.basedir}/../../../include</compilerEndOption> - <compilerEndOption>${all_includes}</compilerEndOption> - <compilerEndOption>${cflags}</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dmlc-core/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/mshadow</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dlpack/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/tvm/nnvm/include</compilerEndOption> + <compilerEndOption>-DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0</compilerEndOption> + <compilerEndOption>-O3 -DNDEBUG=1 -fPIC -msse3 -mf16c</compilerEndOption> + <compilerEndOption>-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs</compilerEndOption> </compilerEndOptions> <linkerStartOptions> <linkerStartOption>-shared</linkerStartOption> </linkerStartOptions> <linkerMiddleOptions> - <linkerMiddleOption>${all_ldpaths}</linkerMiddleOption> <linkerMiddleOption>-Wl,--whole-archive</linkerMiddleOption> - <linkerMiddleOption>${lddeps}</linkerMiddleOption> - <linkerMiddleOption>-Wl,--no-whole-archive</linkerMiddleOption> + <linkerMiddleOption>-Wl,--no-whole-archive -pthread -lm -fopenmp -lrt</linkerMiddleOption> </linkerMiddleOptions> <linkerEndOptions> - <linkerEndOption>${ldflags}</linkerEndOption> - <linkerEndOption>-fopenmp</linkerEndOption> + <linkerEndOption>-Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib</linkerEndOption> </linkerEndOptions> </configuration> @@ -86,7 +92,6 @@ <id>javah</id> <phase>generate-sources</phase> <configuration> - <javahOS>linux</javahOS> <javahProvider>default</javahProvider> <javahOutputDirectory>${project.build.directory}/custom-javah</javahOutputDirectory> <workingDirectory>${basedir}</workingDirectory> @@ -101,6 +106,25 @@ </execution> </executions> </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.6.0</version> + <executions> + <execution> + <id>link-native-lib</id> + <phase>generate-resources</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>ln</executable> + <commandlineArgs>-sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so</commandlineArgs> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index b4a0b1d..12f4d80 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -16,6 +16,10 @@ <packaging>jnilib</packaging> + <properties> + <MXNET_DIR>${project.parent.parent.basedir}/..</MXNET_DIR> + </properties> + <dependencies> <dependency> <groupId>org.apache.mxnet</groupId> @@ -62,8 +66,14 @@ <compilerStartOption>-std=c++0x</compilerStartOption> </compilerStartOptions> <compilerEndOptions> - <compilerEndOption>-I${project.basedir}/../../../include</compilerEndOption> - <compilerEndOption>${cflags}</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dmlc-core/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/mshadow</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dlpack/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/tvm/nnvm/include</compilerEndOption> + <compilerEndOption>-DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0</compilerEndOption> + <compilerEndOption>-g -O0 -fPIC -msse3 -mf16c</compilerEndOption> + <compilerEndOption>-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs</compilerEndOption> </compilerEndOptions> <linkerStartOptions> <linkerStartOption>-shared</linkerStartOption> @@ -72,11 +82,9 @@ <linkerMiddleOption>-framework JavaVM</linkerMiddleOption> <linkerMiddleOption>-Wl,-exported_symbol,_Java_*</linkerMiddleOption> <linkerMiddleOption>-Wl,-x</linkerMiddleOption> - <linkerMiddleOption>${lddeps}</linkerMiddleOption> - <linkerMiddleOption>-force_load ${project.basedir}/../../../lib/libmxnet.a</linkerMiddleOption> </linkerMiddleOptions> <linkerEndOptions> - <linkerEndOption>${ldflags}</linkerEndOption> + <linkerEndOption>-lmxnet -L${MXNET_DIR}/lib</linkerEndOption> </linkerEndOptions> </configuration> @@ -85,7 +93,6 @@ <id>javah</id> <phase>generate-sources</phase> <configuration> - <javahOS>darwin</javahOS> <javahProvider>default</javahProvider> <javahOutputDirectory>${project.build.directory}/custom-javah</javahOutputDirectory> <workingDirectory>${basedir}</workingDirectory> @@ -100,6 +107,36 @@ </execution> </executions> </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.6.0</version> + <executions> + <execution> + <id>post-native-build</id> + <phase>package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>install_name_tool</executable> + <commandlineArgs>-change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib</commandlineArgs> + </configuration> + </execution> + <execution> + <id>link-native-lib</id> + <phase>generate-resources</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>ln</executable> + <commandlineArgs>-sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so</commandlineArgs> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/scala-package/native/README.md b/scala-package/native/README.md new file mode 100644 index 0000000..cb6dd38 --- /dev/null +++ b/scala-package/native/README.md @@ -0,0 +1,63 @@ +# MXNet Scala JNI + +MXNet Scala JNI is a thin wrapper layer of underlying libmxnet.so. + +## javah +JNI native code requires a header file that matches the java/scala interface, +this file is usually generated with javah. + +In our case, jni_helper_func.h is generated and will be used to compile native code. + + +## Linker options + +Scala JNI (libmxnet-scala.so/libmxnet-scala.jnilib) is dynamically linked to libmxnet.so. +MXNet Scala will trying to load libmxnet.so from system LD_LIBRARY_PATH first. +If it failed, the try to resolve libmxnet.so in the same location as libmxnet-scala.so file. + +### Linux +``` +-Wl,-rpath=$ORIGIN -lmxnet +``` +Above option will tell system to looking for libmxnet.so from the same location. + + +### Mac OSX +On Mac, we have to execute install_name_tool command to change library loading path: +```bash +install_name_tool -change lib/libmxnet.so @loader_path/libmxnet.so libmxnet-scala.jnilib +``` + +Other linker options: +* -shared : link as shared library +* -Wl,-install_name,libmxnet-scala.jnilib : avoid use build machine's absolute path +* -framework JavaVM : Stand jni options for mac +* -Wl,-exported_symbol,_Java_* : Stand jni options for mac +* -Wl,-x : Do not put non-global symbols in the output file's symbol table. + + +## Compiler flags + +Scala JNI code technically doesn't need on any of MXNet make flags, +however c_api.h header links to many other dependencies header file, +which requires us to add DMSHADOW_USE_MKL and DMSHADOW_USE_CUDA to compile the JNI code. +These flags are not actually used by JNI and won't impact Scala's behavior. + + +### Linux + +``` +-DMSHADOW_USE_MKL=0 +-DMSHADOW_USE_CUDA=0 +-O3 -DNDEBUG=1 -fPIC -msse3 -mf16c +-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs +``` + +### Mac OSX + +``` +-DMSHADOW_USE_MKL=0 +-DMSHADOW_USE_CUDA=0 +-g -O0 -fPIC -msse3 -mf16c +-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs +``` diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml index 2415cf7..7cfd01a 100644 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ b/scala-package/native/linux-x86_64-cpu/pom.xml @@ -16,6 +16,10 @@ <packaging>so</packaging> + <properties> + <MXNET_DIR>${project.parent.parent.basedir}/..</MXNET_DIR> + </properties> + <dependencies> <dependency> <groupId>org.apache.mxnet</groupId> @@ -62,22 +66,20 @@ <compilerStartOption>-std=c++0x</compilerStartOption> </compilerStartOptions> <compilerEndOptions> - <compilerEndOption>-I${project.basedir}/../../../include</compilerEndOption> - <compilerEndOption>${all_includes}</compilerEndOption> - <compilerEndOption>${cflags}</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dmlc-core/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/mshadow</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dlpack/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/tvm/nnvm/include</compilerEndOption> + <compilerEndOption>-DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0</compilerEndOption> + <compilerEndOption>-O3 -DNDEBUG=1 -fPIC -msse3 -mf16c</compilerEndOption> + <compilerEndOption>-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs</compilerEndOption> </compilerEndOptions> <linkerStartOptions> <linkerStartOption>-shared</linkerStartOption> </linkerStartOptions> - <linkerMiddleOptions> - <linkerMiddleOption>${all_ldpaths}</linkerMiddleOption> - <linkerMiddleOption>-Wl,--whole-archive</linkerMiddleOption> - <linkerMiddleOption>${lddeps}</linkerMiddleOption> - <linkerMiddleOption>-Wl,--no-whole-archive</linkerMiddleOption> - </linkerMiddleOptions> <linkerEndOptions> - <linkerEndOption>${ldflags}</linkerEndOption> - <linkerEndOption>-fopenmp</linkerEndOption> + <linkerEndOption>-Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib</linkerEndOption> </linkerEndOptions> </configuration> @@ -86,7 +88,6 @@ <id>javah</id> <phase>generate-sources</phase> <configuration> - <javahOS>linux</javahOS> <javahProvider>default</javahProvider> <javahOutputDirectory>${project.build.directory}/custom-javah</javahOutputDirectory> <workingDirectory>${basedir}</workingDirectory> diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml index 0186217..668f330 100644 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ b/scala-package/native/linux-x86_64-gpu/pom.xml @@ -16,6 +16,10 @@ <packaging>so</packaging> + <properties> + <MXNET_DIR>${project.parent.parent.basedir}/..</MXNET_DIR> + </properties> + <dependencies> <dependency> <groupId>org.apache.mxnet</groupId> @@ -62,22 +66,20 @@ <compilerStartOption>-std=c++0x</compilerStartOption> </compilerStartOptions> <compilerEndOptions> - <compilerEndOption>-I${project.basedir}/../../../include</compilerEndOption> - <compilerEndOption>${all_includes}</compilerEndOption> - <compilerEndOption>${cflags}</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dmlc-core/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/mshadow</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dlpack/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/tvm/nnvm/include</compilerEndOption> + <compilerEndOption>-DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0</compilerEndOption> + <compilerEndOption>-O3 -DNDEBUG=1 -fPIC -msse3 -mf16c</compilerEndOption> + <compilerEndOption>-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs</compilerEndOption> </compilerEndOptions> <linkerStartOptions> <linkerStartOption>-shared</linkerStartOption> </linkerStartOptions> - <linkerMiddleOptions> - <linkerMiddleOption>${all_ldpaths}</linkerMiddleOption> - <linkerMiddleOption>-Wl,--whole-archive</linkerMiddleOption> - <linkerMiddleOption>${lddeps}</linkerMiddleOption> - <linkerMiddleOption>-Wl,--no-whole-archive</linkerMiddleOption> - </linkerMiddleOptions> <linkerEndOptions> - <linkerEndOption>${ldflags}</linkerEndOption> - <linkerEndOption>-fopenmp</linkerEndOption> + <linkerEndOption>-Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib</linkerEndOption> </linkerEndOptions> </configuration> @@ -86,7 +88,6 @@ <id>javah</id> <phase>generate-sources</phase> <configuration> - <javahOS>linux</javahOS> <javahProvider>default</javahProvider> <javahOutputDirectory>${project.build.directory}/custom-javah</javahOutputDirectory> <workingDirectory>${basedir}</workingDirectory> diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index 0ab7ca1..425ca96 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -16,6 +16,10 @@ <packaging>jnilib</packaging> + <properties> + <MXNET_DIR>${project.parent.parent.basedir}/..</MXNET_DIR> + </properties> + <dependencies> <dependency> <groupId>org.apache.mxnet</groupId> @@ -62,8 +66,14 @@ <compilerStartOption>-std=c++0x</compilerStartOption> </compilerStartOptions> <compilerEndOptions> - <compilerEndOption>-I../../../include</compilerEndOption> - <compilerEndOption>${cflags}</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dmlc-core/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/mshadow</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/dlpack/include</compilerEndOption> + <compilerEndOption>-I${MXNET_DIR}/3rdparty/tvm/nnvm/include</compilerEndOption> + <compilerEndOption>-DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0</compilerEndOption> + <compilerEndOption>-g -O0 -fPIC -msse3 -mf16c</compilerEndOption> + <compilerEndOption>-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs</compilerEndOption> </compilerEndOptions> <linkerStartOptions> <linkerStartOption>-shared</linkerStartOption> @@ -72,12 +82,9 @@ <linkerMiddleOption>-framework JavaVM</linkerMiddleOption> <linkerMiddleOption>-Wl,-exported_symbol,_Java_*</linkerMiddleOption> <linkerMiddleOption>-Wl,-x</linkerMiddleOption> - <linkerMiddleOption>${lddeps}</linkerMiddleOption> - <linkerMiddleOption>-force_load ${project.basedir}/../../../lib/libmxnet.a</linkerMiddleOption> - <linkerMiddleOption>-force_load ${project.basedir}/../../../3rdparty/tvm/nnvm/lib/libnnvm.a</linkerMiddleOption> </linkerMiddleOptions> <linkerEndOptions> - <linkerEndOption>${ldflags}</linkerEndOption> + <linkerEndOption>-Wl,-install_name,libmxnet-scala.jnilib -lmxnet -L${MXNET_DIR}/lib</linkerEndOption> </linkerEndOptions> </configuration> @@ -86,7 +93,6 @@ <id>javah</id> <phase>generate-sources</phase> <configuration> - <javahOS>darwin</javahOS> <javahProvider>default</javahProvider> <javahOutputDirectory>${project.build.directory}/custom-javah</javahOutputDirectory> <workingDirectory>${basedir}</workingDirectory> @@ -101,6 +107,36 @@ </execution> </executions> </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.6.0</version> + <executions> + <execution> + <id>post-native-build</id> + <phase>package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>install_name_tool</executable> + <commandlineArgs>-change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib</commandlineArgs> + </configuration> + </execution> + <execution> + <id>link-native-lib</id> + <phase>generate-resources</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>ln</executable> + <commandlineArgs>-sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so</commandlineArgs> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 151462c..6eb573b 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -39,6 +39,8 @@ <scala.version>2.11.8</scala.version> <scala.binary.version>2.11</scala.binary.version> <build.platform /> + <cxx>g++</cxx> + <dollar>$</dollar> </properties> <packaging>pom</packaging>