This is an automated email from the ASF dual-hosted git repository.
hemant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7ac04658a9 HDDS-9174. Native library loader fails when system property
"native.lib.tmp.dir" is not set (#5190)
7ac04658a9 is described below
commit 7ac04658a9c8379155068529424a055018dc268e
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Tue Sep 5 22:54:25 2023 -0700
HDDS-9174. Native library loader fails when system property
"native.lib.tmp.dir" is not set (#5190)
---
hadoop-hdds/rocks-native/pom.xml | 5 ++
.../hadoop/hdds/utils/NativeLibraryLoader.java | 29 +++++--
.../hadoop/hdds/utils/TestNativeLibraryLoader.java | 91 ++++++++++++++++++++++
pom.xml | 6 ++
4 files changed, 126 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml
index 14fec20de3..6866890d81 100644
--- a/hadoop-hdds/rocks-native/pom.xml
+++ b/hadoop-hdds/rocks-native/pom.xml
@@ -57,6 +57,11 @@
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-inline</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<properties>
<maven.compiler.source>8</maven.compiler.source>
diff --git
a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
index 2ab264a6f6..10df236f88 100644
---
a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
+++
b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.utils;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,6 +29,7 @@ import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
@@ -78,7 +80,8 @@ public class NativeLibraryLoader {
return OS.startsWith("linux");
}
- private static String getLibOsSuffix() {
+ @VisibleForTesting
+ static String getLibOsSuffix() {
if (isMac()) {
return ".dylib";
} else if (isWindows()) {
@@ -126,20 +129,36 @@ public class NativeLibraryLoader {
return isLibraryLoaded(libraryName);
}
+ // Added function to make this testable.
+ @VisibleForTesting
+ static String getSystemProperty(String property) {
+ return System.getProperty(property);
+ }
+
+ // Added function to make this testable
+ @VisibleForTesting
+ static InputStream getResourceStream(String libraryFileName) {
+ return NativeLibraryLoader.class.getClassLoader()
+ .getResourceAsStream(libraryFileName);
+ }
+
private Optional<File> copyResourceFromJarToTemp(final String libraryName)
throws IOException {
final String libraryFileName = getJniLibraryFileName(libraryName);
InputStream is = null;
try {
- is = getClass().getClassLoader().getResourceAsStream(libraryFileName);
+ is = getResourceStream(libraryFileName);
if (is == null) {
return Optional.empty();
}
+ final String nativeLibDir =
+ Objects.nonNull(getSystemProperty(NATIVE_LIB_TMP_DIR)) ?
+ getSystemProperty(NATIVE_LIB_TMP_DIR) : "";
+ final File dir = new File(nativeLibDir).getAbsoluteFile();
+
// create a temporary file to copy the library to
- final File temp = File.createTempFile(libraryName, getLibOsSuffix(),
- new File(Optional.ofNullable(System.getProperty(NATIVE_LIB_TMP_DIR))
- .orElse("")));
+ final File temp = File.createTempFile(libraryName, getLibOsSuffix(),
dir);
if (!temp.exists()) {
return Optional.empty();
} else {
diff --git
a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
new file mode 100644
index 0000000000..472954f2bd
--- /dev/null
+++
b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.utils;
+
+
+import org.apache.ozone.test.tag.Native;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static
org.apache.hadoop.hdds.utils.NativeLibraryLoader.NATIVE_LIB_TMP_DIR;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.same;
+
+/**
+ * Test class for NativeLibraryLoader.
+ */
+public class TestNativeLibraryLoader {
+
+ private static Stream<String> nativeLibraryDirectoryLocations()
+ throws IOException {
+ return Stream.of("", File.createTempFile("prefix", "suffix")
+ .getParentFile().getAbsolutePath(), null);
+ }
+
+ @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+ @ParameterizedTest
+ @MethodSource("nativeLibraryDirectoryLocations")
+ public void testNativeLibraryLoader(
+ String nativeLibraryDirectoryLocation) {
+ Map<String, Boolean> libraryLoadedMap = new HashMap<>();
+ NativeLibraryLoader loader = new NativeLibraryLoader(libraryLoadedMap);
+ try (MockedStatic<NativeLibraryLoader> mockedNativeLibraryLoader =
+ Mockito.mockStatic(NativeLibraryLoader.class,
+ Mockito.CALLS_REAL_METHODS)) {
+ mockedNativeLibraryLoader.when(() ->
+ NativeLibraryLoader.getSystemProperty(same(NATIVE_LIB_TMP_DIR)))
+ .thenReturn(nativeLibraryDirectoryLocation);
+ mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getInstance())
+ .thenReturn(loader);
+ Assertions.assertTrue(NativeLibraryLoader.getInstance()
+ .loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME));
+ Assertions.assertTrue(NativeLibraryLoader
+ .isLibraryLoaded(ROCKS_TOOLS_NATIVE_LIBRARY_NAME));
+ // Mocking to force copy random bytes to create a lib file to
+ // nativeLibraryDirectoryLocation. But load library will fail.
+ mockedNativeLibraryLoader.when(() ->
+ NativeLibraryLoader.getResourceStream(anyString()))
+ .thenReturn(new ByteArrayInputStream(new byte[]{0, 1, 2, 3}));
+ String dummyLibraryName = "dummy_lib";
+ NativeLibraryLoader.getInstance().loadLibrary(dummyLibraryName);
+ NativeLibraryLoader.isLibraryLoaded(dummyLibraryName);
+ // Checking if the resource with random was copied to a temp file.
+ File[] libPath =
+ new File(nativeLibraryDirectoryLocation == null ? "" :
+ nativeLibraryDirectoryLocation)
+ .getAbsoluteFile().listFiles((dir, name) ->
+ name.startsWith(dummyLibraryName) &&
+ name.endsWith(NativeLibraryLoader.getLibOsSuffix()));
+ Assertions.assertNotNull(libPath);
+ Assertions.assertEquals(1, libPath.length);
+ Assertions.assertTrue(libPath[0].delete());
+ }
+
+ }
+}
diff --git a/pom.xml b/pom.xml
index 717dacd8d6..241db48476 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1571,6 +1571,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<artifactId>jgrapht-ext</artifactId>
<version>${jgrapht.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-inline</artifactId>
+ <version>${mockito2.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
