HADOOP-11057. checknative command to probe for winutils.exe on windows. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6dae4b43 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6dae4b43 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6dae4b43 Branch: refs/heads/HDFS-6584 Commit: 6dae4b430c342f9ad44ad8659c372e519f3931c9 Parents: 0de563a Author: cnauroth <cnaur...@apache.org> Authored: Tue Sep 9 21:38:29 2014 -0700 Committer: cnauroth <cnaur...@apache.org> Committed: Tue Sep 9 21:38:29 2014 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../hadoop/util/NativeLibraryChecker.java | 22 +++++++++++++-- .../hadoop/util/TestNativeLibraryChecker.java | 29 ++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dae4b43/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 75ad7db..c60a9b7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -506,6 +506,9 @@ Release 2.6.0 - UNRELEASED HADOOP-11070. Create MiniKMS for testing. (tucu) + HADOOP-11057. checknative command to probe for winutils.exe on windows. + (Xiaoyu Yao via cnauroth) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dae4b43/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java index 0d87bce..6416355 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java @@ -37,7 +37,8 @@ public class NativeLibraryChecker { public static void main(String[] args) { String usage = "NativeLibraryChecker [-a|-h]\n" + " -a use -a to check all libraries are available\n" - + " by default just check hadoop library is available\n" + + " by default just check hadoop library (and\n" + + " winutils.exe on Windows OS) is available\n" + " exit with error code 1 if check failed\n" + " -h print this message\n"; if (args.length > 1 || @@ -62,12 +63,16 @@ public class NativeLibraryChecker { boolean lz4Loaded = nativeHadoopLoaded; boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf); boolean openSslLoaded = false; + boolean winutilsExists = false; + String openSslDetail = ""; String hadoopLibraryName = ""; String zlibLibraryName = ""; String snappyLibraryName = ""; String lz4LibraryName = ""; String bzip2LibraryName = ""; + String winutilsPath = null; + if (nativeHadoopLoaded) { hadoopLibraryName = NativeCodeLoader.getLibraryName(); zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf); @@ -93,6 +98,15 @@ public class NativeLibraryChecker { bzip2LibraryName = Bzip2Factory.getLibraryName(conf); } } + + // winutils.exe is required on Windows + winutilsPath = Shell.getWinUtilsPath(); + if (winutilsPath != null) { + winutilsExists = true; + } else { + winutilsPath = ""; + } + System.out.println("Native library checking:"); System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName); System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName); @@ -100,7 +114,11 @@ public class NativeLibraryChecker { System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName); System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName); System.out.printf("openssl: %b %s\n", openSslLoaded, openSslDetail); - if ((!nativeHadoopLoaded) || + if (Shell.WINDOWS) { + System.out.printf("winutils: %b %s\n", winutilsExists, winutilsPath); + } + + if ((!nativeHadoopLoaded) || (Shell.WINDOWS && (!winutilsExists)) || (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) { // return 1 to indicated check failed ExitUtil.terminate(1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dae4b43/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java index c41b567..7589e5a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.util; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + import junit.framework.TestCase; import org.apache.hadoop.util.ExitUtil.ExitException; @@ -51,4 +54,30 @@ public class TestNativeLibraryChecker extends TestCase { } } + @Test + public void testNativeLibraryCheckerOutput(){ + expectOutput(new String[]{"-a"}); + // no argument + expectOutput(new String[0]); + } + + private void expectOutput(String [] args) { + ExitUtil.disableSystemExit(); + ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + PrintStream originalPs = System.out; + System.setOut(new PrintStream(outContent)); + try { + NativeLibraryChecker.main(args); + } catch (ExitException e) { + ExitUtil.resetFirstExitException(); + } finally { + if (Shell.WINDOWS) { + assertEquals(outContent.toString().indexOf("winutils: true") != -1, true); + } + if (NativeCodeLoader.isNativeCodeLoaded()) { + assertEquals(outContent.toString().indexOf("hadoop: true") != -1, true); + } + System.setOut(originalPs); + } + } }
