Author: eli Date: Wed Oct 3 00:10:22 2012 New Revision: 1393244 URL: http://svn.apache.org/viewvc?rev=1393244&view=rev Log: HADOOP-8756. Fix SEGV when libsnappy is in java.library.path but not LD_LIBRARY_PATH. Contributed by Colin Patrick McCabe
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c - copied unchanged from r1393243, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c Removed: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/LoadSnappy.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-auth/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/core/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java Propchange: hadoop/common/branches/branch-2/hadoop-common-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project:r1393243 Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-auth/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-auth:r1393243 Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common:r1393243 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1393244&r1=1393243&r2=1393244&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 3 00:10:22 2012 @@ -53,6 +53,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-8616. ViewFS configuration requires a trailing slash. (Sandy Ryza via atm) + HADOOP-8756. Fix SEGV when libsnappy is in java.library.path but + not LD_LIBRARY_PATH. (Colin Patrick McCabe via eli) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1393243 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt?rev=1393244&r1=1393243&r2=1393244&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt Wed Oct 3 00:10:22 2012 @@ -123,6 +123,7 @@ add_dual_library(hadoop ${D}/security/JniBasedUnixGroupsMapping.c ${D}/security/JniBasedUnixGroupsNetgroupMapping.c ${D}/security/getGroup.c + ${D}/util/NativeCodeLoader.c ${D}/util/NativeCrc32.c ${D}/util/bulk_crc32.c ) Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake?rev=1393244&r1=1393243&r2=1393244&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake Wed Oct 3 00:10:22 2012 @@ -2,7 +2,6 @@ #define CONFIG_H #cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@" -#cmakedefine HADOOP_RUNAS_HOME "@HADOOP_RUNAS_HOME@" #cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@" #cmakedefine HAVE_SYNC_FILE_RANGE #cmakedefine HAVE_POSIX_FADVISE Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1393243 Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1393243 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java?rev=1393244&r1=1393243&r2=1393244&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java Wed Oct 3 00:10:22 2012 @@ -24,7 +24,6 @@ import java.io.OutputStream; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.compress.snappy.LoadSnappy; import org.apache.hadoop.io.compress.snappy.SnappyCompressor; import org.apache.hadoop.io.compress.snappy.SnappyDecompressor; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -34,11 +33,6 @@ import org.apache.hadoop.util.NativeCode * This class creates snappy compressors/decompressors. */ public class SnappyCodec implements Configurable, CompressionCodec { - - static { - LoadSnappy.isLoaded(); - } - Configuration conf; /** @@ -63,11 +57,26 @@ public class SnappyCodec implements Conf /** * Are the native snappy libraries loaded & initialized? - * - * @return true if loaded & initialized, otherwise false */ + public static void checkNativeCodeLoaded() { + if (!NativeCodeLoader.buildSupportsSnappy()) { + throw new RuntimeException("native snappy library not available: " + + "this version of libhadoop was built without " + + "snappy support."); + } + if (!SnappyCompressor.isNativeCodeLoaded()) { + throw new RuntimeException("native snappy library not available: " + + "SnappyCompressor has not been loaded."); + } + if (!SnappyDecompressor.isNativeCodeLoaded()) { + throw new RuntimeException("native snappy library not available: " + + "SnappyDecompressor has not been loaded."); + } + } + public static boolean isNativeCodeLoaded() { - return LoadSnappy.isLoaded() && NativeCodeLoader.isNativeCodeLoaded(); + return SnappyCompressor.isNativeCodeLoaded() && + SnappyDecompressor.isNativeCodeLoaded(); } /** @@ -97,9 +106,7 @@ public class SnappyCodec implements Conf public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } + checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); @@ -117,10 +124,7 @@ public class SnappyCodec implements Conf */ @Override public Class<? extends Compressor> getCompressorType() { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } - + checkNativeCodeLoaded(); return SnappyCompressor.class; } @@ -131,9 +135,7 @@ public class SnappyCodec implements Conf */ @Override public Compressor createCompressor() { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } + checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); @@ -167,10 +169,7 @@ public class SnappyCodec implements Conf public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } - + checkNativeCodeLoaded(); return new BlockDecompressorStream(in, decompressor, conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT)); @@ -183,10 +182,7 @@ public class SnappyCodec implements Conf */ @Override public Class<? extends Decompressor> getDecompressorType() { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } - + checkNativeCodeLoaded(); return SnappyDecompressor.class; } @@ -197,9 +193,7 @@ public class SnappyCodec implements Conf */ @Override public Decompressor createDecompressor() { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } + checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java?rev=1393244&r1=1393243&r2=1393244&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java Wed Oct 3 00:10:22 2012 @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.Compressor; +import org.apache.hadoop.util.NativeCodeLoader; /** * A {@link Compressor} based on the snappy compression algorithm. @@ -51,22 +52,24 @@ public class SnappyCompressor implements private long bytesRead = 0L; private long bytesWritten = 0L; - + private static boolean nativeSnappyLoaded = false; + static { - if (LoadSnappy.isLoaded()) { - // Initialize the native library + if (NativeCodeLoader.isNativeCodeLoaded() && + NativeCodeLoader.buildSupportsSnappy()) { try { initIDs(); + nativeSnappyLoaded = true; } catch (Throwable t) { - // Ignore failure to load/initialize snappy - LOG.warn(t.toString()); + LOG.error("failed to load SnappyCompressor", t); } - } else { - LOG.error("Cannot load " + SnappyCompressor.class.getName() + - " without snappy library!"); } } - + + public static boolean isNativeCodeLoaded() { + return nativeSnappyLoaded; + } + /** * Creates a new compressor. * Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java?rev=1393244&r1=1393243&r2=1393244&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java Wed Oct 3 00:10:22 2012 @@ -25,6 +25,7 @@ import java.nio.ByteBuffer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.compress.Decompressor; +import org.apache.hadoop.util.NativeCodeLoader; /** * A {@link Decompressor} based on the snappy compression algorithm. @@ -47,21 +48,24 @@ public class SnappyDecompressor implemen private int userBufOff = 0, userBufLen = 0; private boolean finished; + private static boolean nativeSnappyLoaded = false; + static { - if (LoadSnappy.isLoaded()) { - // Initialize the native library + if (NativeCodeLoader.isNativeCodeLoaded() && + NativeCodeLoader.buildSupportsSnappy()) { try { initIDs(); + nativeSnappyLoaded = true; } catch (Throwable t) { - // Ignore failure to load/initialize snappy - LOG.warn(t.toString()); + LOG.error("failed to load SnappyDecompressor", t); } - } else { - LOG.error("Cannot load " + SnappyDecompressor.class.getName() + - " without snappy library!"); } } - + + public static boolean isNativeCodeLoaded() { + return nativeSnappyLoaded; + } + /** * Creates a new compressor. * Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java?rev=1393244&r1=1393243&r2=1393244&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java Wed Oct 3 00:10:22 2012 @@ -75,6 +75,11 @@ public class NativeCodeLoader { } /** + * Returns true only if this build was compiled with support for snappy. + */ + public static native boolean buildSupportsSnappy(); + + /** * Return if native hadoop libraries, if present, can be used for this job. * @param conf configuration * Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1393243 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java?rev=1393244&r1=1393243&r2=1393244&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java Wed Oct 3 00:10:22 2012 @@ -52,7 +52,6 @@ import org.apache.hadoop.io.SequenceFile import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.io.compress.snappy.LoadSnappy; import org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor; import org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater; import org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater; @@ -101,14 +100,9 @@ public class TestCodec { @Test public void testSnappyCodec() throws IOException { - if (LoadSnappy.isAvailable()) { - if (LoadSnappy.isLoaded()) { - codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec"); - codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec"); - } - else { - Assert.fail("Snappy native available but Hadoop native not"); - } + if (SnappyCodec.isNativeCodeLoaded()) { + codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec"); + codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec"); } }