[
https://issues.apache.org/jira/browse/TEZ-4567?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Bilwa S T updated TEZ-4567:
---------------------------
Description:
Currently we use hadoop 3.3.6 version. As part of this Jira HADOOP-17292, Lz4
is a provided dependency in Hadoop Common 3.3.1 for Lz4Codec, so we need to add
the dependency in tez as well. Otherwise we get the below exception when we run
hive job on tez
{code:java}
Caused by: java.lang.NoClassDefFoundError: net/jpountz/lz4/LZ4Factory at
org.apache.hadoop.io.compress.lz4.Lz4Compressor.<init>(Lz4Compressor.java:66)
at org.apache.hadoop.io.compress.Lz4Codec.createCompressor(Lz4Codec.java:119)
at org.apache.hadoop.io.compress.CodecPool.getCompressor(CodecPool.java:152) at
org.apache.hadoop.io.compress.CompressionCodec$Util.createOutputStreamWithCodecPool(CompressionCodec.java:131)
at org.apache.hadoop.io.compress.Lz4Codec.createOutputStream(Lz4Codec.java:70)
at
org.apache.hadoop.hive.ql.exec.Utilities.createCompressedStream(Utilities.java:949)
at
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.getHiveRecordWriter(HiveIgnoreKeyTextOutputFormat.java:80)
at
org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getRecordWriter(HiveFileFormatUtils.java:297)
at
org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:282)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketForFileIdx(FileSinkOperator.java:801)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:752)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:922)
at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926) at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926) at
org.apache.hadoop.hive.ql.exec.UDTFOperator.forwardUDTFOutput(UDTFOperator.java:133)
at
org.apache.hadoop.hive.ql.udf.generic.UDTFCollector.collect(UDTFCollector.java:45)
at
org.apache.hadoop.hive.ql.udf.generic.GenericUDTF.forward(GenericUDTF.java:110)
at
org.apache.hadoop.hive.ql.udf.generic.GenericUDTFInline.process(GenericUDTFInline.java:64)
at org.apache.hadoop.hive.ql.exec.UDTFOperator.process(UDTFOperator.java:116)
at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926) at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:125)
at
org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:154)
at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:556) at
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:92)
... 19 more Caused by: java.lang.ClassNotFoundException:
net.jpountz.lz4.LZ4Factory at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641)
at
java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525) ... 53
more{code}
was:
Currently we use hadoop 3.3.6 version of hadoop. As part of this Jira
HADOOP-17292, Lz4 is a provided dependency in Hadoop Common 3.3.1 for Lz4Codec,
so we need to add the dependency in tez as well. Otherwise we get the below
exception when we run hive job on tez
{code:java}
Caused by: java.lang.NoClassDefFoundError: net/jpountz/lz4/LZ4Factory at
org.apache.hadoop.io.compress.lz4.Lz4Compressor.<init>(Lz4Compressor.java:66)
at org.apache.hadoop.io.compress.Lz4Codec.createCompressor(Lz4Codec.java:119)
at org.apache.hadoop.io.compress.CodecPool.getCompressor(CodecPool.java:152) at
org.apache.hadoop.io.compress.CompressionCodec$Util.createOutputStreamWithCodecPool(CompressionCodec.java:131)
at org.apache.hadoop.io.compress.Lz4Codec.createOutputStream(Lz4Codec.java:70)
at
org.apache.hadoop.hive.ql.exec.Utilities.createCompressedStream(Utilities.java:949)
at
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.getHiveRecordWriter(HiveIgnoreKeyTextOutputFormat.java:80)
at
org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getRecordWriter(HiveFileFormatUtils.java:297)
at
org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:282)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketForFileIdx(FileSinkOperator.java:801)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:752)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:922)
at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926) at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926) at
org.apache.hadoop.hive.ql.exec.UDTFOperator.forwardUDTFOutput(UDTFOperator.java:133)
at
org.apache.hadoop.hive.ql.udf.generic.UDTFCollector.collect(UDTFCollector.java:45)
at
org.apache.hadoop.hive.ql.udf.generic.GenericUDTF.forward(GenericUDTF.java:110)
at
org.apache.hadoop.hive.ql.udf.generic.GenericUDTFInline.process(GenericUDTFInline.java:64)
at org.apache.hadoop.hive.ql.exec.UDTFOperator.process(UDTFOperator.java:116)
at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926) at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:125)
at
org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:154)
at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:556) at
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:92)
... 19 more Caused by: java.lang.ClassNotFoundException:
net.jpountz.lz4.LZ4Factory at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641)
at
java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525) ... 53
more{code}
> Failed to load Lz4 codec
> ------------------------
>
> Key: TEZ-4567
> URL: https://issues.apache.org/jira/browse/TEZ-4567
> Project: Apache Tez
> Issue Type: Bug
> Affects Versions: 0.10.3
> Reporter: Bilwa S T
> Assignee: Bilwa S T
> Priority: Major
>
> Currently we use hadoop 3.3.6 version. As part of this Jira HADOOP-17292, Lz4
> is a provided dependency in Hadoop Common 3.3.1 for Lz4Codec, so we need to
> add the dependency in tez as well. Otherwise we get the below exception when
> we run hive job on tez
> {code:java}
> Caused by: java.lang.NoClassDefFoundError: net/jpountz/lz4/LZ4Factory at
> org.apache.hadoop.io.compress.lz4.Lz4Compressor.<init>(Lz4Compressor.java:66)
> at org.apache.hadoop.io.compress.Lz4Codec.createCompressor(Lz4Codec.java:119)
> at org.apache.hadoop.io.compress.CodecPool.getCompressor(CodecPool.java:152)
> at
> org.apache.hadoop.io.compress.CompressionCodec$Util.createOutputStreamWithCodecPool(CompressionCodec.java:131)
> at
> org.apache.hadoop.io.compress.Lz4Codec.createOutputStream(Lz4Codec.java:70)
> at
> org.apache.hadoop.hive.ql.exec.Utilities.createCompressedStream(Utilities.java:949)
> at
> org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.getHiveRecordWriter(HiveIgnoreKeyTextOutputFormat.java:80)
> at
> org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getRecordWriter(HiveFileFormatUtils.java:297)
> at
> org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:282)
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketForFileIdx(FileSinkOperator.java:801)
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:752)
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:922)
> at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
> org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
> org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926) at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
> org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
> org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926) at
> org.apache.hadoop.hive.ql.exec.UDTFOperator.forwardUDTFOutput(UDTFOperator.java:133)
> at
> org.apache.hadoop.hive.ql.udf.generic.UDTFCollector.collect(UDTFCollector.java:45)
> at
> org.apache.hadoop.hive.ql.udf.generic.GenericUDTF.forward(GenericUDTF.java:110)
> at
> org.apache.hadoop.hive.ql.udf.generic.GenericUDTFInline.process(GenericUDTFInline.java:64)
> at
> org.apache.hadoop.hive.ql.exec.UDTFOperator.process(UDTFOperator.java:116) at
> org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
> org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
> org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926) at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993) at
> org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939) at
> org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:125)
> at
> org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:154)
> at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:556)
> at
> org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:92)
> ... 19 more Caused by: java.lang.ClassNotFoundException:
> net.jpountz.lz4.LZ4Factory at
> java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641)
> at
> java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
> at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525) ... 53
> more{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
