[ 
https://issues.apache.org/jira/browse/TEZ-4557?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Raghav Aggarwal updated TEZ-4557:
---------------------------------
    Description: 
When insert data into table located in encryption zone using Hive with tez 
fails as the httpclient jar has been excluded from hadoop transitive 
dependency. 

Tez: 0.10.2,0.10.3

Hadoop: 3.3.6

Hive: 3.1.2

 

Steps to reproduce issue:

1. Create a encryption key using ranger keyadmin user.
2. hdfs crypto -createZone -keyName test_key -path /user/raghav/encrypt_zone
3. create table tbl(id int) location '/user/raghav/encrypt_zone';
4. insert into tbl values(1);

 

Stacktrace:
{code:java}
Caused by: java.lang.NoClassDefFoundError: 
org/apache/http/client/utils/URIBuilder
    at 
org.apache.hadoop.crypto.key.kms.KMSClientProvider.createURL(KMSClientProvider.java:468)
    at 
org.apache.hadoop.crypto.key.kms.KMSClientProvider.decryptEncryptedKey(KMSClientProvider.java:823)
    at 
org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider$5.call(LoadBalancingKMSClientProvider.java:354)
    at 
org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider$5.call(LoadBalancingKMSClientProvider.java:350)
    at 
org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider.doOp(LoadBalancingKMSClientProvider.java:175)
    at 
org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider.decryptEncryptedKey(LoadBalancingKMSClientProvider.java:350)
    at 
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.decryptEncryptedKey(KeyProviderCryptoExtension.java:535)
    at 
org.apache.hadoop.hdfs.HdfsKMSUtil.decryptEncryptedDataEncryptionKey(HdfsKMSUtil.java:216)
    at 
org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:1002)
    at 
org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:983)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem.safelyCreateWrappedOutputStream(DistributedFileSystem.java:734)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem.access$300(DistributedFileSystem.java:149)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem$8.doCall(DistributedFileSystem.java:572)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem$8.doCall(DistributedFileSystem.java:566)
    at 
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:580)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:507)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:1233)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:1109)
    at 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.getHiveRecordWriter(HiveIgnoreKeyTextOutputFormat.java:81)
    at 
org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getRecordWriter(HiveFileFormatUtils.java:297)
    at 
org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:282)
    at 
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketForFileIdx(FileSinkOperator.java:801)
    at 
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:752)
    at 
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:922)
    at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926)
    at 
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
    at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926)
    at 
org.apache.hadoop.hive.ql.exec.UDTFOperator.forwardUDTFOutput(UDTFOperator.java:133)
    at 
org.apache.hadoop.hive.ql.udf.generic.UDTFCollector.collect(UDTFCollector.java:45)
    at 
org.apache.hadoop.hive.ql.udf.generic.GenericUDTF.forward(GenericUDTF.java:110)
    at 
org.apache.hadoop.hive.ql.udf.generic.GenericUDTFInline.process(GenericUDTFInline.java:64)
    at 
org.apache.hadoop.hive.ql.exec.UDTFOperator.process(UDTFOperator.java:116)
    at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926)
    at 
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
    at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
    at 
org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:125)
    at 
org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:154)
    at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:556)
    at 
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:92){code}

  was:
When insert data into table located in encryption zone using Hive with tez 
fails as the httpclient jar has been excluded from hadoop transitive 
dependency. 

 

Steps to reproduce issue:

1. Create a encryption key using ranger keyadmin user.
2. hdfs crypto -createZone -keyName test_key -path /user/raghav/encrypt_zone
3. create table tbl(id int) location '/user/raghav/encrypt_zone';
4. insert into tbl values(1);

 

Stacktrace:
{code:java}
Caused by: java.lang.NoClassDefFoundError: 
org/apache/http/client/utils/URIBuilder
    at 
org.apache.hadoop.crypto.key.kms.KMSClientProvider.createURL(KMSClientProvider.java:468)
    at 
org.apache.hadoop.crypto.key.kms.KMSClientProvider.decryptEncryptedKey(KMSClientProvider.java:823)
    at 
org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider$5.call(LoadBalancingKMSClientProvider.java:354)
    at 
org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider$5.call(LoadBalancingKMSClientProvider.java:350)
    at 
org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider.doOp(LoadBalancingKMSClientProvider.java:175)
    at 
org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider.decryptEncryptedKey(LoadBalancingKMSClientProvider.java:350)
    at 
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.decryptEncryptedKey(KeyProviderCryptoExtension.java:535)
    at 
org.apache.hadoop.hdfs.HdfsKMSUtil.decryptEncryptedDataEncryptionKey(HdfsKMSUtil.java:216)
    at 
org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:1002)
    at 
org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:983)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem.safelyCreateWrappedOutputStream(DistributedFileSystem.java:734)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem.access$300(DistributedFileSystem.java:149)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem$8.doCall(DistributedFileSystem.java:572)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem$8.doCall(DistributedFileSystem.java:566)
    at 
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:580)
    at 
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:507)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:1233)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:1109)
    at 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.getHiveRecordWriter(HiveIgnoreKeyTextOutputFormat.java:81)
    at 
org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getRecordWriter(HiveFileFormatUtils.java:297)
    at 
org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:282)
    at 
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketForFileIdx(FileSinkOperator.java:801)
    at 
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:752)
    at 
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:922)
    at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926)
    at 
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
    at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926)
    at 
org.apache.hadoop.hive.ql.exec.UDTFOperator.forwardUDTFOutput(UDTFOperator.java:133)
    at 
org.apache.hadoop.hive.ql.udf.generic.UDTFCollector.collect(UDTFCollector.java:45)
    at 
org.apache.hadoop.hive.ql.udf.generic.GenericUDTF.forward(GenericUDTF.java:110)
    at 
org.apache.hadoop.hive.ql.udf.generic.GenericUDTFInline.process(GenericUDTFInline.java:64)
    at 
org.apache.hadoop.hive.ql.exec.UDTFOperator.process(UDTFOperator.java:116)
    at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926)
    at 
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
    at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
    at 
org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:125)
    at 
org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:154)
    at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:556)
    at 
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:92){code}


> Revert TEZ-4303, NoClassDefFoundError because of missing httpclient jar
> -----------------------------------------------------------------------
>
>                 Key: TEZ-4557
>                 URL: https://issues.apache.org/jira/browse/TEZ-4557
>             Project: Apache Tez
>          Issue Type: Bug
>            Reporter: Raghav Aggarwal
>            Assignee: Raghav Aggarwal
>            Priority: Major
>
> When insert data into table located in encryption zone using Hive with tez 
> fails as the httpclient jar has been excluded from hadoop transitive 
> dependency. 
> Tez: 0.10.2,0.10.3
> Hadoop: 3.3.6
> Hive: 3.1.2
>  
> Steps to reproduce issue:
> 1. Create a encryption key using ranger keyadmin user.
> 2. hdfs crypto -createZone -keyName test_key -path /user/raghav/encrypt_zone
> 3. create table tbl(id int) location '/user/raghav/encrypt_zone';
> 4. insert into tbl values(1);
>  
> Stacktrace:
> {code:java}
> Caused by: java.lang.NoClassDefFoundError: 
> org/apache/http/client/utils/URIBuilder
>     at 
> org.apache.hadoop.crypto.key.kms.KMSClientProvider.createURL(KMSClientProvider.java:468)
>     at 
> org.apache.hadoop.crypto.key.kms.KMSClientProvider.decryptEncryptedKey(KMSClientProvider.java:823)
>     at 
> org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider$5.call(LoadBalancingKMSClientProvider.java:354)
>     at 
> org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider$5.call(LoadBalancingKMSClientProvider.java:350)
>     at 
> org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider.doOp(LoadBalancingKMSClientProvider.java:175)
>     at 
> org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider.decryptEncryptedKey(LoadBalancingKMSClientProvider.java:350)
>     at 
> org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.decryptEncryptedKey(KeyProviderCryptoExtension.java:535)
>     at 
> org.apache.hadoop.hdfs.HdfsKMSUtil.decryptEncryptedDataEncryptionKey(HdfsKMSUtil.java:216)
>     at 
> org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:1002)
>     at 
> org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:983)
>     at 
> org.apache.hadoop.hdfs.DistributedFileSystem.safelyCreateWrappedOutputStream(DistributedFileSystem.java:734)
>     at 
> org.apache.hadoop.hdfs.DistributedFileSystem.access$300(DistributedFileSystem.java:149)
>     at 
> org.apache.hadoop.hdfs.DistributedFileSystem$8.doCall(DistributedFileSystem.java:572)
>     at 
> org.apache.hadoop.hdfs.DistributedFileSystem$8.doCall(DistributedFileSystem.java:566)
>     at 
> org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
>     at 
> org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:580)
>     at 
> org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:507)
>     at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:1233)
>     at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:1109)
>     at 
> org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.getHiveRecordWriter(HiveIgnoreKeyTextOutputFormat.java:81)
>     at 
> org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getRecordWriter(HiveFileFormatUtils.java:297)
>     at 
> org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:282)
>     at 
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketForFileIdx(FileSinkOperator.java:801)
>     at 
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:752)
>     at 
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:922)
>     at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
>     at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
>     at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926)
>     at 
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
>     at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
>     at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
>     at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926)
>     at 
> org.apache.hadoop.hive.ql.exec.UDTFOperator.forwardUDTFOutput(UDTFOperator.java:133)
>     at 
> org.apache.hadoop.hive.ql.udf.generic.UDTFCollector.collect(UDTFCollector.java:45)
>     at 
> org.apache.hadoop.hive.ql.udf.generic.GenericUDTF.forward(GenericUDTF.java:110)
>     at 
> org.apache.hadoop.hive.ql.udf.generic.GenericUDTFInline.process(GenericUDTFInline.java:64)
>     at 
> org.apache.hadoop.hive.ql.exec.UDTFOperator.process(UDTFOperator.java:116)
>     at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
>     at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
>     at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:926)
>     at 
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
>     at org.apache.hadoop.hive.ql.exec.Operator.baseForward(Operator.java:993)
>     at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:939)
>     at 
> org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:125)
>     at 
> org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:154)
>     at 
> org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:556)
>     at 
> org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:92){code}



--
This message was sent by Atlassian Jira
(v8.20.10#820010)

Reply via email to