[
https://issues.apache.org/jira/browse/SPARK-34754?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lithiumlee-_- updated SPARK-34754:
----------------------------------
Description:
Submit app to K8S, the executors meet exception
"java.net.UnknownHostException: xx".
The udf jar uri using hdfs ha style, but the exception stack show
"...*createNonHAProxy*..."
hql:
{code:java}
// code placeholder
add jar hdfs://xx/test.jar;
create temporary function test_udf as 'com.xxx.xxx';
create table test.test_udf as
select test_udf('1') name_1;
{code}
exception:
{code:java}
// code placeholder
TaskSetManager: Lost task 0.0 in stage 0.0 (TID 0, 172.30.89.44, executor 1):
java.lang.IllegalArgumentException: java.net.UnknownHostException: xx
at
org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:439)
at
org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:321)
at
org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:176)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:696)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:636)
at
org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:160)
at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2796)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:99)
at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2830)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2812)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:390)
at org.apache.spark.util.Utils$.getHadoopFileSystem(Utils.scala:1866)
at org.apache.spark.util.Utils$.doFetchFile(Utils.scala:721)
at org.apache.spark.util.Utils$.fetchFile(Utils.scala:496)
at
org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$5.apply(Executor.scala:816)
at
org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$5.apply(Executor.scala:808)
at
scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733)
at
scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130)
at
scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130)
at
scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236)
at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40)
at scala.collection.mutable.HashMap.foreach(HashMap.scala:130)
at
scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732)
at
org.apache.spark.executor.Executor.org$apache$spark$executor$Executor$$updateDependencies(Executor.scala:808)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:375)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.UnknownHostException: xx
... 28 more
{code}
was:
Submit app to K8S, the driver already running but meet exception
"java.net.UnknownHostException: xx" when starting executors.
The udf jar uri using ha style, but the exception stack is
"...*createNonHAProxy*..."
hql:
{code:java}
// code placeholder
add jar hdfs://xx/test.jar;
create temporary function test_udf as 'com.xxx.xxx';
create table test.test_udf as
select test_udf('1') name_1;
{code}
exception:
{code:java}
// code placeholder
TaskSetManager: Lost task 0.0 in stage 0.0 (TID 0, 172.30.89.44, executor 1):
java.lang.IllegalArgumentException: java.net.UnknownHostException: xx
at
org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:439)
at
org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:321)
at
org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:176)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:696)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:636)
at
org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:160)
at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2796)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:99)
at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2830)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2812)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:390)
at org.apache.spark.util.Utils$.getHadoopFileSystem(Utils.scala:1866)
at org.apache.spark.util.Utils$.doFetchFile(Utils.scala:721)
at org.apache.spark.util.Utils$.fetchFile(Utils.scala:496)
at
org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$5.apply(Executor.scala:816)
at
org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$5.apply(Executor.scala:808)
at
scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733)
at
scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130)
at
scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130)
at
scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236)
at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40)
at scala.collection.mutable.HashMap.foreach(HashMap.scala:130)
at
scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732)
at
org.apache.spark.executor.Executor.org$apache$spark$executor$Executor$$updateDependencies(Executor.scala:808)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:375)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.UnknownHostException: xx
... 28 more
{code}
> sparksql 'add jar' not support hdfs ha mode in k8s
> ------------------------------------------------------
>
> Key: SPARK-34754
> URL: https://issues.apache.org/jira/browse/SPARK-34754
> Project: Spark
> Issue Type: Bug
> Components: Kubernetes
> Affects Versions: 2.4.7
> Reporter: lithiumlee-_-
> Priority: Major
>
> Submit app to K8S, the executors meet exception
> "java.net.UnknownHostException: xx".
> The udf jar uri using hdfs ha style, but the exception stack show
> "...*createNonHAProxy*..."
>
> hql:
> {code:java}
> // code placeholder
> add jar hdfs://xx/test.jar;
> create temporary function test_udf as 'com.xxx.xxx';
> create table test.test_udf as
> select test_udf('1') name_1;
> {code}
>
>
> exception:
> {code:java}
> // code placeholder
> TaskSetManager: Lost task 0.0 in stage 0.0 (TID 0, 172.30.89.44, executor
> 1): java.lang.IllegalArgumentException: java.net.UnknownHostException: xx
> at
> org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:439)
> at
> org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:321)
> at
> org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:176)
> at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:696)
> at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:636)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:160)
> at
> org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2796)
> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:99)
> at
> org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2830)
> at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2812)
> at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:390)
> at org.apache.spark.util.Utils$.getHadoopFileSystem(Utils.scala:1866)
> at org.apache.spark.util.Utils$.doFetchFile(Utils.scala:721)
> at org.apache.spark.util.Utils$.fetchFile(Utils.scala:496)
> at
> org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$5.apply(Executor.scala:816)
> at
> org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$5.apply(Executor.scala:808)
> at
> scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733)
> at
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130)
> at
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130)
> at
> scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236)
> at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40)
> at scala.collection.mutable.HashMap.foreach(HashMap.scala:130)
> at
> scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732)
> at
> org.apache.spark.executor.Executor.org$apache$spark$executor$Executor$$updateDependencies(Executor.scala:808)
> at
> org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:375)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.net.UnknownHostException: xx
> ... 28 more
> {code}
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
