Yang Wang created FLINK-16499:
---------------------------------
Summary: Flink shaded hadoop could not work when Yarn timeline
service is enabled
Key: FLINK-16499
URL: https://issues.apache.org/jira/browse/FLINK-16499
Project: Flink
Issue Type: Bug
Components: BuildSystem / Shaded
Reporter: Yang Wang
When the Yarn timeline service is enabled (via
{{yarn.timeline-service.enabled=true}} in yarn-site.xml), flink-shaded-hadoop
could not work to submit Flink job to Yarn cluster. The following exception
will be thrown.
The root cause is the {{jersey-core-*.jar}} is not bundled into
{{flink-shaded-hadoop-*.jar}}.
{code:java}
2020-03-09 03:35:34,396 ERROR org.apache.flink.client.cli.CliFrontend
[] - Fatal error while running command line interface.2020-03-09
03:35:34,396 ERROR org.apache.flink.client.cli.CliFrontend
[] - Fatal error while running command line
interface.java.lang.NoClassDefFoundError: javax/ws/rs/ext/MessageBodyReader at
java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_242] at
java.lang.ClassLoader.defineClass(ClassLoader.java:757) ~[?:1.8.0_242] at
java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
~[?:1.8.0_242] at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
~[?:1.8.0_242] at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
~[?:1.8.0_242] at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
~[?:1.8.0_242] at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
~[?:1.8.0_242] at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_242] at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
~[?:1.8.0_242] at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
~[?:1.8.0_242] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
~[?:1.8.0_242] at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
~[?:1.8.0_242] at java.lang.ClassLoader.defineClass1(Native Method)
~[?:1.8.0_242] at java.lang.ClassLoader.defineClass(ClassLoader.java:757)
~[?:1.8.0_242] at
java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
~[?:1.8.0_242] at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
~[?:1.8.0_242] at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
~[?:1.8.0_242] at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
~[?:1.8.0_242] at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
~[?:1.8.0_242] at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_242] at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
~[?:1.8.0_242] at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
~[?:1.8.0_242] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
~[?:1.8.0_242] at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
~[?:1.8.0_242] at java.lang.ClassLoader.defineClass1(Native Method)
~[?:1.8.0_242] at java.lang.ClassLoader.defineClass(ClassLoader.java:757)
~[?:1.8.0_242] at
java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
~[?:1.8.0_242] at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
~[?:1.8.0_242] at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
~[?:1.8.0_242] at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
~[?:1.8.0_242] at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
~[?:1.8.0_242] at java.security.AccessController.doPrivileged(Native Method)
~[?:1.8.0_242] at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
~[?:1.8.0_242] at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
~[?:1.8.0_242] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
~[?:1.8.0_242] at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
~[?:1.8.0_242] at
org.apache.hadoop.yarn.util.timeline.TimelineUtils.<clinit>(TimelineUtils.java:50)
~[flink-shaded-hadoop-2-uber-2.8.3-7.0.jar:2.8.3-7.0] at
org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:179)
~[flink-shaded-hadoop-2-uber-2.8.3-7.0.jar:2.8.3-7.0] at
org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
~[flink-shaded-hadoop-2-uber-2.8.3-7.0.jar:2.8.3-7.0] at
org.apache.flink.yarn.YarnClusterClientFactory.getClusterDescriptor(YarnClusterClientFactory.java:71)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.yarn.YarnClusterClientFactory.createClusterDescriptor(YarnClusterClientFactory.java:56)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.yarn.YarnClusterClientFactory.createClusterDescriptor(YarnClusterClientFactory.java:42)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.client.deployment.executors.AbstractJobClusterExecutor.execute(AbstractJobClusterExecutor.java:64)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.executeAsync(StreamExecutionEnvironment.java:1733)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.streaming.api.environment.StreamContextEnvironment.executeAsync(StreamContextEnvironment.java:94)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.streaming.api.environment.StreamContextEnvironment.execute(StreamContextEnvironment.java:63)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1620)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1602)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
main.Job.main(Job.java:28) ~[?:?] at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_242] at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_242] at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_242] at java.lang.reflect.Method.invoke(Method.java:498)
~[?:1.8.0_242] at
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:321)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:205)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:138)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:664)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:213)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:895)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:968)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_242] at
javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_242] at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836)
~[flink-shaded-hadoop-2-uber-2.8.3-7.0.jar:2.8.3-7.0] at
org.apache.flink.runtime.security.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)
~[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT] at
org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:968)
[flink-dist_2.11-1.10.0-vvr-0.1-SNAPSHOT.jar:1.10.0-vvr-0.1-SNAPSHOT]Caused by:
java.lang.ClassNotFoundException: javax.ws.rs.ext.MessageBodyReader at
java.net.URLClassLoader.findClass(URLClassLoader.java:382) ~[?:1.8.0_242] at
java.lang.ClassLoader.loadClass(ClassLoader.java:419) ~[?:1.8.0_242] at
sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_242] at
java.lang.ClassLoader.loadClass(ClassLoader.java:352) ~[?:1.8.0_242] ... 65
morejava.lang.NoClassDefFoundError: javax/ws/rs/ext/MessageBodyReader at
java.lang.ClassLoader.defineClass1(Native Method) at
java.lang.ClassLoader.defineClass(ClassLoader.java:757) at
java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at
java.net.URLClassLoader.defineClass(URLClassLoader.java:468) at
java.net.URLClassLoader.access$100(URLClassLoader.java:74) at
java.net.URLClassLoader$1.run(URLClassLoader.java:369) at
java.net.URLClassLoader$1.run(URLClassLoader.java:363) at
java.security.AccessController.doPrivileged(Native Method) at
java.net.URLClassLoader.findClass(URLClassLoader.java:362) at
java.lang.ClassLoader.loadClass(ClassLoader.java:419) at
sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) at
java.lang.ClassLoader.loadClass(ClassLoader.java:352) at
java.lang.ClassLoader.defineClass1(Native Method) at
java.lang.ClassLoader.defineClass(ClassLoader.java:757) at
java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at
java.net.URLClassLoader.defineClass(URLClassLoader.java:468) at
java.net.URLClassLoader.access$100(URLClassLoader.java:74) at
java.net.URLClassLoader$1.run(URLClassLoader.java:369) at
java.net.URLClassLoader$1.run(URLClassLoader.java:363) at
java.security.AccessController.doPrivileged(Native Method) at
java.net.URLClassLoader.findClass(URLClassLoader.java:362) at
java.lang.ClassLoader.loadClass(ClassLoader.java:419) at
sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) at
java.lang.ClassLoader.loadClass(ClassLoader.java:352) at
java.lang.ClassLoader.defineClass1(Native Method) at
java.lang.ClassLoader.defineClass(ClassLoader.java:757) at
java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at
java.net.URLClassLoader.defineClass(URLClassLoader.java:468) at
java.net.URLClassLoader.access$100(URLClassLoader.java:74) at
java.net.URLClassLoader$1.run(URLClassLoader.java:369) at
java.net.URLClassLoader$1.run(URLClassLoader.java:363) at
java.security.AccessController.doPrivileged(Native Method) at
java.net.URLClassLoader.findClass(URLClassLoader.java:362) at
java.lang.ClassLoader.loadClass(ClassLoader.java:419) at
sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) at
java.lang.ClassLoader.loadClass(ClassLoader.java:352) at
org.apache.hadoop.yarn.util.timeline.TimelineUtils.<clinit>(TimelineUtils.java:50)
at
org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:179)
at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163) at
org.apache.flink.yarn.YarnClusterClientFactory.getClusterDescriptor(YarnClusterClientFactory.java:71)
at
org.apache.flink.yarn.YarnClusterClientFactory.createClusterDescriptor(YarnClusterClientFactory.java:56)
at
org.apache.flink.yarn.YarnClusterClientFactory.createClusterDescriptor(YarnClusterClientFactory.java:42)
at
org.apache.flink.client.deployment.executors.AbstractJobClusterExecutor.execute(AbstractJobClusterExecutor.java:64)
at
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.executeAsync(StreamExecutionEnvironment.java:1733)
at
org.apache.flink.streaming.api.environment.StreamContextEnvironment.executeAsync(StreamContextEnvironment.java:94)
at
org.apache.flink.streaming.api.environment.StreamContextEnvironment.execute(StreamContextEnvironment.java:63)
at
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1620)
at
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1602)
at main.Job.main(Job.java:28) at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:321)
at
org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:205)
at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:138) at
org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:664) at
org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:213) at
org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:895)
at org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:968)
at java.security.AccessController.doPrivileged(Native Method) at
javax.security.auth.Subject.doAs(Subject.java:422) at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836)
at
org.apache.flink.runtime.security.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)
at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:968)Caused
by: java.lang.ClassNotFoundException: javax.ws.rs.ext.MessageBodyReader at
java.net.URLClassLoader.findClass(URLClassLoader.java:382) at
java.lang.ClassLoader.loadClass(ClassLoader.java:419) at
sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) at
java.lang.ClassLoader.loadClass(ClassLoader.java:352) ... 65 more
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
