[
https://issues.apache.org/jira/browse/FLINK-27821?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Hector Miuler Malpica Gallegos updated FLINK-27821:
---------------------------------------------------
Description:
My operator was installed with following command:
{{git clone g...@github.com:apache/flink-kubernetes-operator.git}}
{{git checkout 207b17b}}
{{cd flink-kubernetes-operator}}
{{helm -debug upgrade -i flink-kubernetes-operator
helm/flink-kubernetes-operator --set
image.repository=ghcr.io/apache/flink-kubernetes-operator -set
image.tag=207b17b}}
Then I create a flinkDeployment and flinkSessionJob, then I deleted the
deployment that generated the flinkDeployment, then reinstall operator and
finally I wanted to delete the flinkdeployment
kubectl logs -f pod/flink-kubernetes-operator-5cf66cbbcb-bpl9p
{{2022-05-27 13:40:22,027 o.a.f.k.o.c.FlinkDeploymentController [INFO
][flink-system/migration] Deleting FlinkDeployment}}
{{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.c.AbstractChannel [WARN ] Force-closing
a channel whose registration task was not accepted by an event loop: [id:
0xb2062900]}}
{{java.util.concurrent.RejectedExecutionException: event executor terminated}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:483)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123)}}
{{ at
org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:467)}}
{{ at
org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}}
{{ at
org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}}
{{ at
org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)}}
{{ at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}}
{{ at java.base/java.lang.Thread.run(Unknown Source)}}
{{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.u.c.D.rejectedExecution [ERROR] Failed
to submit a listener notification task. Event loop shut down?}}
{{java.util.concurrent.RejectedExecutionException: event executor terminated}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:499)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30)}}
{{ at
org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:471)}}
{{ at
org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}}
{{ at
org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}}
{{ at
org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)}}
{{ at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}}
{{ at java.base/java.lang.Thread.run(Unknown Source)}}
{{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver
[ERROR][flink-system/migration-cosmosdb] REST service in session cluster is bad
now}}
{{java.util.concurrent.TimeoutException}}
{{ at java.base/java.util.concurrent.CompletableFuture.timedGet(Unknown
Source)}}
{{ at java.base/java.util.concurrent.CompletableFuture.get(Unknown
Source)}}
{{ at
org.apache.flink.kubernetes.operator.service.FlinkService.listJobs(FlinkService.java:331)}}
{{ at
org.apache.flink.kubernetes.operator.observer.deployment.SessionObserver.observeFlinkCluster(SessionObserver.java:45)}}
{{ at
org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:92)}}
{{ at
org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:56)}}
{{ at
org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:101)}}
{{ at
org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:59)}}
{{ at
io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:68)}}
{{ at
io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:50)}}
{{ at
io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34)}}
{{ at
io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:49)}}
{{ at
io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:252)}}
{{ at
io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:72)}}
{{ at
io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:50)}}
{{ at
io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:349)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}}
{{ at java.base/java.lang.Thread.run(Unknown Source)}}
{{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [INFO
][flink-system/migration-cosmosdb] Observing JobManager deployment. Previous
status: READY}}
{{2022-05-27 13:40:34,049 o.a.f.k.o.o.d.SessionObserver
[ERROR][flink-system/migration-cosmosdb] Missing JobManager deployment}}
was:
My operator was installed with following command:
{{git clone g...@github.com:apache/flink-kubernetes-operator.git}}
{{git checkout 207b17b}}
{{cd flink-kubernetes-operator}}
{{helm -debug upgrade -i flink-kubernetes-operator
helm/flink-kubernetes-operator --set
image.repository=ghcr.io/apache/flink-kubernetes-operator -set
image.tag=207b17b}}
Then I create a flinkDeployment and flinkSessionJob, then I deleted the
deployment that generated the flinkDeployment, then reinstall operator and
finally I wanted to delete the flinkdeployment
kubectl logs -f pod/flink-kubernetes-operator-5cf66cbbcb-bpl9p
{{2022-05-27 13:40:22,027 o.a.f.k.o.c.FlinkDeploymentController [INFO
][flink-system/migration] Deleting FlinkDeployment}}
{{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.c.AbstractChannel [WARN ] Force-closing
a channel whose registration task was not accepted by an event loop: [id:
0xb2062900]}}
{{java.util.concurrent.RejectedExecutionException: event executor terminated}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:483)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123)}}
{{ at
org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:467)}}
{{ at
org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}}
{{ at
org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}}
{{ at
org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)}}
{{ at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}}
{{ at java.base/java.lang.Thread.run(Unknown Source)}}
{{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.u.c.D.rejectedExecution [ERROR] Failed
to submit a listener notification task. Event loop shut down?}}
{{java.util.concurrent.RejectedExecutionException: event executor terminated}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:499)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95)}}
{{ at
org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30)}}
{{ at
org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:471)}}
{{ at
org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}}
{{ at
org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}}
{{ at
org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)}}
{{ at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown
Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}}
{{ at java.base/java.lang.Thread.run(Unknown Source)}}
{{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver
[ERROR][flink-system/migration-cosmosdb] REST service in session cluster is bad
now}}
{{java.util.concurrent.TimeoutException}}
{{ at java.base/java.util.concurrent.CompletableFuture.timedGet(Unknown
Source)}}
{{ at java.base/java.util.concurrent.CompletableFuture.get(Unknown
Source)}}
{{ at
org.apache.flink.kubernetes.operator.service.FlinkService.listJobs(FlinkService.java:331)}}
{{ at
org.apache.flink.kubernetes.operator.observer.deployment.SessionObserver.observeFlinkCluster(SessionObserver.java:45)}}
{{ at
org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:92)}}
{{ at
org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:56)}}
{{ at
org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:101)}}
{{ at
org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:59)}}
{{ at
io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:68)}}
{{ at
io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:50)}}
{{ at
io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34)}}
{{ at
io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:49)}}
{{ at
io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:252)}}
{{ at
io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:72)}}
{{ at
io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:50)}}
{{ at
io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:349)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}}
{{ at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}}
{{ at java.base/java.lang.Thread.run(Unknown Source)}}
{{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [INFO
][flink-system/migration-cosmosdb] Observing JobManager deployment. Previous
status: READY}}
{{2022-05-27 13:40:34,049 o.a.f.k.o.o.d.SessionObserver
[ERROR][flink-system/migration-cosmosdb] Missing JobManager deployment}}
> Cannot delete flinkdeployment when the pod and deployment deleted manually
> --------------------------------------------------------------------------
>
> Key: FLINK-27821
> URL: https://issues.apache.org/jira/browse/FLINK-27821
> Project: Flink
> Issue Type: Bug
> Components: Kubernetes Operator
> Affects Versions: kubernetes-operator-1.1.0
> Reporter: Hector Miuler Malpica Gallegos
> Priority: Major
>
> My operator was installed with following command:
>
> {{git clone g...@github.com:apache/flink-kubernetes-operator.git}}
> {{git checkout 207b17b}}
> {{cd flink-kubernetes-operator}}
> {{helm -debug upgrade -i flink-kubernetes-operator
> helm/flink-kubernetes-operator --set
> image.repository=ghcr.io/apache/flink-kubernetes-operator -set
> image.tag=207b17b}}
>
> Then I create a flinkDeployment and flinkSessionJob, then I deleted the
> deployment that generated the flinkDeployment, then reinstall operator and
> finally I wanted to delete the flinkdeployment
>
> kubectl logs -f pod/flink-kubernetes-operator-5cf66cbbcb-bpl9p
>
> {{2022-05-27 13:40:22,027 o.a.f.k.o.c.FlinkDeploymentController [INFO
> ][flink-system/migration] Deleting FlinkDeployment}}
> {{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.c.AbstractChannel [WARN ]
> Force-closing a channel whose registration task was not accepted by an event
> loop: [id: 0xb2062900]}}
> {{java.util.concurrent.RejectedExecutionException: event executor terminated}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:483)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123)}}
> {{ at
> org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:467)}}
> {{ at
> org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}}
> {{ at
> org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}}
> {{ at
> org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown
> Source)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown
> Source)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown
> Source)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown
> Source)}}
> {{ at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}}
> {{ at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}}
> {{ at java.base/java.lang.Thread.run(Unknown Source)}}
> {{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.u.c.D.rejectedExecution [ERROR]
> Failed to submit a listener notification task. Event loop shut down?}}
> {{java.util.concurrent.RejectedExecutionException: event executor terminated}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:499)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95)}}
> {{ at
> org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30)}}
> {{ at
> org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:471)}}
> {{ at
> org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}}
> {{ at
> org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}}
> {{ at
> org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown
> Source)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown
> Source)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown
> Source)}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown
> Source)}}
> {{ at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}}
> {{ at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}}
> {{ at java.base/java.lang.Thread.run(Unknown Source)}}
> {{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver
> [ERROR][flink-system/migration-cosmosdb] REST service in session cluster is
> bad now}}
> {{java.util.concurrent.TimeoutException}}
> {{ at
> java.base/java.util.concurrent.CompletableFuture.timedGet(Unknown Source)}}
> {{ at java.base/java.util.concurrent.CompletableFuture.get(Unknown
> Source)}}
> {{ at
> org.apache.flink.kubernetes.operator.service.FlinkService.listJobs(FlinkService.java:331)}}
> {{ at
> org.apache.flink.kubernetes.operator.observer.deployment.SessionObserver.observeFlinkCluster(SessionObserver.java:45)}}
> {{ at
> org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:92)}}
> {{ at
> org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:56)}}
> {{ at
> org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:101)}}
> {{ at
> org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:59)}}
> {{ at
> io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:68)}}
> {{ at
> io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:50)}}
> {{ at
> io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34)}}
> {{ at
> io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:49)}}
> {{ at
> io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:252)}}
> {{ at
> io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:72)}}
> {{ at
> io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:50)}}
> {{ at
> io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:349)}}
> {{ at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}}
> {{ at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}}
> {{ at java.base/java.lang.Thread.run(Unknown Source)}}
> {{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [INFO
> ][flink-system/migration-cosmosdb] Observing JobManager deployment. Previous
> status: READY}}
> {{2022-05-27 13:40:34,049 o.a.f.k.o.o.d.SessionObserver
> [ERROR][flink-system/migration-cosmosdb] Missing JobManager deployment}}
>
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
