[ https://issues.apache.org/jira/browse/FLINK-27821?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Hector Miuler Malpica Gallegos updated FLINK-27821: --------------------------------------------------- Description: My operator was installed with following command: ``` git clone g...@github.com:apache/flink-kubernetes-operator.git git checkout 207b17b cd flink-kubernetes-operator helm --debug upgrade -i \ flink-kubernetes-operator helm/flink-kubernetes-operator \ --set image.repository=ghcr.io/apache/flink-kubernetes-operator \ --set image.tag=207b17b ``` Then I create a flinkDeployment and flinkSessionJob, then I deleted the deployment that generated the flinkDeployment, then reinstall operator and finally I wanted to delete the flinkdeployment kubectl logs -f pod/flink-kubernetes-operator-5cf66cbbcb-bpl9p ``` 2022-05-27 13:40:22,027 o.a.f.k.o.c.FlinkDeploymentController [INFO ][flink-system/migration] Deleting FlinkDeployment 2022-05-27 13:40:34,047 o.a.f.s.n.i.n.c.AbstractChannel [WARN ] Force-closing a channel whose registration task was not accepted by an event loop: [id: 0xb2062900] java.util.concurrent.RejectedExecutionException: event executor terminated at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815) at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:483) at org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87) at org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81) at org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86) at org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323) at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155) at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139) at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123) at org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:467) at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390) at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304) at org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863) at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2022-05-27 13:40:34,047 o.a.f.s.n.i.n.u.c.D.rejectedExecution [ERROR] Failed to submit a listener notification task. Event loop shut down? java.util.concurrent.RejectedExecutionException: event executor terminated at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:499) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184) at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95) at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30) at org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:471) at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390) at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304) at org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863) at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [ERROR][flink-system/migration-cosmosdb] REST service in session cluster is bad now java.util.concurrent.TimeoutException at java.base/java.util.concurrent.CompletableFuture.timedGet(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.get(Unknown Source) at org.apache.flink.kubernetes.operator.service.FlinkService.listJobs(FlinkService.java:331) at org.apache.flink.kubernetes.operator.observer.deployment.SessionObserver.observeFlinkCluster(SessionObserver.java:45) at org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:92) at org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:56) at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:101) at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:59) at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:68) at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:50) at io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34) at io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:49) at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:252) at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:72) at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:50) at io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:349) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [INFO ][flink-system/migration-cosmosdb] Observing JobManager deployment. Previous status: READY 2022-05-27 13:40:34,049 o.a.f.k.o.o.d.SessionObserver [ERROR][flink-system/migration-cosmosdb] Missing JobManager deployment ``` was: My operator was installed with following command: ``` git clone g...@github.com:apache/flink-kubernetes-operator.git git checkout 207b17b cd flink-kubernetes-operator helm --debug upgrade -i \ flink-kubernetes-operator helm/flink-kubernetes-operator \ --set image.repository=ghcr.io/apache/flink-kubernetes-operator \ --set image.tag=207b17b ``` Then I create a flinkDeployment and flinkSessionJob, then I delete the deployment of the flinkDeployment, and finally I wanted to delete the flinkdeployment kubectl logs -f pod/flink-kubernetes-operator-5cf66cbbcb-bpl9p ``` 2022-05-27 13:40:22,027 o.a.f.k.o.c.FlinkDeploymentController [INFO ][flink-system/migration] Deleting FlinkDeployment 2022-05-27 13:40:34,047 o.a.f.s.n.i.n.c.AbstractChannel [WARN ] Force-closing a channel whose registration task was not accepted by an event loop: [id: 0xb2062900] java.util.concurrent.RejectedExecutionException: event executor terminated at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815) at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:483) at org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87) at org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81) at org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86) at org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323) at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155) at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139) at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123) at org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:467) at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390) at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304) at org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863) at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2022-05-27 13:40:34,047 o.a.f.s.n.i.n.u.c.D.rejectedExecution [ERROR] Failed to submit a listener notification task. Event loop shut down? java.util.concurrent.RejectedExecutionException: event executor terminated at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:499) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184) at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95) at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30) at org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:471) at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390) at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304) at org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863) at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown Source) at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [ERROR][flink-system/migration-cosmosdb] REST service in session cluster is bad now java.util.concurrent.TimeoutException at java.base/java.util.concurrent.CompletableFuture.timedGet(Unknown Source) at java.base/java.util.concurrent.CompletableFuture.get(Unknown Source) at org.apache.flink.kubernetes.operator.service.FlinkService.listJobs(FlinkService.java:331) at org.apache.flink.kubernetes.operator.observer.deployment.SessionObserver.observeFlinkCluster(SessionObserver.java:45) at org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:92) at org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:56) at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:101) at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:59) at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:68) at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:50) at io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34) at io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:49) at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:252) at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:72) at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:50) at io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:349) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [INFO ][flink-system/migration-cosmosdb] Observing JobManager deployment. Previous status: READY 2022-05-27 13:40:34,049 o.a.f.k.o.o.d.SessionObserver [ERROR][flink-system/migration-cosmosdb] Missing JobManager deployment ``` > Cannot delete flinkdeployment when the pod and deployment deleted manually > -------------------------------------------------------------------------- > > Key: FLINK-27821 > URL: https://issues.apache.org/jira/browse/FLINK-27821 > Project: Flink > Issue Type: Bug > Components: Kubernetes Operator > Affects Versions: kubernetes-operator-1.1.0 > Reporter: Hector Miuler Malpica Gallegos > Priority: Major > > My operator was installed with following command: > ``` > git clone g...@github.com:apache/flink-kubernetes-operator.git git checkout > 207b17b > cd flink-kubernetes-operator > helm --debug upgrade -i \ > flink-kubernetes-operator helm/flink-kubernetes-operator \ > --set image.repository=ghcr.io/apache/flink-kubernetes-operator \ > --set image.tag=207b17b > ``` > Then I create a flinkDeployment and flinkSessionJob, then I deleted the > deployment that generated the flinkDeployment, then reinstall operator and > finally I wanted to delete the flinkdeployment > > kubectl logs -f pod/flink-kubernetes-operator-5cf66cbbcb-bpl9p > ``` > 2022-05-27 13:40:22,027 o.a.f.k.o.c.FlinkDeploymentController [INFO > ][flink-system/migration] Deleting FlinkDeployment > 2022-05-27 13:40:34,047 o.a.f.s.n.i.n.c.AbstractChannel [WARN ] Force-closing > a channel whose registration task was not accepted by an event loop: [id: > 0xb2062900] > java.util.concurrent.RejectedExecutionException: event executor terminated > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815) > at > org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:483) > at > org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87) > at > org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81) > at > org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86) > at > org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323) > at > org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155) > at > org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139) > at > org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123) > at > org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:467) > at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390) > at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304) > at > org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863) > at > java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown > Source) > at > java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source) > at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown > Source) > at > java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown > Source) > at > java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown > Source) > at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) > at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) > at java.base/java.lang.Thread.run(Unknown Source) > 2022-05-27 13:40:34,047 o.a.f.s.n.i.n.u.c.D.rejectedExecution [ERROR] Failed > to submit a listener notification task. Event loop shut down? > java.util.concurrent.RejectedExecutionException: event executor terminated > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:499) > at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184) > at > org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95) > at > org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30) > at > org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:471) > at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390) > at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304) > at > org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863) > at > java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown > Source) > at > java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source) > at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown > Source) > at > java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown > Source) > at > java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown > Source) > at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) > at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) > at java.base/java.lang.Thread.run(Unknown Source) > 2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver > [ERROR][flink-system/migration-cosmosdb] REST service in session cluster is > bad now > java.util.concurrent.TimeoutException > at java.base/java.util.concurrent.CompletableFuture.timedGet(Unknown > Source) > at java.base/java.util.concurrent.CompletableFuture.get(Unknown > Source) > at > org.apache.flink.kubernetes.operator.service.FlinkService.listJobs(FlinkService.java:331) > at > org.apache.flink.kubernetes.operator.observer.deployment.SessionObserver.observeFlinkCluster(SessionObserver.java:45) > at > org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:92) > at > org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:56) > at > org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:101) > at > org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:59) > at > io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:68) > at > io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:50) > at > io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34) > at > io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:49) > at > io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:252) > at > io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:72) > at > io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:50) > at > io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:349) > at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) > at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) > at java.base/java.lang.Thread.run(Unknown Source) > 2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [INFO > ][flink-system/migration-cosmosdb] Observing JobManager deployment. Previous > status: READY > 2022-05-27 13:40:34,049 o.a.f.k.o.o.d.SessionObserver > [ERROR][flink-system/migration-cosmosdb] Missing JobManager deployment > ``` -- This message was sent by Atlassian Jira (v8.20.7#820007)