[
https://issues.apache.org/jira/browse/FLINK-17159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17218836#comment-17218836
]
Robert Metzger commented on FLINK-17159:
----------------------------------------
Uff. So I was really not able to even reproduce this issue, even though it
seems to happen quite frequently.
Ideally we would reproduce it on DEBUG log level so that we can see why ES is
not able to process the ping request.
Does it make sense to use {{CommonTestUtils.waitUntilCondition()}} and retry
for 30 seconds for the cluster to come up properly?
> ES6 ElasticsearchSinkITCase unstable
> ------------------------------------
>
> Key: FLINK-17159
> URL: https://issues.apache.org/jira/browse/FLINK-17159
> Project: Flink
> Issue Type: Bug
> Components: Connectors / ElasticSearch, Tests
> Affects Versions: 1.11.0, 1.12.0
> Reporter: Chesnay Schepler
> Priority: Critical
> Labels: pull-request-available, test-stability
> Fix For: 1.12.0, 1.11.3
>
>
> [https://dev.azure.com/rmetzger/Flink/_build/results?buildId=7482&view=logs&j=64110e28-73be-50d7-9369-8750330e0bf1&t=aa84fb9a-59ae-5696-70f7-011bc086e59b]
> {code:java}
> 2020-04-15T02:37:04.4289477Z [ERROR]
> testElasticsearchSinkWithSmile(org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSinkITCase)
> Time elapsed: 0.145 s <<< ERROR!
> 2020-04-15T02:37:04.4290310Z
> org.apache.flink.runtime.client.JobExecutionException: Job execution failed.
> 2020-04-15T02:37:04.4290790Z at
> org.apache.flink.runtime.jobmaster.JobResult.toJobExecutionResult(JobResult.java:147)
> 2020-04-15T02:37:04.4291404Z at
> org.apache.flink.runtime.minicluster.MiniCluster.executeJobBlocking(MiniCluster.java:659)
> 2020-04-15T02:37:04.4291956Z at
> org.apache.flink.streaming.util.TestStreamEnvironment.execute(TestStreamEnvironment.java:77)
> 2020-04-15T02:37:04.4292548Z at
> org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1643)
> 2020-04-15T02:37:04.4293254Z at
> org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkTestBase.runElasticSearchSinkTest(ElasticsearchSinkTestBase.java:128)
> 2020-04-15T02:37:04.4293990Z at
> org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkTestBase.runElasticsearchSinkSmileTest(ElasticsearchSinkTestBase.java:106)
> 2020-04-15T02:37:04.4295096Z at
> org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSinkITCase.testElasticsearchSinkWithSmile(ElasticsearchSinkITCase.java:45)
> 2020-04-15T02:37:04.4295923Z at
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 2020-04-15T02:37:04.4296489Z at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 2020-04-15T02:37:04.4297076Z at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 2020-04-15T02:37:04.4297513Z at
> java.lang.reflect.Method.invoke(Method.java:498)
> 2020-04-15T02:37:04.4297951Z at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> 2020-04-15T02:37:04.4298688Z at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> 2020-04-15T02:37:04.4299374Z at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> 2020-04-15T02:37:04.4300069Z at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> 2020-04-15T02:37:04.4300960Z at
> org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
> 2020-04-15T02:37:04.4301705Z at
> org.junit.rules.RunRules.evaluate(RunRules.java:20)
> 2020-04-15T02:37:04.4302204Z at
> org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
> 2020-04-15T02:37:04.4302661Z at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
> 2020-04-15T02:37:04.4303234Z at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
> 2020-04-15T02:37:04.4303706Z at
> org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
> 2020-04-15T02:37:04.4304127Z at
> org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
> 2020-04-15T02:37:04.4304716Z at
> org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
> 2020-04-15T02:37:04.4305394Z at
> org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
> 2020-04-15T02:37:04.4305965Z at
> org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
> 2020-04-15T02:37:04.4306425Z at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> 2020-04-15T02:37:04.4306942Z at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> 2020-04-15T02:37:04.4307466Z at
> org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
> 2020-04-15T02:37:04.4307920Z at
> org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
> 2020-04-15T02:37:04.4308375Z at
> org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
> 2020-04-15T02:37:04.4308782Z at
> org.junit.rules.RunRules.evaluate(RunRules.java:20)
> 2020-04-15T02:37:04.4309182Z at
> org.junit.runners.ParentRunner.run(ParentRunner.java:363)
> 2020-04-15T02:37:04.4310366Z at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365)
> 2020-04-15T02:37:04.4311140Z at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273)
> 2020-04-15T02:37:04.4311721Z at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
> 2020-04-15T02:37:04.4312241Z at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159)
> 2020-04-15T02:37:04.4312796Z at
> org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384)
> 2020-04-15T02:37:04.4313365Z at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345)
> 2020-04-15T02:37:04.4313866Z at
> org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126)
> 2020-04-15T02:37:04.4314361Z at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418)
> 2020-04-15T02:37:04.4315033Z Caused by:
> org.apache.flink.runtime.JobException: Recovery is suppressed by
> NoRestartBackoffTimeStrategy
> 2020-04-15T02:37:04.4315830Z at
> org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:112)
> 2020-04-15T02:37:04.4316563Z at
> org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getFailureHandlingResult(ExecutionFailureHandler.java:78)
> 2020-04-15T02:37:04.4317399Z at
> org.apache.flink.runtime.scheduler.DefaultScheduler.handleTaskFailure(DefaultScheduler.java:189)
> 2020-04-15T02:37:04.4317990Z at
> org.apache.flink.runtime.scheduler.DefaultScheduler.maybeHandleTaskFailure(DefaultScheduler.java:183)
> 2020-04-15T02:37:04.4318606Z at
> org.apache.flink.runtime.scheduler.DefaultScheduler.updateTaskExecutionStateInternal(DefaultScheduler.java:177)
> 2020-04-15T02:37:04.4319199Z at
> org.apache.flink.runtime.scheduler.SchedulerBase.updateTaskExecutionState(SchedulerBase.java:497)
> 2020-04-15T02:37:04.4319882Z at
> org.apache.flink.runtime.jobmaster.JobMaster.updateTaskExecutionState(JobMaster.java:384)
> 2020-04-15T02:37:04.4320391Z at
> sun.reflect.GeneratedMethodAccessor18.invoke(Unknown Source)
> 2020-04-15T02:37:04.4320841Z at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 2020-04-15T02:37:04.4321370Z at
> java.lang.reflect.Method.invoke(Method.java:498)
> 2020-04-15T02:37:04.4321970Z at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:284)
> 2020-04-15T02:37:04.4322519Z at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:199)
> 2020-04-15T02:37:04.4323058Z at
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:74)
> 2020-04-15T02:37:04.4323605Z at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:152)
> 2020-04-15T02:37:04.4324085Z at
> akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:26)
> 2020-04-15T02:37:04.4324493Z at
> akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:21)
> 2020-04-15T02:37:04.4325143Z at
> scala.PartialFunction$class.applyOrElse(PartialFunction.scala:123)
> 2020-04-15T02:37:04.4325736Z at
> akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:21)
> 2020-04-15T02:37:04.4326355Z at
> scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:170)
> 2020-04-15T02:37:04.4327020Z at
> scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
> 2020-04-15T02:37:04.4327649Z at
> scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
> 2020-04-15T02:37:04.4328273Z at
> akka.actor.Actor$class.aroundReceive(Actor.scala:517)
> 2020-04-15T02:37:04.4328892Z at
> akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:225)
> 2020-04-15T02:37:04.4329530Z at
> akka.actor.ActorCell.receiveMessage(ActorCell.scala:592)
> 2020-04-15T02:37:04.4330086Z at
> akka.actor.ActorCell.invoke(ActorCell.scala:561)
> 2020-04-15T02:37:04.4330628Z at
> akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258)
> 2020-04-15T02:37:04.4331371Z at akka.dispatch.Mailbox.run(Mailbox.scala:225)
> 2020-04-15T02:37:04.4332070Z at akka.dispatch.Mailbox.exec(Mailbox.scala:235)
> 2020-04-15T02:37:04.4332649Z at
> akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
> 2020-04-15T02:37:04.4333289Z at
> akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
> 2020-04-15T02:37:04.4334051Z at
> akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 2020-04-15T02:37:04.4335024Z at
> akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 2020-04-15T02:37:04.4336070Z Caused by:
> org.elasticsearch.ElasticsearchStatusException: method [HEAD], host
> [http://127.0.0.1:9200], URI [/], status line [HTTP/1.1 503 Service
> Unavailable]
> 2020-04-15T02:37:04.4337153Z at
> org.elasticsearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:625)
> 2020-04-15T02:37:04.4338036Z at
> org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:535)
> 2020-04-15T02:37:04.4338716Z at
> org.elasticsearch.client.RestHighLevelClient.ping(RestHighLevelClient.java:275)
> 2020-04-15T02:37:04.4339386Z at
> org.apache.flink.streaming.connectors.elasticsearch6.Elasticsearch6ApiCallBridge.verifyClientConnection(Elasticsearch6ApiCallBridge.java:137)
> 2020-04-15T02:37:04.4340137Z at
> org.apache.flink.streaming.connectors.elasticsearch6.Elasticsearch6ApiCallBridge.verifyClientConnection(Elasticsearch6ApiCallBridge.java:47)
> 2020-04-15T02:37:04.4341285Z at
> org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase.open(ElasticsearchSinkBase.java:301)
> 2020-04-15T02:37:04.4341889Z at
> org.apache.flink.api.common.functions.util.FunctionUtils.openFunction(FunctionUtils.java:36)
> 2020-04-15T02:37:04.4342464Z at
> org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.open(AbstractUdfStreamOperator.java:102)
> 2020-04-15T02:37:04.4343022Z at
> org.apache.flink.streaming.api.operators.StreamSink.open(StreamSink.java:48)
> 2020-04-15T02:37:04.4343583Z at
> org.apache.flink.streaming.runtime.tasks.OperatorChain.initializeStateAndOpenOperators(OperatorChain.java:294)
> 2020-04-15T02:37:04.4344168Z at
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:445)
> 2020-04-15T02:37:04.4345048Z at
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:47)
> 2020-04-15T02:37:04.4345632Z at
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:441)
> 2020-04-15T02:37:04.4346152Z at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:462)
> 2020-04-15T02:37:04.4346631Z at
> org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:718)
> 2020-04-15T02:37:04.4347099Z at
> org.apache.flink.runtime.taskmanager.Task.run(Task.java:542)
> 2020-04-15T02:37:04.4347460Z at java.lang.Thread.run(Thread.java:748)
> 2020-04-15T02:37:04.4348018Z Caused by:
> org.elasticsearch.client.ResponseException: method [HEAD], host
> [http://127.0.0.1:9200], URI [/], status line [HTTP/1.1 503 Service
> Unavailable]
> 2020-04-15T02:37:04.4348667Z at
> org.elasticsearch.client.RestClient$SyncResponseListener.get(RestClient.java:705)
> 2020-04-15T02:37:04.4349324Z at
> org.elasticsearch.client.RestClient.performRequest(RestClient.java:235)
> 2020-04-15T02:37:04.4349785Z at
> org.elasticsearch.client.RestClient.performRequest(RestClient.java:198)
> 2020-04-15T02:37:04.4350299Z at
> org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:522)
> 2020-04-15T02:37:04.4350652Z ... 15 more
> 2020-04-15T02:37:04.4351229Z Caused by:
> org.elasticsearch.client.ResponseException: method [HEAD], host
> [http://127.0.0.1:9200], URI [/], status line [HTTP/1.1 503 Service
> Unavailable]
> 2020-04-15T02:37:04.4351858Z at
> org.elasticsearch.client.RestClient$1.completed(RestClient.java:377)
> 2020-04-15T02:37:04.4352286Z at
> org.elasticsearch.client.RestClient$1.completed(RestClient.java:366)
> 2020-04-15T02:37:04.4352743Z at
> org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:119)
> 2020-04-15T02:37:04.4353303Z at
> org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.responseCompleted(DefaultClientExchangeHandlerImpl.java:177)
> 2020-04-15T02:37:04.4354080Z at
> org.apache.http.nio.protocol.HttpAsyncRequestExecutor.processResponse(HttpAsyncRequestExecutor.java:436)
> 2020-04-15T02:37:04.4354827Z at
> org.apache.http.nio.protocol.HttpAsyncRequestExecutor.responseReceived(HttpAsyncRequestExecutor.java:309)
> 2020-04-15T02:37:04.4355434Z at
> org.apache.http.impl.nio.DefaultNHttpClientConnection.consumeInput(DefaultNHttpClientConnection.java:255)
> 2020-04-15T02:37:04.4356012Z at
> org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:81)
> 2020-04-15T02:37:04.4356559Z at
> org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:39)
> 2020-04-15T02:37:04.4357140Z at
> org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:114)
> 2020-04-15T02:37:04.4357661Z at
> org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162)
> 2020-04-15T02:37:04.4358171Z at
> org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337)
> 2020-04-15T02:37:04.4358728Z at
> org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315)
> 2020-04-15T02:37:04.4359267Z at
> org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276)
> 2020-04-15T02:37:04.4363545Z at
> org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
> 2020-04-15T02:37:04.4364304Z at
> org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588)
> 2020-04-15T02:37:04.4365089Z ... 1 more
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
