[ https://issues.apache.org/jira/browse/AMBARI-11506?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Jeff Sposetti updated AMBARI-11506: ----------------------------------- Fix Version/s: (was: 2.0.1) 2.0.2 > Heartbeats lost during large cluster deploy > ------------------------------------------- > > Key: AMBARI-11506 > URL: https://issues.apache.org/jira/browse/AMBARI-11506 > Project: Ambari > Issue Type: Bug > Components: ambari-server > Affects Versions: 2.0.1, 2.1.0 > Reporter: Myroslav Papirkovskyy > Assignee: Myroslav Papirkovskyy > Priority: Critical > Fix For: 2.1.0, 2.0.2 > > Attachments: AMBARI-11506_branch-2.0.maint.patch > > > During deploy random hosts often encounter HB lost. > Sometimes with EOF exception: > {code} > 27 May 2015 20:06:42,964 WARN [qtp-ambari-agent-185] ServletHandler:563 - > /agent/v1/heartbeat/perf400-c-326.c.pramod-thangali.internal > com.google.gson.JsonIOException: org.eclipse.jetty.io.EofException > at com.google.gson.Gson.toJson(Gson.java:588) > at com.google.gson.Gson.toJson(Gson.java:565) > at > org.apache.ambari.server.api.GsonJsonProvider.writeTo(GsonJsonProvider.java:83) > at > com.sun.jersey.spi.container.ContainerResponse.write(ContainerResponse.java:306) > at > com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1437) > at > com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349) > at > com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339) > at > com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416) > at > com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537) > at > com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:708) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:770) > at > org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684) > at > org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1496) > at > org.apache.ambari.server.security.SecurityFilter.doFilter(SecurityFilter.java:67) > at > org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) > at > org.apache.ambari.server.api.AmbariPersistFilter.doFilter(AmbariPersistFilter.java:47) > at > org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) > at > org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501) > at > org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) > at > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429) > at > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) > at org.eclipse.jetty.server.Server.handle(Server.java:370) > at > org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494) > at > org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:982) > at > org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1043) > at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) > at > org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) > at > org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) > at > org.eclipse.jetty.io.nio.SslConnection.handle(SslConnection.java:196) > at > org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) > at > org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) > at > org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) > at > org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.eclipse.jetty.io.EofException > at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:142) > at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107) > at > com.sun.jersey.spi.container.servlet.WebComponent$Writer.write(WebComponent.java:307) > at > com.sun.jersey.spi.container.ContainerResponse$CommittingOutputStream.write(ContainerResponse.java:134) > at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) > at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282) > at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125) > at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135) > at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220) > at com.google.gson.stream.JsonWriter.string(JsonWriter.java:544) > at com.google.gson.stream.JsonWriter.value(JsonWriter.java:404) > at > com.google.gson.internal.bind.TypeAdapters$13.write(TypeAdapters.java:353) > at > com.google.gson.internal.bind.TypeAdapters$13.write(TypeAdapters.java:337) > at > com.google.gson.internal.bind.TypeAdapterRuntimeTypeWrapper.write(TypeAdapterRuntimeTypeWrapper.java:68) > at > com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$1.write(ReflectiveTypeAdapterFactory.java:89) > at > com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$Adapter.write(ReflectiveTypeAdapterFactory.java:195) > at > com.google.gson.internal.bind.TypeAdapterRuntimeTypeWrapper.write(TypeAdapterRuntimeTypeWrapper.java:68) > at > com.google.gson.internal.bind.CollectionTypeAdapterFactory$Adapter.write(CollectionTypeAdapterFactory.java:96) > at > com.google.gson.internal.bind.CollectionTypeAdapterFactory$Adapter.write(CollectionTypeAdapterFactory.java:60) > at > com.google.gson.internal.bind.TypeAdapterRuntimeTypeWrapper.write(TypeAdapterRuntimeTypeWrapper.java:68) > at > com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$1.write(ReflectiveTypeAdapterFactory.java:89) > at > com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$Adapter.write(ReflectiveTypeAdapterFactory.java:195) > at > com.google.gson.internal.bind.TypeAdapterRuntimeTypeWrapper.write(TypeAdapterRuntimeTypeWrapper.java:68) > at > com.google.gson.internal.bind.CollectionTypeAdapterFactory$Adapter.write(CollectionTypeAdapterFactory.java:96) > at > com.google.gson.internal.bind.CollectionTypeAdapterFactory$Adapter.write(CollectionTypeAdapterFactory.java:60) > at > com.google.gson.internal.bind.TypeAdapterRuntimeTypeWrapper.write(TypeAdapterRuntimeTypeWrapper.java:68) > at > com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$1.write(ReflectiveTypeAdapterFactory.java:89) > at > com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$Adapter.write(ReflectiveTypeAdapterFactory.java:195) > at com.google.gson.Gson.toJson(Gson.java:586) > ... 35 more > {code} > And following entry on agent side: > {code} > "INFO 2015-05-27 20:06:02,123 security.py:135 - Encountered communication > error. Details: SSLError('The read operation timed out',) > ERROR 2015-05-27 20:06:02,123 Controller.py:330 - Connection to perf400-a-1 > was lost (details=Request to > https://perf400-a-1:8441/agent/v1/heartbeat/perf400-c-326.c.pramod-thangali.internal > failed due to Error occured during connecting to the server: The read > operation timed out)" > {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332)