[
http://issues.apache.org/jira/browse/HIVEMIND-162?page=comments#action_12359617
]
Jeff Lubetkin commented on HIVEMIND-162:
----------------------------------------
Well, this is an improvement, but not a huge one (510 TPS now, still about 45%
cpu). We seem to now be hitting a sync bottleneck on the classloader. Of 22
http daemon threads, 17 are blocked on java.lang.Class.forName0, a native
method (see Stack #1). One is actually inside that lock (see Stack #2) and the
rest are either waiting for a request to handle, or handling an actual request.
Given that the thread inside the lock is doing filesystem stuff, could we be
I/O bound here?
=========== STACK #1 ============
[java] "http-0.0.0.0-8080-22" daemon prio=1 tid=0x8ae379d8 nid=0x38c4
waiting for monitor entry [0x8a91f000..0x8a9204c0]
[java] at java.lang.Class.forName0(Native Method)
[java] at java.lang.Class.forName(Class.java:242)
[java] at
org.apache.hivemind.impl.DefaultClassResolver.lookupClass(DefaultClassResolver.java:101)
[java] at
org.apache.hivemind.impl.DefaultClassResolver.checkForClass(DefaultClassResolver.java:108)
[java] at
org.apache.hivemind.impl.ModuleImpl.resolveType(ModuleImpl.java:191)
[java] at
org.apache.hivemind.util.InstanceCreationUtils.createInstance(InstanceCreationUtils.java:50)
[java] at
org.apache.hivemind.impl.CreateClassServiceConstructor.constructCoreServiceImplementation(CreateClassServiceConstructor.java:35)
[java] at
org.apache.hivemind.impl.servicemodel.AbstractServiceModelImpl.constructCoreServiceImplementation(AbstractServiceModelImpl.java:108)
[java] at
org.apache.hivemind.impl.servicemodel.ThreadedServiceModel.constructInstanceForCurrentThread(ThreadedServiceModel.java:164)
[java] at
org.apache.hivemind.impl.servicemodel.ThreadedServiceModel.getServiceImplementationForCurrentThread(ThreadedServiceModel.java:155)
[java] at
$RequestGlobals_108065173f1._service($RequestGlobals_108065173f1.java)
[java] at
$RequestGlobals_108065173f1.store($RequestGlobals_108065173f1.java)
[java] at
$RequestGlobals_108065173f2.store($RequestGlobals_108065173f2.java)
[java] at
org.apache.tapestry.services.impl.WebRequestServicerPipelineBridge.service(WebRequestServicerPipelineBridge.java:49)
[java] at
$ServletRequestServicer_108065173d7.service($ServletRequestServicer_108065173d7.java)
[java] at
org.apache.tapestry.request.DecodedRequestInjector.service(DecodedRequestInjector.java:55)
[java] at
$ServletRequestServicerFilter_108065173d3.service($ServletRequestServicerFilter_108065173d3.java)
[java] at
$ServletRequestServicer_108065173d9.service($ServletRequestServicer_108065173d9.java)
[java] at
org.apache.tapestry.multipart.MultipartDecoderFilter.service(MultipartDecoderFilter.java:52)
[java] at
$ServletRequestServicerFilter_108065173d1.service($ServletRequestServicerFilter_108065173d1.java)
[java] at
$ServletRequestServicer_108065173d9.service($ServletRequestServicer_108065173d9.java)
[java] at
org.apache.tapestry.services.impl.SetupRequestEncoding.service(SetupRequestEncoding.java:53)
[java] at
$ServletRequestServicerFilter_108065173d5.service($ServletRequestServicerFilter_108065173d5.java)
[java] at
$ServletRequestServicer_108065173d9.service($ServletRequestServicer_108065173d9.java)
[java] at
$ServletRequestServicer_108065173cb.service($ServletRequestServicer_108065173cb.java)
[java] at
org.apache.tapestry.ApplicationServlet.doService(ApplicationServlet.java:123)
[java] at com.zillow.web.ZillowServlet.doService(ZillowServlet.java:35)
[java] at
org.apache.tapestry.ApplicationServlet.doGet(ApplicationServlet.java:79)
[java] at javax.servlet.http.HttpServlet.service(HttpServlet.java:697)
[java] at javax.servlet.http.HttpServlet.service(HttpServlet.java:810)
[java] at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
[java] at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
[java] at
com.zillow.web.filters.JSessionIDRemovalFilter.doFilter(JSessionIDRemovalFilter.java:73)
[java] at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
[java] at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
[java] at
com.zillow.web.filters.RequestMeteringFilter.doFilter(RequestMeteringFilter.java:31)
[java] at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
[java] at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
[java] at
org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:81)
[java] at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
[java] at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
[java] at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
[java] at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
[java] at
org.jboss.web.tomcat.security.CustomPrincipalValve.invoke(CustomPrincipalValve.java:39)
[java] at
org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:159)
[java] at
org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:59)
[java] at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
[java] at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
[java] at
org.apache.catalina.valves.FastCommonAccessLogValve.invoke(FastCommonAccessLogValve.java:481)
[java] at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
[java] at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
[java] at
org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:856)
[java] at
org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:744)
[java] at
org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
[java] at
org.apache.tomcat.util.net.MasterSlaveWorkerThread.run(MasterSlaveWorkerThread.java:112)
[java] at java.lang.Thread.run(Thread.java:595)
=========== STACK #2 ============
[java] "http-0.0.0.0-8080-19" daemon prio=1 tid=0x89e1b588 nid=0x38c0
runnable [0x87f7e000..0x87f7f6c0]
[java] at java.io.UnixFileSystem.getBooleanAttributes0(Native Method)
[java] at
java.io.UnixFileSystem.getBooleanAttributes(UnixFileSystem.java:228)
[java] at java.io.File.exists(File.java:702)
[java] at
sun.misc.URLClassPath$FileLoader.getResource(URLClassPath.java:893)
[java] at sun.misc.URLClassPath.getResource(URLClassPath.java:161)
[java] at java.net.URLClassLoader$1.run(URLClassLoader.java:192)
[java] at java.security.AccessController.doPrivileged(Native Method)
[java] at java.net.URLClassLoader.findClass(URLClassLoader.java:188)
[java] at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
[java] - locked <0x91d20188> (a sun.misc.Launcher$AppClassLoader)
[java] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:268)
[java] - locked <0x91d20188> (a sun.misc.Launcher$AppClassLoader)
[java] at java.lang.ClassLoader.loadClass(ClassLoader.java:251)
[java] at
org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1247)
[java] at
org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1181)
[java] at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:319)
[java] - locked <0x9216efb0> (a
org.jboss.web.tomcat.tc5.WebAppClassLoader)
[java] at java.lang.Class.forName0(Native Method)
[java] at java.lang.Class.forName(Class.java:242)
[java] at
org.apache.hivemind.impl.DefaultClassResolver.lookupClass(DefaultClassResolver.java:101)
[java] at
org.apache.hivemind.impl.DefaultClassResolver.checkForClass(DefaultClassResolver.java:108)
[java] at
org.apache.hivemind.impl.ModuleImpl.resolveType(ModuleImpl.java:191)
[java] at
org.apache.hivemind.util.InstanceCreationUtils.createInstance(InstanceCreationUtils.java:50)
[java] at
org.apache.hivemind.impl.CreateClassServiceConstructor.constructCoreServiceImplementation(CreateClassServiceConstructor.java:35)
[java] at
org.apache.hivemind.impl.servicemodel.AbstractServiceModelImpl.constructCoreServiceImplementation(AbstractServiceModelImpl.java:108)
[java] at
org.apache.hivemind.impl.servicemodel.ThreadedServiceModel.constructInstanceForCurrentThread(ThreadedServiceModel.java:164)
[java] at
org.apache.hivemind.impl.servicemodel.ThreadedServiceModel.getServiceImplementationForCurrentThread(ThreadedServiceModel.java:155)
[java] at
$RequestGlobals_108065173f1._service($RequestGlobals_108065173f1.java)
[java] at
$RequestGlobals_108065173f1.store($RequestGlobals_108065173f1.java)
[java] at
$RequestGlobals_108065173f2.store($RequestGlobals_108065173f2.java)
[java] at
org.apache.tapestry.services.impl.WebRequestServicerPipelineBridge.service(WebRequestServicerPipelineBridge.java:49)
[java] at
$ServletRequestServicer_108065173d7.service($ServletRequestServicer_108065173d7.java)
[java] at
org.apache.tapestry.request.DecodedRequestInjector.service(DecodedRequestInjector.java:55)
[java] at
$ServletRequestServicerFilter_108065173d3.service($ServletRequestServicerFilter_108065173d3.java)
[java] at
$ServletRequestServicer_108065173d9.service($ServletRequestServicer_108065173d9.java)
[java] at
org.apache.tapestry.multipart.MultipartDecoderFilter.service(MultipartDecoderFilter.java:52)
[java] at
$ServletRequestServicerFilter_108065173d1.service($ServletRequestServicerFilter_108065173d1.java)
[java] at
$ServletRequestServicer_108065173d9.service($ServletRequestServicer_108065173d9.java)
[java] at
org.apache.tapestry.services.impl.SetupRequestEncoding.service(SetupRequestEncoding.java:53)
[java] at
$ServletRequestServicerFilter_108065173d5.service($ServletRequestServicerFilter_108065173d5.java)
[java] at
$ServletRequestServicer_108065173d9.service($ServletRequestServicer_108065173d9.java)
[java] at
$ServletRequestServicer_108065173cb.service($ServletRequestServicer_108065173cb.java)
[java] at
org.apache.tapestry.ApplicationServlet.doService(ApplicationServlet.java:123)
[java] at com.zillow.web.ZillowServlet.doService(ZillowServlet.java:35)
[java] at
org.apache.tapestry.ApplicationServlet.doGet(ApplicationServlet.java:79)
[java] at javax.servlet.http.HttpServlet.service(HttpServlet.java:697)
[java] at javax.servlet.http.HttpServlet.service(HttpServlet.java:810)
[java] at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
[java] at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
[java] at
com.zillow.web.filters.JSessionIDRemovalFilter.doFilter(JSessionIDRemovalFilter.java:73)
[java] at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
[java] at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
[java] at
com.zillow.web.filters.RequestMeteringFilter.doFilter(RequestMeteringFilter.java:31)
[java] at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
[java] at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
[java] at
org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:81)
[java] at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
[java] at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
[java] at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
[java] at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
[java] at
org.jboss.web.tomcat.security.CustomPrincipalValve.invoke(CustomPrincipalValve.java:39)
[java] at
org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:159)
[java] at
org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:59)
[java] at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
[java] at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
[java] at
org.apache.catalina.valves.FastCommonAccessLogValve.invoke(FastCommonAccessLogValve.java:481)
[java] at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
[java] at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
[java] at
org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:856)
[java] at
org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:744)
[java] at
org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
[java] at
org.apache.tomcat.util.net.MasterSlaveWorkerThread.run(MasterSlaveWorkerThread.java:112)
[java] at java.lang.Thread.run(Thread.java:595)
> Performance bottleneck with threaded services
> ---------------------------------------------
>
> Key: HIVEMIND-162
> URL: http://issues.apache.org/jira/browse/HIVEMIND-162
> Project: HiveMind
> Type: Bug
> Components: framework
> Versions: 1.1
> Environment: Linux 2.6.14.2, JBoss 4.0.3, Tomcat 5.5
> Quad Xeon 3.2GHz, 1GB RAM
> Reporter: Jeff Lubetkin
> Attachments: hivemind-1.2-alpha-1.jar
>
> Note: This may be better classified as a Hivemind issue, but it's affecting
> Tapestry throughput so I'm putting it here.
> We've been running some perf tests using the Grinder (with 20 threads)
> generating as much load as possible on a single non-trivial page. The page
> doesn't touch many of our biz logic services, but does have some complex
> componentry to render.
> We were seeing performance ramp just fine until we reached about 200TPS,
> using only 50% CPU. No matter how many clients we threw at it, we couldn't
> get it any higher. A thread dump showed that most threads were bottlenecked
> on a synchronized method in HiveMind
> (servicemodel.ThreadedServiceModel.constructServiceForCurrentThread, see
> Stack #1 below). This was the construction of the threaded
> ClientPropertyPersistenceStrategy service. Since we don't use the client
> strategy, I did a little hivemodule.xml magic and got rid of it from the
> PersistenceStrategy configuration. This gave us a huge increase in
> throughput, up to 490TPS still using only about 50% CPU, but we still
> bottlenecked. Again, a threaddump showed the culprit as
> constructServiceForCurrentThread, this time in the storage of the
> RequestGlobals (see Stack #2). We can't remove this service, so we've hit a
> ceiling.
> Until either Tapestry changes its usage of threaded services, or Hivemind is
> changed to not synchronize in this way, this looks like a ceiling for
> Tapestry performance. It'd be nice to be able to use all of the CPU on the
> box :)
> ================== STACK #1 ==================
> [java] "http-0.0.0.0-8080-7" daemon prio=1 tid=0x6eaf62e0 nid=0x1261
> waiting for monitor entry [0x6f69f000..0x6f6a0
> 840]
> [java] at
> org.apache.hivemind.impl.servicemodel.ThreadedServiceModel.constructServiceForCurrentThread(ThreadedS
> erviceModel.java:166)
> [java] - waiting to lock <0x4e861d88> (a
> org.apache.hivemind.impl.servicemodel.ThreadedServiceModel)
> [java] at
> org.apache.hivemind.impl.servicemodel.ThreadedServiceModel.getServiceImplementationForCurrentThread(T
> hreadedServiceModel.java:157)
> [java] at
> $PropertyPersistenceStrategy_107fc3dfcff._service($PropertyPersistenceStrategy_107fc3dfcff.java)
> [java] at
> $PropertyPersistenceStrategy_107fc3dfcff.getStoredChanges($PropertyPersistenceStrategy_107fc3dfcff.ja
> va)
> [java] at
> $PropertyPersistenceStrategy_107fc3dfd00.getStoredChanges($PropertyPersistenceStrategy_107fc3dfd00.ja
> va)
> [java] at
> org.apache.tapestry.record.PropertyPersistenceStrategySourceImpl.getAllStoredChanges(PropertyPersiste
> nceStrategySourceImpl.java:73)
> [java] at
> $PropertyPersistenceStrategySource_107fc3dfc3c.getAllStoredChanges($PropertyPersistenceStrategySource
> _107fc3dfc3c.java)
> [java] at
> org.apache.tapestry.record.PageRecorderImpl.getChanges(PageRecorderImpl.java:68)
> [java] at
> org.apache.tapestry.record.PageRecorderImpl.rollback(PageRecorderImpl.java:73)
> [java] at
> org.apache.tapestry.engine.RequestCycle.loadPage(RequestCycle.java:277)
> [java] at
> org.apache.tapestry.engine.RequestCycle.getPage(RequestCycle.java:249)
> [java] at
> org.apache.tapestry.engine.RequestCycle.activate(RequestCycle.java:612)
> [java] at
> org.apache.tapestry.engine.PageService.service(PageService.java:66)
> [java] at
> $IEngineService_107fc3dfc4c.service($IEngineService_107fc3dfc4c.java)
> [java] at
> org.apache.tapestry.services.impl.EngineServiceOuterProxy.service(EngineServiceOuterProxy.java:65)
> [java] at
> org.apache.tapestry.engine.AbstractEngine.service(AbstractEngine.java:248)
> [java] at
> org.apache.tapestry.services.impl.InvokeEngineTerminator.service(InvokeEngineTerminator.java:60)
> [java] at
> $WebRequestServicer_107fc3dfc28.service($WebRequestServicer_107fc3dfc28.java)
> [java] at
> $WebRequestServicer_107fc3dfc24.service($WebRequestServicer_107fc3dfc24.java)
> [java] at
> org.apache.tapestry.services.impl.WebRequestServicerPipelineBridge.service(WebRequestServicerPipeline
> Bridge.java:56)
> [java] etc...
> ================== STACK #2 ==================
> [java] "http-0.0.0.0-8080-52" daemon prio=1 tid=0x89613328 nid=0x20a9
> waiting for monitor entry [0x86d7e000..0x86d7
> f740]
> [java] at
> org.apache.hivemind.impl.servicemodel.ThreadedServiceModel.constructServiceForCurrentThread(ThreadedS
> erviceModel.java:166)
> [java] - waiting to lock <0x9503a218> (a
> org.apache.hivemind.impl.servicemodel.ThreadedServiceModel)
> [java] at
> org.apache.hivemind.impl.servicemodel.ThreadedServiceModel.getServiceImplementationForCurrentThread(T
> hreadedServiceModel.java:157)
> [java] at
> $RequestGlobals_1080261890e._service($RequestGlobals_1080261890e.java)
> [java] at
> $RequestGlobals_1080261890e.store($RequestGlobals_1080261890e.java)
> [java] at
> $RequestGlobals_1080261890f.store($RequestGlobals_1080261890f.java)
> [java] at
> org.apache.tapestry.services.impl.WebRequestServicerPipelineBridge.service(WebRequestServicerPipeline
> Bridge.java:49)
> [java] at
> $ServletRequestServicer_108026188f4.service($ServletRequestServicer_108026188f4.java)
> [java] at
> org.apache.tapestry.request.DecodedRequestInjector.service(DecodedRequestInjector.java:55)
> [java] at
> $ServletRequestServicerFilter_108026188f0.service($ServletRequestServicerFilter_108026188f0.java)
> [java] at
> $ServletRequestServicer_108026188f6.service($ServletRequestServicer_108026188f6.java)
> [java] at
> org.apache.tapestry.multipart.MultipartDecoderFilter.service(MultipartDecoderFilter.java:52)
> [java] at
> $ServletRequestServicerFilter_108026188ee.service($ServletRequestServicerFilter_108026188ee.java)
> [java] at
> $ServletRequestServicer_108026188f6.service($ServletRequestServicer_108026188f6.java)
> [java] at
> org.apache.tapestry.services.impl.SetupRequestEncoding.service(SetupRequestEncoding.java:53)
> [java] at
> $ServletRequestServicerFilter_108026188f2.service($ServletRequestServicerFilter_108026188f2.java)
> [java] at
> $ServletRequestServicer_108026188f6.service($ServletRequestServicer_108026188f6.java)
> [java] at
> $ServletRequestServicer_108026188e8.service($ServletRequestServicer_108026188e8.java)
> [java] at
> org.apache.tapestry.ApplicationServlet.doService(ApplicationServlet.java:123)
> [java] at
> com.zillow.web.ZillowServlet.doService(ZillowServlet.java:35)
> [java] at
> org.apache.tapestry.ApplicationServlet.doGet(ApplicationServlet.java:79)
> [java] at
> javax.servlet.http.HttpServlet.service(HttpServlet.java:697)
> [java] at
> javax.servlet.http.HttpServlet.service(HttpServlet.java:810)
> [java] at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
> [java] etc...
--
This message is automatically generated by JIRA.
-
If you think it was sent incorrectly contact one of the administrators:
http://issues.apache.org/jira/secure/Administrators.jspa
-
For more information on JIRA, see:
http://www.atlassian.com/software/jira
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
