Hi Isuru,
Even with MySQL, similar issue seems to exist
in started successfully
[2015-05-11 09:36:43,786] INFO
{org.apache.stratos.autoscaler.client.CloudControllerClient} - Trying to
spawn an instance via cloud controller: [cluster]
single-cartridge-app.my-as-manager.appserver-manager.domain [partition]
partition-1 [network-partition-id] network-partition-1
[2015-05-11 09:37:43,396] ERROR
{org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO} - *Failed to
delete the resource with id 386. Lock wait timeout exceeded; try restarting
transaction*
java.sql.SQLException: Lock wait timeout exceeded; try restarting
transaction
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1084)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4232)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4164)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2615)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2776)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2838)
at
com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2082)
at
com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2334)
at
com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2262)
at
com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2246)
at
org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.deleteResource(JDBCResourceDAO.java:1555)
at
org.wso2.carbon.registry.core.jdbc.Repository.removeResource(Repository.java:615)
at
org.wso2.carbon.registry.core.jdbc.Repository.prepareUpdate(Repository.java:362)
at org.wso2.carbon.registry.core.jdbc.Repository.put(Repository.java:287)
at
org.wso2.carbon.registry.core.jdbc.EmbeddedRegistry.put(EmbeddedRegistry.java:709)
at
org.wso2.carbon.registry.core.caching.CacheBackedRegistry.put(CacheBackedRegistry.java:465)
at
org.wso2.carbon.registry.core.session.UserRegistry.put(UserRegistry.java:657)
at
org.apache.stratos.cloud.controller.registry.RegistryManager.persist(RegistryManager.java:73)
at
org.apache.stratos.cloud.controller.util.CloudControllerUtil.persistTopology(CloudControllerUtil.java:276)
at
org.apache.stratos.cloud.controller.messaging.topology.TopologyManager.updateTopology(TopologyManager.java:103)
at
org.apache.stratos.cloud.controller.messaging.topology.TopologyBuilder.handleMemberCreatedEvent(TopologyBuilder.java:434)
at
org.apache.stratos.cloud.controller.services.impl.CloudControllerServiceImpl.startInstance(CloudControllerServiceImpl.java:486)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.apache.axis2.rpc.receivers.RPCUtil.invokeServiceClass(RPCUtil.java:212)
at
org.apache.axis2.rpc.receivers.RPCMessageReceiver.invokeBusinessLogic(RPCMessageReceiver.java:117)
at
org.apache.axis2.receivers.AbstractInOutMessageReceiver.invokeBusinessLogic(AbstractInOutMessageReceiver.java:40)
at
org.apache.axis2.receivers.AbstractMessageReceiver.receive(AbstractMessageReceiver.java:110)
at org.apache.axis2.engine.AxisEngine.receive(AxisEngine.java:180)
at
org.apache.axis2.transport.http.HTTPTransportUtils.processHTTPPostRequest(HTTPTransportUtils.java:172)
at org.apache.axis2.transport.http.AxisServlet.doPost(AxisServlet.java:146)
at
org.wso2.carbon.core.transports.CarbonServlet.doPost(CarbonServlet.java:231)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:755)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)
at
org.eclipse.equinox.http.servlet.internal.ServletRegistration.service(ServletRegistration.java:61)
at
org.eclipse.equinox.http.servlet.internal.ProxyServlet.processAlias(ProxyServlet.java:128)
at
org.eclipse.equinox.http.servlet.internal.ProxyServlet.service(ProxyServlet.java:68)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)
at
org.wso2.carbon.tomcat.ext.servlet.DelegationServlet.service(DelegationServlet.java:68)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at
org.wso2.carbon.tomcat.ext.filter.CharacterSetFilter.doFilter(CharacterSetFilter.java:61)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:222)
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123)
at
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:171)
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:99)
at
org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:178)
at
org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at
org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:56)
at
org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at
org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:141)
at
org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:156)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:936)
at
org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:52)
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
at
org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1004)
at
org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:589)
at
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1653)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
[2015-05-11 09:37:43,397] ERROR
{org.apache.stratos.cloud.controller.registry.RegistryManager} - Failed to
persist resource in registry: /cloud.controller/topology
org.wso2.carbon.registry.core.exceptions.RegistryException: Failed to
delete the resource with id 386. Lock wait timeout exceeded; try restarting
transaction
at
org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.deleteResource(JDBCResourceDAO.java:1563)
at
org.wso2.carbon.registry.core.jdbc.Repository.removeResource(Repository.java:615)
at
org.wso2.carbon.registry.core.jdbc.Repository.prepareUpdate(Repository.java:362)
at org.wso2.carbon.registry.core.jdbc.Repository.put(Repository.java:287)
at
org.wso2.carbon.registry.core.jdbc.EmbeddedRegistry.put(EmbeddedRegistry.java:709)
at
org.wso2.carbon.registry.core.caching.CacheBackedRegistry.put(CacheBackedRegistry.java:465)
at
org.wso2.carbon.registry.core.session.UserRegistry.put(UserRegistry.java:657)
at
org.apache.stratos.cloud.controller.registry.RegistryManager.persist(RegistryManager.java:73)
at
org.apache.stratos.cloud.controller.util.CloudControllerUtil.persistTopology(CloudControllerUtil.java:276)
at
org.apache.stratos.cloud.controller.messaging.topology.TopologyManager.updateTopology(TopologyManager.java:103)
at
org.apache.stratos.cloud.controller.messaging.topology.TopologyBuilder.handleMemberCreatedEvent(TopologyBuilder.java:434)
at
org.apache.stratos.cloud.controller.services.impl.CloudControllerServiceImpl.startInstance(CloudControllerServiceImpl.java:486)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.apache.axis2.rpc.receivers.RPCUtil.invokeServiceClass(RPCUtil.java:212)
at org.apache.axis2.rpc.receivers.RPCMessageRecei
On Fri, May 8, 2015 at 9:48 AM, Udara Liyanage <ud...@wso2.com> wrote:
> Hi,
>
> This seems to occur when metadata service too save properties in registry.
> This occurs when multiple clients(cartridge agent) call metadata service to
> publish property values. I did not see this error when only one client
> talks to metadata service.
> I guess h2 can not handle this. I will try with MySQL as Isuruh mentioned
> since this is blocking for me now.
>
> On Thu, May 7, 2015 at 5:12 PM, Isuru Haththotuwa <isu...@apache.org>
> wrote:
>
>> This seems to be an issue with H2. There are many occasions that this
>> error has been reported when using an H2 DB. Can you try:
>>
>> 1. replacing the currently used DB with a new H2 (from a fresh pack)
>> 2. use a MySQL DB
>>
>> and see if the error is reproducible?
>>
>> On Thu, May 7, 2015 at 4:42 PM, Imesh Gunaratne <im...@apache.org> wrote:
>>
>>> It looks like a database table lock is not being released until it
>>> reaches the timeout:
>>>
>>> Timeout trying to lock table "REG_RESOURCE"; SQL statement:
>>> SELECT REG_VERSION FROM REG_RESOURCE WHERE REG_PATH_ID=? AND REG_NAME=?
>>> AND REG_TENANT_ID=? [50200-140]
>>>
>>> The concern is we do not handle any database locks in Stratos code, they
>>> are handled by the Carbon Registry. Shall we try to narrow down the problem
>>> by running it again without using the Metadata API?
>>>
>>> Thanks
>>>
>>>
>>> On Thu, May 7, 2015 at 12:37 PM, Udara Liyanage <ud...@wso2.com> wrote:
>>>
>>>> Hi,
>>>>
>>>> I frequently get this exception when running Stratos for longtime (10+
>>>> minutes). Please note that most failures occurs when metadata service also
>>>> save properties in the registry (not all the time, this occurs when
>>>> metadata is not persisting too).
>>>>
>>>>
>>>> 2015-05-07 07:03:11,725] ERROR
>>>> {org.apache.stratos.cloud.controller.registry.RegistryManager} - * Failed
>>>> to persist resource in registry: /cloud.controller/topology*
>>>> org.wso2.carbon.registry.core.exceptions.RegistryException: Failed to
>>>> check the existence of the resource
>>>> /_system/governance/cloud.controller/topology. Timeout trying to lock table
>>>> "REG_RESOURCE"; SQL statement:
>>>> SELECT REG_VERSION FROM REG_RESOURCE WHERE REG_PATH_ID=? AND REG_NAME=?
>>>> AND REG_TENANT_ID=? [50200-140]
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.getVersion(JDBCResourceDAO.java:206)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.resourceExists(JDBCResourceDAO.java:147)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.resourceExists(JDBCResourceDAO.java:156)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.Repository.resourceExists(Repository.java:134)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.EmbeddedRegistry.put(EmbeddedRegistry.java:688)
>>>> at
>>>> org.wso2.carbon.registry.core.caching.CacheBackedRegistry.put(CacheBackedRegistry.java:465)
>>>> at
>>>> org.wso2.carbon.registry.core.session.UserRegistry.put(UserRegistry.java:657)
>>>> at
>>>> org.apache.stratos.cloud.controller.registry.RegistryManager.persist(RegistryManager.java:73)
>>>> at
>>>> org.apache.stratos.cloud.controller.util.CloudControllerUtil.persistTopology(CloudControllerUtil.java:276)
>>>> at
>>>> org.apache.stratos.cloud.controller.messaging.topology.TopologyManager.updateTopology(TopologyManager.java:103)
>>>> at
>>>> org.apache.stratos.cloud.controller.messaging.topology.TopologyBuilder.handleMemberStarted(TopologyBuilder.java:555)
>>>> at
>>>> org.apache.stratos.cloud.controller.messaging.receiver.instance.status.InstanceStatusTopicReceiver$2.onEvent(InstanceStatusTopicReceiver.java:75)
>>>> at
>>>> org.apache.stratos.messaging.listener.EventListener.update(EventListener.java:42)
>>>> at java.util.Observable.notifyObservers(Observable.java:159)
>>>> at
>>>> org.apache.stratos.messaging.event.EventObservable.notifyEventListeners(EventObservable.java:51)
>>>> at
>>>> org.apache.stratos.messaging.message.processor.instance.status.InstanceStatusMemberStartedMessageProcessor.process(InstanceStatusMemberStartedMessageProcessor.java:49)
>>>> at
>>>> org.apache.stratos.messaging.message.processor.instance.status.InstanceStatusMemberActivatedMessageProcessor.process(InstanceStatusMemberActivatedMessageProcessor.java:53)
>>>> at
>>>> org.apache.stratos.messaging.message.processor.MessageProcessorChain.process(MessageProcessorChain.java:61)
>>>> at
>>>> org.apache.stratos.messaging.message.receiver.instance.status.InstanceStatusEventMessageDelegator.run(InstanceStatusEventMessageDelegator.java:71)
>>>> at
>>>> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
>>>> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
>>>> at
>>>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>>>> at
>>>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>>>> at java.lang.Thread.run(Thread.java:744)
>>>> Caused by: org.h2.jdbc.JdbcSQLException: Timeout trying to lock table
>>>> "REG_RESOURCE"; SQL statement:
>>>> SELECT REG_VERSION FROM REG_RESOURCE WHERE REG_PATH_ID=? AND REG_NAME=?
>>>> AND REG_TENANT_ID=? [50200-140]
>>>> at org.h2.message.DbException.getJdbcSQLException(DbException.java:327)
>>>> at org.h2.message.DbException.get(DbException.java:167)
>>>> at org.h2.message.DbException.get(DbException.java:144)
>>>> at org.h2.table.RegularTable.doLock(RegularTable.java:466)
>>>> at org.h2.table.RegularTable.lock(RegularTable.java:404)
>>>> at org.h2.table.TableFilter.lock(TableFilter.java:139)
>>>> at org.h2.command.dml.Select.queryWithoutCache(Select.java:554)
>>>> at org.h2.command.dml.Query.query(Query.java:241)
>>>> at org.h2.command.CommandContainer.query(CommandContainer.java:80)
>>>> at org.h2.command.Command.executeQuery(Command.java:132)
>>>> at
>>>> org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:96)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.getVersion(JDBCResourceDAO.java:197)
>>>> ... 23 more
>>>> [2015-05-07 07:03:11,726] FATAL
>>>> {org.apache.stratos.cloud.controller.util.CloudControllerUtil} - Failed to
>>>> persist the Topology in registry.
>>>> org.wso2.carbon.registry.core.exceptions.RegistryException: Failed to
>>>> persist resource in registry: /cloud.controller/topology
>>>> at
>>>> org.apache.stratos.cloud.controller.registry.RegistryManager.persist(RegistryManager.java:85)
>>>> at
>>>> org.apache.stratos.cloud.controller.util.CloudControllerUtil.persistTopology(CloudControllerUtil.java:276)
>>>> at
>>>> org.apache.stratos.cloud.controller.messaging.topology.TopologyManager.updateTopology(TopologyManager.java:103)
>>>> at
>>>> org.apache.stratos.cloud.controller.messaging.topology.TopologyBuilder.handleMemberStarted(TopologyBuilder.java:555)
>>>> at
>>>> org.apache.stratos.cloud.controller.messaging.receiver.instance.status.InstanceStatusTopicReceiver$2.onEvent(InstanceStatusTopicReceiver.java:75)
>>>> at
>>>> org.apache.stratos.messaging.listener.EventListener.update(EventListener.java:42)
>>>> at java.util.Observable.notifyObservers(Observable.java:159)
>>>> at
>>>> org.apache.stratos.messaging.event.EventObservable.notifyEventListeners(EventObservable.java:51)
>>>> at
>>>> org.apache.stratos.messaging.message.processor.instance.status.InstanceStatusMemberStartedMessageProcessor.process(InstanceStatusMemberStartedMessageProcessor.java:49)
>>>> at
>>>> org.apache.stratos.messaging.message.processor.instance.status.InstanceStatusMemberActivatedMessageProcessor.process(InstanceStatusMemberActivatedMessageProcessor.java:53)
>>>> at
>>>> org.apache.stratos.messaging.message.processor.MessageProcessorChain.process(MessageProcessorChain.java:61)
>>>> at
>>>> org.apache.stratos.messaging.message.receiver.instance.status.InstanceStatusEventMessageDelegator.run(InstanceStatusEventMessageDelegator.java:71)
>>>> at
>>>> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
>>>> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
>>>> at
>>>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>>>> at
>>>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>>>> at java.lang.Thread.run(Thread.java:744)
>>>> Caused by: org.wso2.carbon.registry.core.exceptions.RegistryException:
>>>> Failed to check the existence of the resource
>>>> /_system/governance/cloud.controller/topology. Timeout trying to lock table
>>>> "REG_RESOURCE"; SQL statement:
>>>> SELECT REG_VERSION FROM REG_RESOURCE WHERE REG_PATH_ID=? AND REG_NAME=?
>>>> AND REG_TENANT_ID=? [50200-140]
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.getVersion(JDBCResourceDAO.java:206)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.resourceExists(JDBCResourceDAO.java:147)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.resourceExists(JDBCResourceDAO.java:156)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.Repository.resourceExists(Repository.java:134)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.EmbeddedRegistry.put(EmbeddedRegistry.java:688)
>>>> at
>>>> org.wso2.carbon.registry.core.caching.CacheBackedRegistry.put(CacheBackedRegistry.java:465)
>>>> at
>>>> org.wso2.carbon.registry.core.session.UserRegistry.put(UserRegistry.java:657)
>>>> at
>>>> org.apache.stratos.cloud.controller.registry.RegistryManager.persist(RegistryManager.java:73)
>>>> ... 16 more
>>>> Caused by: org.h2.jdbc.JdbcSQLException: Timeout trying to lock table
>>>> "REG_RESOURCE"; SQL statement:
>>>> SELECT REG_VERSION FROM REG_RESOURCE WHERE REG_PATH_ID=? AND REG_NAME=?
>>>> AND REG_TENANT_ID=? [50200-140]
>>>> at org.h2.message.DbException.getJdbcSQLException(DbException.java:327)
>>>> at org.h2.message.DbException.get(DbException.java:167)
>>>> at org.h2.message.DbException.get(DbException.java:144)
>>>> at org.h2.table.RegularTable.doLock(RegularTable.java:466)
>>>> at org.h2.table.RegularTable.lock(RegularTable.java:404)
>>>> at org.h2.table.TableFilter.lock(TableFilter.java:139)
>>>> at org.h2.command.dml.Select.queryWithoutCache(Select.java:554)
>>>> at org.h2.command.dml.Query.query(Query.java:241)
>>>> at org.h2.command.CommandContainer.query(CommandContainer.java:80)
>>>> at org.h2.command.Command.executeQuery(Command.java:132)
>>>> at
>>>> org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:96)
>>>> at
>>>> org.wso2.carbon.registry.core.jdbc.dao.JDBCResourceDAO.getVersion(JDBCResourceDAO.java:197)
>>>> ... 23 more
>>>>
>>>>
>>>> --
>>>>
>>>> Udara Liyanage
>>>> Software Engineer
>>>> WSO2, Inc.: http://wso2.com
>>>> lean. enterprise. middleware
>>>>
>>>> web: http://udaraliyanage.wordpress.com
>>>> phone: +94 71 443 6897
>>>>
>>>
>>>
>>>
>>> --
>>> Imesh Gunaratne
>>>
>>> Senior Technical Lead, WSO2
>>> Committer & PMC Member, Apache Stratos
>>>
>>> --
>>> Thanks and Regards,
>>>
>>> Isuru H.
>>> +94 716 358 048* <http://wso2.com/>*
>>>
>>>
>>> * <http://wso2.com/>*
>>>
>>>
>>> * <http://wso2.com/>*
>>>
>>>
>>>
>
>
> --
>
> Udara Liyanage
> Software Engineer
> WSO2, Inc.: http://wso2.com
> lean. enterprise. middleware
>
> web: http://udaraliyanage.wordpress.com
> phone: +94 71 443 6897
>
--
Udara Liyanage
Software Engineer
WSO2, Inc.: http://wso2.com
lean. enterprise. middleware
web: http://udaraliyanage.wordpress.com
phone: +94 71 443 6897
