@Singleton
@Lock(LockType.WRITE) ??? <- this is a bottlenck
PS: i dont know if that's my gmail but i dont have the stacks, justt
the thread names
Romain Manni-Bucau
Twitter: @rmannibucau
Blog: http://rmannibucau.wordpress.com/
LinkedIn: http://fr.linkedin.com/in/rmannibucau
Github: https://github.com/rmannibucau
2014-02-11 15:26 GMT+01:00 Leonardo K. Shikida <shik...@gmail.com>:
> Hi
>
> I really don't know where is the bottleneck.
>
> I've created a new project from the scratch to try to isolate the
> variables.
>
> Now I have something as simple as
>
> producer:
>
> public void createTasks() throws JMSException {
> for(int i=0;i<1000;i++){
> if (i%100 == 0){
> System.out.println(i);
> }
> MyEntity my = new MyEntity("xyz"+i);
> my = this.baseService.getMyEntityDAO().addAndFlush(my);
> jmsService.enqueue(my);
> }
> }
>
>
> jms
>
> @Singleton
> @Lock(LockType.WRITE)
> public class JmsService implements Serializable{
> public void enqueue(MyEntity target) throws JMSException{
> Connection connection = null;
> Session session = null;
> try {
> ConnectionFactory connectionFactory = this.connectionFactory;
> connection = connectionFactory.createConnection();
> connection.start();
>
> session = connection.createSession(false,
> Session.AUTO_ACKNOWLEDGE);
> Queue queue = this.driverJobQueue;
> MessageProducer producer = session.createProducer(queue);
>
> Message message = session.createMessage();
> message.setLongProperty("id",target.getId());
> message.setStringProperty("type", "xyz");
> producer.send(message,DeliveryMode.PERSISTENT,1,0);
> } finally {
> // Clean up
> if (session != null){
> session.close();
> }
>
> if (connection != null){
> connection.close();
> }
> }
>
> }
>
> consumer
>
> import javax.annotation.PostConstruct;
> import javax.ejb.ActivationConfigProperty;
> import javax.ejb.EJB;
> import javax.ejb.MessageDriven;
> import javax.inject.Inject;
> import javax.jms.Message;
> import javax.jms.MessageListener;
> import javax.persistence.EntityManager;
> import javax.persistence.PersistenceContext;
>
> @MessageDriven(activationConfig = {
> @ActivationConfigProperty(
> propertyName = "destinationType",
> propertyValue = "javax.jms.Queue"),
> @ActivationConfigProperty(
> propertyName = "destination",
> propertyValue = "DriverJobQueue"),
> @ActivationConfigProperty(
> propertyName = "messageSelector",
> propertyValue = "type = 'xyz'"),
> @ActivationConfigProperty(
> propertyName = "acknowledgeMode",
> propertyValue = "Auto-acknowledge"),
> @ActivationConfigProperty(
> propertyName = "maxSessions",
> propertyValue = "30"),
> @ActivationConfigProperty(
> propertyName = "maxMessagesPerBatch",
> propertyValue = "30"),
> @ActivationConfigProperty(
> propertyName = "maxMessagesPerSessions",
> propertyValue = "30") })
>
> public class MyWorker implements MessageListener {
> @Override
> public void onMessage(Message msg) {
> MyEntity entity = null;
> Long id = null;
> try {
> id = msg.getLongProperty("id");
>
> if (entity != null) {
> System.out.println(id);
> }else{
> System.out.println("ops");
> }
>
> System.out.println(this+" is processing "+entity);
>
> } catch (Throwable e) {
> e.printStackTrace();
> }
>
> try {
> Thread.sleep(10000);
> } catch (InterruptedException e) {
> e.printStackTrace();
> }
>
> System.out.println(this+" finished "+entity);
>
> }
> }
>
> tomee.xml
>
> <Resource id="Default JMS Resource Adapter"
> type="ActiveMQResourceAdapter">
> BrokerXmlConfig = xbean:file:/pathto/activemq.xml
> ServerUrl = tcp://0.0.0.0:61616
> threadPoolSize 30 <<<<<<< does that really works?
> </Resource>
>
> <Container id="MyJmsMdbContainer" ctype="MESSAGE">
> ResourceAdapter = Default JMS Resource Adapter
> </Container>
>
> <Resource id="Fooo" type="javax.jms.ConnectionFactory">
> PoolMaxSize 30
> </Resource>
>
> <Container id="myAllContainer" type="STATELESS">
> PoolSize 30
> StrictPooling true
> </Container>
>
> <Container id="msg" type="MESSAGE">
> InstanceLimit 30
> </Container>
>
> <Container id="Foo" type="BMP_ENTITY">
> PoolSize 30
> </Container>
>
> activemq.xml
>
>
> <beans xmlns="http://www.springframework.org/schema/beans"; xmlns:amq="
> http://activemq.apache.org/schema/core"; xmlns:xsi="
> http://www.w3.org/2001/XMLSchema-instance";
> xsi:schemaLocation="http://www.springframework.org/schema/beans
> http://www.springframework.org/schema/beans/spring-beans.xsd
> http://activemq.apache.org/schema/core
> http://activemq.apache.org/schema/core/activemq-core.xsd";>
>
> <broker xmlns="http://activemq.apache.org/schema/core";
> brokerName="localhost" dataDirectory="${activemq.data}">
> <destinationPolicy>
> <policyMap>
> <policyEntries>
> <policyEntry topic=">" producerFlowControl="true">
> <pendingMessageLimitStrategy>
> <constantPendingMessageLimitStrategy
> limit="1000" />
> </pendingMessageLimitStrategy>
> </policyEntry>
> <policyEntry queue=">" producerFlowControl="true"
> memoryLimit="1mb">
> </policyEntry>
> </policyEntries>
> </policyMap>
> </destinationPolicy>
>
> <persistenceAdapter>
> <jdbcPersistenceAdapter dataSource="#oracle-ds" />
> </persistenceAdapter>
>
> <systemUsage>
> <systemUsage>
> <memoryUsage>
> <memoryUsage limit="128 mb" />
> </memoryUsage>
> <storeUsage>
> <storeUsage limit="100 gb" />
> </storeUsage>
> <tempUsage>
> <tempUsage limit="50 gb" />
> </tempUsage>
> </systemUsage>
> </systemUsage>
>
> <transportConnectors>
> <transportConnector name="tcp" uri="tcp://0.0.0.0:61616" />
> </transportConnectors>
> </broker>
>
> <bean id="oracle-ds" class="org.apache.commons.dbcp.BasicDataSource"
> destroy-method="close">
> <property name="driverClassName" value="oracle.jdbc.OracleDriver" />
> <property name="url" value="jdbc:oracle:thin:@localhost:1521:XE" />
> <property name="username" value="xxx" />
> <property name="password" value="xxx" />
> <property name="poolPreparedStatements" value="true" />
> <property name="maxActive " value="30" />
> </bean>
>
> </beans>
>
> kill -3 on catalina gives me
>
> [leoks@oc7612866413 ~]$ grep prio 2.txt | sort
> "ActiveMQ Broker[localhost] Scheduler" daemon prio=10
> tid=0x00007fa8fcea3000 nid=0x411e in Object.wait() [0x00007fa8f0d78000]
> "ActiveMQ BrokerService[localhost] Task-1" daemon prio=10
> tid=0x00007fa88c041800 nid=0x4171 waiting on condition [0x00007fa8e8217000]
> "ActiveMQ BrokerService[localhost] Task-2" daemon prio=10
> tid=0x00007fa89c5a8800 nid=0x4175 waiting on condition [0x00007fa873dfc000]
> "ActiveMQ BrokerService[localhost] Task-3" daemon prio=10
> tid=0x00007fa874020000 nid=0x42b5 waiting on condition [0x00007fa8f107b000]
> "ActiveMQ InactivityMonitor ReadCheckTimer" daemon prio=10
> tid=0x00007fa8a003d800 nid=0x4173 in Object.wait() [0x00007fa873ffe000]
> "ActiveMQ InactivityMonitor Worker" daemon prio=10 tid=0x00007fa88c05a800
> nid=0x41b3 waiting on condition [0x00007fa8e891e000]
> "ActiveMQ InactivityMonitor Worker" daemon prio=10 tid=0x00007fa88c05d000
> nid=0x4220 waiting on condition [0x00007fa8f117c000]
> "ActiveMQ InactivityMonitor WriteCheckTimer" daemon prio=10
> tid=0x00007fa8a003e800 nid=0x4174 in Object.wait() [0x00007fa873efd000]
> "ActiveMQ JDBC PA Scheduled Task" daemon prio=10 tid=0x00007fa878003800
> nid=0x413d waiting on condition [0x00007fa8f0772000]
> "ActiveMQ JDBC PA Scheduled Task" daemon prio=10 tid=0x00007fa89403a000
> nid=0x41bc waiting on condition [0x00007fa872bea000]
> "ActiveMQ JDBC PA Scheduled Task" daemon prio=10 tid=0x00007fa8a0041800
> nid=0x4205 waiting on condition [0x00007fa8728e7000]
> "ActiveMQ JDBC PA Scheduled Task" daemon prio=10 tid=0x00007fa8fcf12000
> nid=0x4137 waiting on condition [0x00007fa8f0974000]
> "ActiveMQ JDBC PA Scheduled Task" daemon prio=10 tid=0x00007fa8fcf15800
> nid=0x4138 waiting on condition [0x00007fa8f0671000]
> "ActiveMQ Transport Server: tcp://0.0.0.0:61616" daemon prio=10
> tid=0x00007fa8fc7b2000 nid=0x4143 runnable [0x00007fa8f137e000]
> "ActiveMQ Transport Server Thread Handler: tcp://0.0.0.0:61616" daemon
> prio=10 tid=0x00007fa8fc7b1000 nid=0x4142 waiting on condition
> [0x00007fa8f127d000]
> "ActiveMQ Transport: tcp:///127.0.0.1:53438@61616" daemon prio=10
> tid=0x00007fa89c5a7800 nid=0x4172 runnable [0x00007fa8e8116000]
> "ActiveMQ Transport: tcp:///127.0.0.1:53468@61616" daemon prio=10
> tid=0x00007fa87400f000 nid=0x41fd runnable [0x00007fa8729e8000]
> "ActiveMQ Transport: tcp://oc7612866413.ibm.com/127.0.0.1:61616@53438"
> prio=10 tid=0x00007fa88c035800 nid=0x4170 runnable [0x00007fa8e851a000]
> "ActiveMQ Transport: tcp://oc7612866413.ibm.com/127.0.0.1:61616@53468"
> prio=10 tid=0x00007fa89c50b800 nid=0x41fc runnable [0x00007fa8e881d000]
> "ajp-bio-8009-Acceptor-0" daemon prio=10 tid=0x00007fa8fc13a000 nid=0x4188
> runnable [0x00007fa8736f5000]
> "ajp-bio-8009-AsyncTimeout" daemon prio=10 tid=0x00007fa8fc13c000
> nid=0x4189 waiting on condition [0x00007fa8735f4000]
> "Attach Listener" daemon prio=10 tid=0x00007fa8d0001000 nid=0x415b runnable
> [0x0000000000000000]
> "C2 CompilerThread0" daemon prio=10 tid=0x00007fa8fc0a3800 nid=0x40f6
> waiting on condition [0x0000000000000000]
> "C2 CompilerThread1" daemon prio=10 tid=0x00007fa8fc0a6800 nid=0x40f7
> waiting on condition [0x0000000000000000]
> "ContainerBackgroundProcessor[StandardEngine[Catalina]]" daemon prio=10
> tid=0x00007fa8fc853000 nid=0x4182 waiting on condition [0x00007fa873af9000]
> "Default JMS Resource Adapter-worker- - 10" daemon prio=10
> tid=0x00007fa8a00a4800 nid=0x42c3 waiting on condition [0x00007fa871fdd000]
> "Default JMS Resource Adapter-worker- - 11" daemon prio=10
> tid=0x00007fa8a00a6000 nid=0x42c4 waiting on condition [0x00007fa871edc000]
> "Default JMS Resource Adapter-worker- - 1" daemon prio=10
> tid=0x00007fa89c597800 nid=0x416e waiting on condition [0x00007fa8e8419000]
> "Default JMS Resource Adapter-worker- - 2" daemon prio=10
> tid=0x00007fa8a005e000 nid=0x42b7 waiting on condition [0x00007fa8727e5000]
> "Default JMS Resource Adapter-worker- - 3" daemon prio=10
> tid=0x00007fa8a005f000 nid=0x42b8 waiting on condition [0x00007fa8726e4000]
> "Default JMS Resource Adapter-worker- - 4" daemon prio=10
> tid=0x00007fa8a0063800 nid=0x42b9 waiting on condition [0x00007fa8725e3000]
> "Default JMS Resource Adapter-worker- - 5" daemon prio=10
> tid=0x00007fa8a0065800 nid=0x42ba waiting on condition [0x00007fa8724e2000]
> "Default JMS Resource Adapter-worker- - 6" daemon prio=10
> tid=0x00007fa8a0069000 nid=0x42bb waiting on condition [0x00007fa8723e1000]
> "Default JMS Resource Adapter-worker- - 7" daemon prio=10
> tid=0x00007fa8a0067800 nid=0x42bc waiting on condition [0x00007fa8722e0000]
> "Default JMS Resource Adapter-worker- - 8" daemon prio=10
> tid=0x00007fa8a0086800 nid=0x42be waiting on condition [0x00007fa8721df000]
> "Default JMS Resource Adapter-worker- - 9" daemon prio=10
> tid=0x00007fa8a00a1800 nid=0x42c2 waiting on condition [0x00007fa8720de000]
> "Finalizer" daemon prio=10 tid=0x00007fa8fc072800 nid=0x40eb in
> Object.wait() [0x00007fa8f22d1000]
> "GC Daemon" daemon prio=10 tid=0x00007fa8fc9a2000 nid=0x4103 in
> Object.wait() [0x00007fa8f158a000]
> "GC task thread#0 (ParallelGC)" prio=10 tid=0x00007fa8fc021800 nid=0x40e5
> runnable
> "GC task thread#1 (ParallelGC)" prio=10 tid=0x00007fa8fc023800 nid=0x40e6
> runnable
> "GC task thread#2 (ParallelGC)" prio=10 tid=0x00007fa8fc025800 nid=0x40e7
> runnable
> "GC task thread#3 (ParallelGC)" prio=10 tid=0x00007fa8fc027000 nid=0x40e8
> runnable
> "http-bio-8080-Acceptor-0" daemon prio=10 tid=0x00007fa8fc853800 nid=0x4183
> runnable [0x00007fa8739f8000]
> "http-bio-8080-AsyncTimeout" daemon prio=10 tid=0x00007fa8fc855800
> nid=0x4184 waiting on condition [0x00007fa873bfa000]
> "http-bio-8080-exec-10" daemon prio=10 tid=0x00007fa874011800 nid=0x419c
> waiting on condition [0x00007fa872ceb000]
> "http-bio-8080-exec-1" daemon prio=10 tid=0x00007fa88c054800 nid=0x4185
> waiting on condition [0x00007fa873cfb000]
> "http-bio-8080-exec-2" daemon prio=10 tid=0x00007fa89c755000 nid=0x418a
> waiting on condition [0x00007fa8734f3000]
> "http-bio-8080-exec-3" daemon prio=10 tid=0x00007fa88c055800 nid=0x4194
> waiting on condition [0x00007fa8733f2000]
> "http-bio-8080-exec-4" daemon prio=10 tid=0x00007fa89c8ea000 nid=0x4195
> waiting on condition [0x00007fa8732f1000]
> "http-bio-8080-exec-5" daemon prio=10 tid=0x00007fa88c056000 nid=0x4196
> waiting on condition [0x00007fa8731f0000]
> "http-bio-8080-exec-6" daemon prio=10 tid=0x00007fa88c057800 nid=0x4197
> waiting on condition [0x00007fa8730ef000]
> "http-bio-8080-exec-7" daemon prio=10 tid=0x00007fa88c059000 nid=0x4198
> waiting on condition [0x00007fa872fee000]
> "http-bio-8080-exec-8" daemon prio=10 tid=0x00007fa89c8f6800 nid=0x419a
> waiting on condition [0x00007fa872eed000]
> "http-bio-8080-exec-9" daemon prio=10 tid=0x00007fa874013000 nid=0x419b
> waiting on condition [0x00007fa872dec000]
> "http-bio-8443-Acceptor-0" daemon prio=10 tid=0x00007fa8fc137000 nid=0x4186
> runnable [0x00007fa8738f7000]
> "http-bio-8443-AsyncTimeout" daemon prio=10 tid=0x00007fa8fc138000
> nid=0x4187 waiting on condition [0x00007fa8737f6000]
> "JDWP Command Reader" daemon prio=10 tid=0x00007fa8c4001000 nid=0x40f3
> runnable [0x0000000000000000]
> "JDWP Event Helper Thread" daemon prio=10 tid=0x00007fa8fc0a1000 nid=0x40f2
> runnable [0x0000000000000000]
> "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00007fa8fc09d800
> nid=0x40ef runnable [0x0000000000000000]
> "main" prio=10 tid=0x00007fa8fc00c000 nid=0x40e4 runnable
> [0x00007fa902425000]
> "oracle.jdbc.driver.BlockSource.ThreadedCachingBlockSource.BlockReleaser"
> daemon prio=10 tid=0x00007fa8fce2f000 nid=0x4127 in Object.wait()
> [0x00007fa8f147f000]
> "org.apache.openejb.pool.scheduler.1" daemon prio=10 tid=0x00007fa8fc902000
> nid=0x4147 waiting on condition [0x00007fa8f046f000]
> "org.apache.openejb.pool.scheduler.2" daemon prio=10 tid=0x00007fa8fc903000
> nid=0x4148 waiting on condition [0x00007fa8f036e000]
> "org.apache.openejb.pool.scheduler.3" daemon prio=10 tid=0x00007fa8fc904800
> nid=0x4149 waiting on condition [0x00007fa8f026d000]
> "org.apache.openejb.pool.scheduler.4" daemon prio=10 tid=0x00007fa8fc906800
> nid=0x414a waiting on condition [0x00007fa8f016c000]
> "org.apache.openejb.pool.scheduler.5" daemon prio=10 tid=0x00007fa89c5b0000
> nid=0x416f waiting on condition [0x00007fa8e8318000]
> "PoolIdleReleaseTimer" daemon prio=10 tid=0x00007fa8fcc15800 nid=0x4146 in
> Object.wait() [0x00007fa8f0c77000]
> "Reference Handler" daemon prio=10 tid=0x00007fa8fc06e800 nid=0x40ea in
> Object.wait() [0x00007fa8f23d2000]
> "RetryTimer" daemon prio=10 tid=0x00007fa8fcb39000 nid=0x411d in
> Object.wait() [0x00007fa8f0b76000]
> "RMI Reaper" prio=10 tid=0x00007fa8a4065800 nid=0x4122 in Object.wait()
> [0x00007fa8f0a75000]
> "RMI RenewClean-[127.0.0.1:40595]" daemon prio=10 tid=0x00007fa8a000b800
> nid=0x4125 in Object.wait() [0x00007fa8f0873000]
> "RMI Scheduler(0)" daemon prio=10 tid=0x00007fa8a406f800 nid=0x4124 waiting
> on condition [0x00007fa8f0e79000]
> "RMI TCP Accept-0" daemon prio=10 tid=0x00007fa8a4064800 nid=0x4121
> runnable [0x00007fa8f0570000]
> "RMI TCP Accept-1099" daemon prio=10 tid=0x00007fa8fcea7800 nid=0x411f
> runnable [0x00007fa8f0f7a000]
> "Service Thread" daemon prio=10 tid=0x00007fa8fc0a9000 nid=0x40f8 runnable
> [0x0000000000000000]
> "Signal Dispatcher" daemon prio=10 tid=0x00007fa8fc088800 nid=0x40ed
> waiting on condition [0x0000000000000000]
> "Thread-19" daemon prio=10 tid=0x00007fa89c178800 nid=0x414f runnable
> [0x00007fa8e871c000]
> "Thread-23" daemon prio=10 tid=0x00007fa89c8a5800 nid=0x41e5 waiting on
> condition [0x00007fa872ae9000]
> "VM Periodic Task Thread" prio=10 tid=0x00007fa8fc0b3800 nid=0x40f9 waiting
> on condition
> "VM Thread" prio=10 tid=0x00007fa8fc06c800 nid=0x40e9 runnable
>
> Oracle (should not affect actually, I guess, but I am including here
> because JMS messages are persisted)
>
> SQL> show parameter session
>
> NAME TYPE VALUE
> ------------------------------------ -----------
> ------------------------------
> java_max_sessionspace_size integer 0
> java_soft_sessionspace_limit integer 0
> license_max_sessions integer 0
> license_sessions_warning integer 0
> session_cached_cursors integer 50
> session_max_open_files integer 10
> sessions integer 324 <<<<<<<<<<<
> shared_server_sessions integer
> SQL> show parameter processes
>
> NAME TYPE VALUE
> ------------------------------------ -----------
> ------------------------------
> aq_tm_processes integer 0
> db_writer_processes integer 1
> gcs_server_processes integer 0
> global_txn_processes integer 1
> job_queue_processes integer 0
> log_archive_max_processes integer 4
> processes integer 200 <<<<<<<<<<
>
>
> Still getting 10 JMS workers and 10 simultaneous threads consuming the
> messages.
>
> Don't know what else to try.
>
> How can I increase the number of workers?
>
> TIA
>
> []
>
> Leo
>
>
> []
>
> Leo
>
>
> On Mon, Feb 10, 2014 at 4:44 PM, Romain Manni-Bucau
> <rmannibu...@gmail.com>wrote:
>
>> Yeah, another pool surely (db, stateless, other...)
>> Le 10 févr. 2014 19:35, "Leonardo K. Shikida" <shik...@gmail.com> a écrit
>> :
>>
>> > I've set all pools to 100. The thread dump (if I understand it well)
>> seems
>> > to say that there are 99 threads waiting.
>> >
>> > Does it mean that the pool was set correctly but there's a bottleneck
>> > somewhere?
>> >
>> > []
>> >
>> > Leo
>> >
>> > [leoks@oc7612866413 ~]$ grep "Default JMS Resource Adapter" dump.txt
>> > "Default JMS Resource Adapter-worker- - 99" daemon prio=10
>> > tid=0x00007f41946e3000 nid=0x2f00 waiting on condition
>> [0x00007f414b4aa000]
>> > "Default JMS Resource Adapter-worker- - 98" daemon prio=10
>> > tid=0x00007f41946e1000 nid=0x2ef8 waiting on condition
>> [0x00007f41430d2000]
>> > "Default JMS Resource Adapter-worker- - 97" daemon prio=10
>> > tid=0x00007f41946df000 nid=0x2ef3 waiting on condition
>> [0x00007f41432d4000]
>> > "Default JMS Resource Adapter-worker- - 96" daemon prio=10
>> > tid=0x00007f41946dd000 nid=0x2ef0 waiting on condition
>> [0x00007f41433d5000]
>> > "Default JMS Resource Adapter-worker- - 95" daemon prio=10
>> > tid=0x00007f41944a1000 nid=0x2ed9 waiting on condition
>> [0x00007f41434d6000]
>> > "Default JMS Resource Adapter-worker- - 94" daemon prio=10
>> > tid=0x00007f419449f000 nid=0x2ed8 waiting on condition
>> [0x00007f41435d7000]
>> > "Default JMS Resource Adapter-worker- - 93" daemon prio=10
>> > tid=0x00007f419449d000 nid=0x2ed0 waiting on condition
>> [0x00007f41436d8000]
>> > "Default JMS Resource Adapter-worker- - 92" daemon prio=10
>> > tid=0x00007f419449b000 nid=0x2ecf waiting on condition
>> [0x00007f41437d9000]
>> > "Default JMS Resource Adapter-worker- - 91" daemon prio=10
>> > tid=0x00007f4194b46000 nid=0x2ece waiting on condition
>> [0x00007f41438da000]
>> > "Default JMS Resource Adapter-worker- - 90" daemon prio=10
>> > tid=0x00007f4194b44000 nid=0x2ecd waiting on condition
>> [0x00007f41439db000]
>> > "Default JMS Resource Adapter-worker- - 89" daemon prio=10
>> > tid=0x00007f4194b42000 nid=0x2ecc waiting on condition
>> [0x00007f4143adc000]
>> > "Default JMS Resource Adapter-worker- - 88" daemon prio=10
>> > tid=0x00007f4194b40000 nid=0x2ecb waiting on condition
>> [0x00007f4143bdd000]
>> > "Default JMS Resource Adapter-worker- - 87" daemon prio=10
>> > tid=0x00007f4194112800 nid=0x2eca waiting on condition
>> [0x00007f4143cde000]
>> > "Default JMS Resource Adapter-worker- - 86" daemon prio=10
>> > tid=0x00007f4194110800 nid=0x2ec9 waiting on condition
>> [0x00007f4143ddf000]
>> > "Default JMS Resource Adapter-worker- - 85" daemon prio=10
>> > tid=0x00007f419410e800 nid=0x2ec8 waiting on condition
>> [0x00007f4143ee0000]
>> > "Default JMS Resource Adapter-worker- - 84" daemon prio=10
>> > tid=0x00007f419410c800 nid=0x2ec7 waiting on condition
>> [0x00007f4143fe1000]
>> > "Default JMS Resource Adapter-worker- - 83" daemon prio=10
>> > tid=0x00007f41944d8800 nid=0x2ec6 waiting on condition
>> [0x00007f41440e2000]
>> > "Default JMS Resource Adapter-worker- - 82" daemon prio=10
>> > tid=0x00007f41944d6000 nid=0x2ec5 waiting on condition
>> [0x00007f41441e3000]
>> > "Default JMS Resource Adapter-worker- - 81" daemon prio=10
>> > tid=0x00007f41944d4000 nid=0x2ec4 waiting on condition
>> [0x00007f41442e4000]
>> > "Default JMS Resource Adapter-worker- - 80" daemon prio=10
>> > tid=0x00007f41944d2800 nid=0x2ec3 waiting on condition
>> [0x00007f41443e5000]
>> > "Default JMS Resource Adapter-worker- - 79" daemon prio=10
>> > tid=0x00007f4194647800 nid=0x2ec1 waiting on condition
>> [0x00007f41444e6000]
>> > "Default JMS Resource Adapter-worker- - 78" daemon prio=10
>> > tid=0x00007f4194645800 nid=0x2ec0 waiting on condition
>> [0x00007f41445e7000]
>> > "Default JMS Resource Adapter-worker- - 77" daemon prio=10
>> > tid=0x00007f4194644000 nid=0x2ebe waiting on condition
>> [0x00007f41446e8000]
>> > "Default JMS Resource Adapter-worker- - 76" daemon prio=10
>> > tid=0x00007f4194642800 nid=0x2ebd waiting on condition
>> [0x00007f41447e9000]
>> > "Default JMS Resource Adapter-worker- - 75" daemon prio=10
>> > tid=0x00007f41943d8000 nid=0x2eb3 waiting on condition
>> [0x00007f41448ea000]
>> > "Default JMS Resource Adapter-worker- - 74" daemon prio=10
>> > tid=0x00007f41943d6000 nid=0x2eb2 waiting on condition
>> [0x00007f41449eb000]
>> > "Default JMS Resource Adapter-worker- - 73" daemon prio=10
>> > tid=0x00007f41943d4000 nid=0x2eb1 waiting on condition
>> [0x00007f4144aec000]
>> > "Default JMS Resource Adapter-worker- - 72" daemon prio=10
>> > tid=0x00007f4194ea4000 nid=0x2eb0 waiting on condition
>> [0x00007f4144bed000]
>> > "Default JMS Resource Adapter-worker- - 71" daemon prio=10
>> > tid=0x00007f4194ea2000 nid=0x2eaf waiting on condition
>> [0x00007f4144cee000]
>> > "Default JMS Resource Adapter-worker- - 70" daemon prio=10
>> > tid=0x00007f4194e9f800 nid=0x2eae waiting on condition
>> [0x00007f4144def000]
>> > "Default JMS Resource Adapter-worker- - 69" daemon prio=10
>> > tid=0x00007f4194e9d800 nid=0x2ead waiting on condition
>> [0x00007f4144ef0000]
>> > "Default JMS Resource Adapter-worker- - 68" daemon prio=10
>> > tid=0x00007f419425f800 nid=0x2eac waiting on condition
>> [0x00007f4144ff1000]
>> > "Default JMS Resource Adapter-worker- - 67" daemon prio=10
>> > tid=0x00007f419425d800 nid=0x2eab waiting on condition
>> [0x00007f41450f2000]
>> > "Default JMS Resource Adapter-worker- - 66" daemon prio=10
>> > tid=0x00007f419425c000 nid=0x2eaa waiting on condition
>> [0x00007f41451f3000]
>> > "Default JMS Resource Adapter-worker- - 65" daemon prio=10
>> > tid=0x00007f419425a800 nid=0x2ea9 waiting on condition
>> [0x00007f41452f4000]
>> > "Default JMS Resource Adapter-worker- - 64" daemon prio=10
>> > tid=0x00007f4194259000 nid=0x2ea8 waiting on condition
>> [0x00007f41453f5000]
>> > "Default JMS Resource Adapter-worker- - 63" daemon prio=10
>> > tid=0x00007f419495b800 nid=0x2ea7 waiting on condition
>> [0x00007f41454f6000]
>> > "Default JMS Resource Adapter-worker- - 62" daemon prio=10
>> > tid=0x00007f4194959800 nid=0x2ea6 waiting on condition
>> [0x00007f41455f7000]
>> > "Default JMS Resource Adapter-worker- - 61" daemon prio=10
>> > tid=0x00007f4194957800 nid=0x2ea5 waiting on condition
>> [0x00007f41456f8000]
>> > "Default JMS Resource Adapter-worker- - 60" daemon prio=10
>> > tid=0x00007f41954b6800 nid=0x2ea4 waiting on condition
>> [0x00007f41457f9000]
>> > "Default JMS Resource Adapter-worker- - 59" daemon prio=10
>> > tid=0x00007f41954b4800 nid=0x2ea3 waiting on condition
>> [0x00007f41458fa000]
>> > "Default JMS Resource Adapter-worker- - 58" daemon prio=10
>> > tid=0x00007f41954b2800 nid=0x2ea2 waiting on condition
>> [0x00007f41459fb000]
>> > "Default JMS Resource Adapter-worker- - 57" daemon prio=10
>> > tid=0x00007f41954b0800 nid=0x2ea1 waiting on condition
>> [0x00007f4145afc000]
>> > "Default JMS Resource Adapter-worker- - 56" daemon prio=10
>> > tid=0x00007f4194ede000 nid=0x2ea0 waiting on condition
>> [0x00007f4145bfd000]
>> > "Default JMS Resource Adapter-worker- - 55" daemon prio=10
>> > tid=0x00007f4194edc000 nid=0x2e98 waiting on condition
>> [0x00007f4145cfe000]
>> > "Default JMS Resource Adapter-worker- - 54" daemon prio=10
>> > tid=0x00007f4194eda000 nid=0x2e97 waiting on condition
>> [0x00007f4145dff000]
>> > "Default JMS Resource Adapter-worker- - 53" daemon prio=10
>> > tid=0x00007f4194ed7800 nid=0x2e96 waiting on condition
>> [0x00007f4145f00000]
>> > "Default JMS Resource Adapter-worker- - 52" daemon prio=10
>> > tid=0x00007f4194399800 nid=0x2e95 waiting on condition
>> [0x00007f4146001000]
>> > "Default JMS Resource Adapter-worker- - 51" daemon prio=10
>> > tid=0x00007f4194398000 nid=0x2e94 waiting on condition
>> [0x00007f4146102000]
>> > "Default JMS Resource Adapter-worker- - 50" daemon prio=10
>> > tid=0x00007f4194396800 nid=0x2e8f waiting on condition
>> [0x00007f4146203000]
>> > "Default JMS Resource Adapter-worker- - 49" daemon prio=10
>> > tid=0x00007f4194394800 nid=0x2e8e waiting on condition
>> [0x00007f4146304000]
>> > "Default JMS Resource Adapter-worker- - 48" daemon prio=10
>> > tid=0x00007f4194392800 nid=0x2e8d waiting on condition
>> [0x00007f4146405000]
>> > "Default JMS Resource Adapter-worker- - 47" daemon prio=10
>> > tid=0x00007f4194ac1000 nid=0x2e8c waiting on condition
>> [0x00007f4146506000]
>> > "Default JMS Resource Adapter-worker- - 46" daemon prio=10
>> > tid=0x00007f4194abf000 nid=0x2e8b waiting on condition
>> [0x00007f4146607000]
>> > "Default JMS Resource Adapter-worker- - 45" daemon prio=10
>> > tid=0x00007f4194abd000 nid=0x2e8a waiting on condition
>> [0x00007f4146708000]
>> > "Default JMS Resource Adapter-worker- - 44" daemon prio=10
>> > tid=0x00007f4194abb000 nid=0x2e89 waiting on condition
>> [0x00007f4146809000]
>> > "Default JMS Resource Adapter-worker- - 43" daemon prio=10
>> > tid=0x00007f4194508800 nid=0x2e88 waiting on condition
>> [0x00007f414690a000]
>> > "Default JMS Resource Adapter-worker- - 42" daemon prio=10
>> > tid=0x00007f4194506800 nid=0x2e87 waiting on condition
>> [0x00007f4146a0b000]
>> > "Default JMS Resource Adapter-worker- - 41" daemon prio=10
>> > tid=0x00007f4194505000 nid=0x2e86 waiting on condition
>> [0x00007f4146b0c000]
>> > "Default JMS Resource Adapter-worker- - 40" daemon prio=10
>> > tid=0x00007f4194503800 nid=0x2e85 waiting on condition
>> [0x00007f4146c0d000]
>> > "Default JMS Resource Adapter-worker- - 39" daemon prio=10
>> > tid=0x00007f4194c86000 nid=0x2e84 waiting on condition
>> [0x00007f4146d0e000]
>> > "Default JMS Resource Adapter-worker- - 38" daemon prio=10
>> > tid=0x00007f4194c84800 nid=0x2e83 waiting on condition
>> [0x00007f4146e0f000]
>> > "Default JMS Resource Adapter-worker- - 37" daemon prio=10
>> > tid=0x00007f4194c83000 nid=0x2e7b waiting on condition
>> [0x00007f4146f10000]
>> > "Default JMS Resource Adapter-worker- - 36" daemon prio=10
>> > tid=0x00007f4194c81800 nid=0x2e7a waiting on condition
>> [0x00007f4147011000]
>> > "Default JMS Resource Adapter-worker- - 35" daemon prio=10
>> > tid=0x00007f4194c80000 nid=0x2e79 waiting on condition
>> [0x00007f4147112000]
>> > "Default JMS Resource Adapter-worker- - 34" daemon prio=10
>> > tid=0x00007f4194c7f800 nid=0x2e78 waiting on condition
>> [0x00007f4147213000]
>> > "Default JMS Resource Adapter-worker- - 33" daemon prio=10
>> > tid=0x00007f41940f0000 nid=0x2e77 waiting on condition
>> [0x00007f4147314000]
>> > "Default JMS Resource Adapter-worker- - 32" daemon prio=10
>> > tid=0x00007f41940ef000 nid=0x2e76 waiting on condition
>> [0x00007f4147617000]
>> > "Default JMS Resource Adapter-worker- - 31" daemon prio=10
>> > tid=0x00007f41940ed800 nid=0x2e75 waiting on condition
>> [0x00007f4147415000]
>> > "Default JMS Resource Adapter-worker- - 30" daemon prio=10
>> > tid=0x00007f41940ec000 nid=0x2e74 waiting on condition
>> [0x00007f414803d000]
>> > "Default JMS Resource Adapter-worker- - 29" daemon prio=10
>> > tid=0x00007f41940ea800 nid=0x2e73 waiting on condition
>> [0x00007f4147a8d000]
>> > "Default JMS Resource Adapter-worker- - 28" daemon prio=10
>> > tid=0x00007f4194c7b000 nid=0x2e72 waiting on condition
>> [0x00007f414798c000]
>> > "Default JMS Resource Adapter-worker- - 27" daemon prio=10
>> > tid=0x00007f4194c7a000 nid=0x2e70 waiting on condition
>> [0x00007f4147e02000]
>> > "Default JMS Resource Adapter-worker- - 26" daemon prio=10
>> > tid=0x00007f4194d75000 nid=0x2e6f waiting on condition
>> [0x00007f4149187000]
>> > "Default JMS Resource Adapter-worker- - 25" daemon prio=10
>> > tid=0x00007f41940c1800 nid=0x2e6e waiting on condition
>> [0x00007f414877d000]
>> > "Default JMS Resource Adapter-worker- - 24" daemon prio=10
>> > tid=0x00007f41943a0800 nid=0x2e6d waiting on condition
>> [0x00007f414867c000]
>> > "Default JMS Resource Adapter-worker- - 23" daemon prio=10
>> > tid=0x00007f4194975000 nid=0x2e6c waiting on condition
>> [0x00007f414847a000]
>> > "Default JMS Resource Adapter-worker- - 22" daemon prio=10
>> > tid=0x00007f419546d000 nid=0x2e6a waiting on condition
>> [0x00007f414857b000]
>> > "Default JMS Resource Adapter-worker- - 21" daemon prio=10
>> > tid=0x00007f4194440000 nid=0x2e62 waiting on condition
>> [0x00007f4147cc8000]
>> > "Default JMS Resource Adapter-worker- - 20" daemon prio=10
>> > tid=0x00007f4194163800 nid=0x2e60 waiting on condition
>> [0x00007f4147bc7000]
>> > "Default JMS Resource Adapter-worker- - 19" daemon prio=10
>> > tid=0x00007f4194455000 nid=0x2e5e waiting on condition
>> [0x00007f414e3d9000]
>> > "Default JMS Resource Adapter-worker- - 18" daemon prio=10
>> > tid=0x00007f419550a000 nid=0x2e5d waiting on condition
>> [0x00007f414887e000]
>> > "Default JMS Resource Adapter-worker- - 17" daemon prio=10
>> > tid=0x00007f4198008000 nid=0x18ae waiting on condition
>> [0x00007f4148c82000]
>> > "Default JMS Resource Adapter-worker- - 16" daemon prio=10
>> > tid=0x00007f418406b000 nid=0x18ad waiting on condition
>> [0x00007f4148d83000]
>> > "Default JMS Resource Adapter-worker- - 15" daemon prio=10
>> > tid=0x00007f41b403e000 nid=0x18ac waiting on condition
>> [0x00007f4148e84000]
>> > "Default JMS Resource Adapter-worker- - 14" daemon prio=10
>> > tid=0x00007f41600be000 nid=0x18ab waiting on condition
>> [0x00007f41cc1e8000]
>> > "Default JMS Resource Adapter-worker- - 13" daemon prio=10
>> > tid=0x00007f4194a0b800 nid=0x15d7 waiting on condition
>> [0x00007f414c2b8000]
>> > "Default JMS Resource Adapter-worker- - 12" daemon prio=10
>> > tid=0x00007f41943f3800 nid=0x15d5 waiting on condition
>> [0x00007f414c3b9000]
>> > "Default JMS Resource Adapter-worker- - 11" daemon prio=10
>> > tid=0x00007f41943f2800 nid=0x15d1 waiting on condition
>> [0x00007f414c4ba000]
>> > "Default JMS Resource Adapter-worker- - 10" daemon prio=10
>> > tid=0x00007f4194a0b000 nid=0x15cf in Object.wait() [0x00007f414c6bb000]
>> > "Default JMS Resource Adapter-worker- - 9" daemon prio=10
>> > tid=0x00007f4194a0a800 nid=0x15ca waiting on condition
>> [0x00007f414c5bb000]
>> > "Default JMS Resource Adapter-worker- - 8" daemon prio=10
>> > tid=0x00007f4194136800 nid=0x15c7 waiting on condition
>> [0x00007f414cac0000]
>> > "Default JMS Resource Adapter-worker- - 7" daemon prio=10
>> > tid=0x00007f41941b9800 nid=0x15c5 waiting on condition
>> [0x00007f414c8be000]
>> > "Default JMS Resource Adapter-worker- - 6" daemon prio=10
>> > tid=0x00007f4194af7800 nid=0x15bf waiting on condition
>> [0x00007f414cbc1000]
>> > "Default JMS Resource Adapter-worker- - 5" daemon prio=10
>> > tid=0x00007f4194788800 nid=0x15bd waiting on condition
>> [0x00007f414d2c8000]
>> > "Default JMS Resource Adapter-worker- - 4" daemon prio=10
>> > tid=0x00007f4194788000 nid=0x15b9 waiting on condition
>> [0x00007f414d0c6000]
>> > "Default JMS Resource Adapter-worker- - 3" daemon prio=10
>> > tid=0x00007f415cb24800 nid=0x1567 waiting on condition
>> [0x00007f414f4f3000]
>> > "Default JMS Resource Adapter-worker- - 2" daemon prio=10
>> > tid=0x00007f415cb24000 nid=0x1566 waiting on condition
>> [0x00007f414f3f2000]
>> > "Default JMS Resource Adapter-worker- - 1" daemon prio=10
>> > tid=0x00007f415cb1f800 nid=0x1565 waiting on condition
>> [0x00007f414f1f0000]
>> > [leoks@oc7612866413 ~]$ grep "Default JMS Resource Adapter" dump.txt |
>> wc
>> > 99 1385 13352
>> >
>> >
>> > []
>> >
>> > Leo
>> >
>> >
>> > On Mon, Feb 10, 2014 at 4:19 PM, Romain Manni-Bucau
>> > <rmannibu...@gmail.com>wrote:
>> >
>> > > do a thread dump during it
>> > > Romain Manni-Bucau
>> > > Twitter: @rmannibucau
>> > > Blog: http://rmannibucau.wordpress.com/
>> > > LinkedIn: http://fr.linkedin.com/in/rmannibucau
>> > > Github: https://github.com/rmannibucau
>> > >
>> > >
>> > >
>> > > 2014-02-10 18:54 GMT+01:00 Leonardo K. Shikida <shik...@gmail.com>:
>> > > > not sure how do I check that
>> > > >
>> > > > I only have one producer that enqueues like 10K messages at once and
>> > then
>> > > > consumers start to consume (each message is processed in 2-3 minutes
>> by
>> > > the
>> > > > MDB)
>> > > >
>> > > > sometimes it reaches 20 consumers, but usually it stabilizes in 10
>> > > > consumers (everytime one MDB is consuming, I set a flag in the db)
>> > > >
>> > > >
>> > > >
>> > > > []
>> > > >
>> > > > Leo
>> > > >
>> > > >
>> > > > On Mon, Feb 10, 2014 at 3:45 PM, Romain Manni-Bucau
>> > > > <rmannibu...@gmail.com>wrote:
>> > > >
>> > > >> Seems ok. Are your threads waiting for amq?
>> > > >> Le 10 févr. 2014 18:15, "Leonardo K. Shikida" <shik...@gmail.com> a
>> > > écrit
>> > > >> :
>> > > >>
>> > > >> > My activemq is this. Limits look pretty high for me.
>> > > >> >
>> > > >> > <!-- Licensed to the Apache Software Foundation (ASF) under
>> one
>> > or
>> > > >> more
>> > > >> > contributor license agreements. See the NOTICE file distributed
>> with
>> > > this
>> > > >> > work for additional information regarding
>> > > >> > copyright ownership. The ASF licenses this file to You
>> under
>> > > the
>> > > >> > Apache License, Version 2.0 (the "License"); you may not use this
>> > file
>> > > >> > except in compliance with the License. You may
>> > > >> > obtain a copy of the License at
>> > > >> > http://www.apache.org/licenses/LICENSE-2.0 Unless required by
>> > > applicable
>> > > >> > law or agreed to in writing, software distributed under the
>> License
>> > is
>> > > >> > distributed
>> > > >> > on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
>> ANY
>> > > >> KIND,
>> > > >> > either express or implied. See the License for the specific
>> language
>> > > >> > governing permissions and limitations under the
>> > > >> > License. -->
>> > > >> > <beans xmlns="http://www.springframework.org/schema/beans";
>> > > >> xmlns:amq="
>> > > >> > http://activemq.apache.org/schema/core"; xmlns:xsi="
>> > > >> > http://www.w3.org/2001/XMLSchema-instance";
>> > > >> > xsi:schemaLocation="
>> > > http://www.springframework.org/schema/beans
>> > > >> > http://www.springframework.org/schema/beans/spring-beans.xsd
>> > > >> > http://activemq.apache.org/schema/core
>> > > >> > http://activemq.apache.org/schema/core/activemq-core.xsd";>
>> > > >> >
>> > > >> >
>> > > >> > <broker xmlns="http://activemq.apache.org/schema/core";
>> > > >> > brokerName="localhost" dataDirectory="${activemq.data}">
>> > > >> > <persistenceAdapter>
>> > > >> > <jdbcPersistenceAdapter
>> dataSource="#oracle-ds"/>
>> > > >> > </persistenceAdapter>
>> > > >> >
>> > > >> > <systemUsage>
>> > > >> > <systemUsage>
>> > > >> > <memoryUsage>
>> > > >> > <memoryUsage limit="1024 mb" />
>> > > >> > </memoryUsage>
>> > > >> > <storeUsage>
>> > > >> > <storeUsage limit="500 gb" />
>> > > >> > </storeUsage>
>> > > >> > <tempUsage>
>> > > >> > <tempUsage limit="500 gb" />
>> > > >> > </tempUsage>
>> > > >> > </systemUsage>
>> > > >> > </systemUsage>
>> > > >> >
>> > > >> > <transportConnectors>
>> > > >> > <transportConnector name="tcp" uri="tcp://
>> > > 0.0.0.0:61616
>> > > >> "/>
>> > > >> > </transportConnectors>
>> > > >> > </broker>
>> > > >> >
>> > > >> > <bean id="oracle-ds"
>> > > >> > class="org.apache.commons.dbcp.BasicDataSource"
>> > > destroy-method="close">
>> > > >> > <property name="driverClassName"
>> > > >> > value="oracle.jdbc.OracleDriver"/>
>> > > >> > <property name="url"
>> value="jdbc:oracle:thin:@localhost
>> > > >> > :1521:XE"/>
>> > > >> > <property name="username" value="xxx"/>
>> > > >> > <property name="password" value="xxx"/>
>> > > >> > <property name="poolPreparedStatements" value="true"/>
>> > > >> > </bean>
>> > > >> >
>> > > >> > </beans>
>> > > >> >
>> > > >> >
>> > > >> > []
>> > > >> >
>> > > >> > Leo
>> > > >> >
>> > > >> >
>> > > >> > On Mon, Feb 10, 2014 at 2:42 PM, Romain Manni-Bucau
>> > > >> > <rmannibu...@gmail.com>wrote:
>> > > >> >
>> > > >> > > Hi
>> > > >> > >
>> > > >> > > any specific config in your activemq.xml which could limit it?
>> > maybe
>> > > >> > > check through jmx (active it in activemq.xml) you have not a
>> limit
>> > > >> > > set)
>> > > >> > > Romain Manni-Bucau
>> > > >> > > Twitter: @rmannibucau
>> > > >> > > Blog: http://rmannibucau.wordpress.com/
>> > > >> > > LinkedIn: http://fr.linkedin.com/in/rmannibucau
>> > > >> > > Github: https://github.com/rmannibucau
>> > > >> > >
>> > > >> > >
>> > > >> > >
>> > > >> > > 2014-02-10 17:40 GMT+01:00 Leonardo K. Shikida <
>> shik...@gmail.com
>> > >:
>> > > >> > > > Hi
>> > > >> > > >
>> > > >> > > > I was following this to try to push more MDBs do consume from
>> a
>> > > >> single
>> > > >> > > queue
>> > > >> > > >
>> > > >> > > > http://openejb.979440.n4.nabble.com/30-Limit-td981453.html
>> > > >> > > >
>> > > >> > > > I am trying this
>> > > >> > > >
>> > > >> > > > tomee.xml
>> > > >> > > >
>> > > >> > > > <Resource id="Default JMS Resource Adapter"
>> > > >> > > > type="ActiveMQResourceAdapter">
>> > > >> > > > BrokerXmlConfig = xbean:file:/pathto/activemq.xml
>> > > >> > > > ServerUrl = tcp://0.0.0.0:61616
>> > > >> > > > threadPoolSize 100
>> > > >> > > > </Resource>
>> > > >> > > >
>> > > >> > > > <Container id="Foo" type="MESSAGE">
>> > > >> > > > InstanceLimit 100
>> > > >> > > > </Container>
>> > > >> > > >
>> > > >> > > > and in the MDB
>> > > >> > > >
>> > > >> > > > (...)
>> > > >> > > > @ActivationConfigProperty(
>> > > >> > > > propertyName = "maxSessions",
>> > > >> > > > propertyValue = "100"),
>> > > >> > > > @ActivationConfigProperty(
>> > > >> > > > propertyName = "maxMessagesPerSessions",
>> > > >> > > > propertyValue = "100") })
>> > > >> > > > public class MyWorker implements MessageListener {
>> > > >> > > >
>> > > >> > > > After that, my consumers went from 10 (default) to 20, but I'd
>> > > like
>> > > >> to
>> > > >> > > push
>> > > >> > > > to 100.
>> > > >> > > >
>> > > >> > > > Am I missing something?
>> > > >> > > >
>> > > >> > > > TIA
>> > > >> > > >
>> > > >> > > > Leo
>> > > >> > >
>> > > >> >
>> > > >>
>> > >
>> >
>>
