Strange bottleneck after concurrent users reach 10K

Thu, 24 Sep 2009 18:26:47 -0700 

I using oliophp to stress a machine (16Core) as web server with two other DB
nodes ( a master_slave cluster, master using a SATA disk while slave using a
SSD disk)
When #concurrent users scaling  from 9K 10K 11K 12K 13K 14K 15K 16K the
throughput increasing and then decreasing.
9k      *1810.323*
10K    *1969.393*
11K    *1859.053*
12K    *1842.368*
13K    *1849.213*
14K    *1843.955*

 It seems that there are some bottleneck here.

Detail to see the attached run.xml
My ramp time is 300s while steady time is 600s and the rampdown is 60s
The client start up:
    Time between starts (ms) :1
    Start simultaneously: No
    Start agents in parallel: No

But my profiling data shows that the CPU( Highest is about 80%~90% when
#concurrent user is 10000, softirq% is about 14% with *4tx and 4rx* queues )
/ Networks Bandwidth(70% of 1Gb) /Memory Usage/Disk are not the bottleneck.
The Apache error log is very clean with no exception and error. At the same
time I have disabled the static images serving (Just disable all *img* tag
in the HTML)

>From the pictures in
http://docs.google.com/present/view?id=df7282nf_30x8gwmrch&autoStart=true ,
when 9K concurrent user, the response time is steady enough, when 10K, there
is pulse lasting 600sec (what happen?) and down to very small enough in the
last 300sec.
I want to know what cause the strange pulse when concurrent users reach 10K?

<?xml version="1.0" encoding="UTF-8"?>
<olio>
    <jvmConfig xmlns="http://faban.sunsource.net/ns/fabanharness";>
        <javaHome>/usr/java/jdk1.6.0_11</javaHome>
        <jvmOptions>-Xmx7g -Xms7g -XX:+DisableExplicitGC</jvmOptions>
    </jvmConfig>
    
    <fa:runConfig definition="org.apache.olio.workload.driver.UIDriver" xmlns="http://faban.sunsource.net/ns/fabandriver"; xmlns:fa="http://faban.sunsource.net/ns/faban"; xmlns:fh="http://faban.sunsource.net/ns/fabanharness";>
        <fh:description>Enter description for this run here</fh:description>
        
        <fa:hostConfig>
            <fa:host>jicsr108 jicsr109 jicsr110 jicsr111</fa:host>
            <fh:enabled>true</fh:enabled>
            <fh:cpus>0</fh:cpus>
            <fh:tools></fh:tools>
            <fh:userCommands/>
        </fa:hostConfig>
        
        <fa:scale>10000</fa:scale>
        
        <fa:runControl>
            <fa:rampUp>300</fa:rampUp>
            <fa:steadyState>600</fa:steadyState>
            <fa:rampDown>60</fa:rampDown>
        </fa:runControl>

        <outputDir>/tmp/cloudstress/OlioDriver</outputDir>
        
        <audit>false</audit>
        <threadStart>            
            <delay>1</delay>            
            <simultaneous>false</simultaneous>            
            <parallel>false</parallel>
        </threadStart>
        
        <stats>
            <maxRunTime>6</maxRunTime>
            <interval>30</interval>
        </stats>
        
        <runtimeStats enabled="false">
            <interval>5</interval>
        </runtimeStats>
        
        <driverConfig name="UIDriver">            
            <agents>8</agents>            
            <stats>
                <interval>30</interval>
            </stats>            
            <runtimeStats target="9988"/>
            <properties>
                <property name="serverType">php</property>
            </properties>
        </driverConfig>
    </fa:runConfig>
    
    <webServer>
        <fa:hostConfig xmlns="http://faban.sunsource.net/ns/fabanharness"; xmlns:fa="http://faban.sunsource.net/ns/faban";><fa:host>jicsr117</fa:host>
            <fa:hostPorts>jicsr117:80</fa:hostPorts>
            <enabled>true</enabled>
            <cpus>0</cpus>
            <tools></tools>           
            <userCommands/>            
        </fa:hostConfig>
        <type>apache</type>
        <hostBinPath>/opt/jiclamp/apache/bin</hostBinPath>
        <hostLogPath>/opt/jiclamp/apache/logs</hostLogPath>
        <hostConfPath>/opt/jiclamp/apache/conf</hostConfPath>
        <phpIniPath>/opt/jiclamp/apache/conf</phpIniPath>
    </webServer>
    <dbServer>
        <fa:hostConfig xmlns="http://faban.sunsource.net/ns/fabanharness"; xmlns:fa="http://faban.sunsource.net/ns/faban";>
            <fa:host>jicsr102 jicsr115</fa:host>
            <enabled>true</enabled>
            <cpus>0</cpus>
            <tools></tools>
            <userCommands/>
        </fa:hostConfig>
        <dbDriver>com.mysql.jdbc.Driver</dbDriver>
        <connectURL>jdbc:mysql://localhost/olio?user=olio&amp;password=olio&amp;relaxAutoCommit=true&amp;sessionVariables=FOREIGN_KEY_CHECKS=0</connectURL>
        <autoCommit>true</autoCommit>
        <reloadDB></reloadDB>
        <scale>10000</scale>
        <dbHomePath>/opt/jiclamp/mysql</dbHomePath>
        <dbConfPath>/opt/jiclamp/mysql/etc</dbConfPath>
    </dbServer>
    <dataStorage>
        <fa:hostConfig xmlns="http://faban.sunsource.net/ns/fabanharness"; xmlns:fa="http://faban.sunsource.net/ns/faban";>
            <fa:host>jicsr117</fa:host>
            <enabled>true</enabled>
            <cpus>0</cpus>
            <tools>NONE</tools>
            <userCommands/>
        </fa:hostConfig>
        <reloadMedia>false</reloadMedia>
        <mediaDir>/opt/oliofile</mediaDir>
    </dataStorage>
    <cacheServers>
        <fa:hostConfig xmlns="http://faban.sunsource.net/ns/fabanharness"; xmlns:fa="http://faban.sunsource.net/ns/faban";>
		<fa:host>jicsr117</fa:host>
            <fa:hostPorts>jicsr117:11000</fa:hostPorts>
            <enabled>true</enabled>
            <cpus>0</cpus>
            <tools>NONE</tools>
            <userCommands/>
        </fa:hostConfig>
        <cacheBinPath>/opt/memcached/bin</cacheBinPath>
    </cacheServers>
    
</olio>

Reply via email to