Strange bottleneck after concurrent users reach 10K

[Mingfan Lu]

I using oliophp to stress a machine (16Core) as web server with two other DB
nodes ( a master_slave cluster, master using a SATA disk while slave using a
SSD disk)
When #concurrent users scaling  from 9K 10K 11K 12K 13K 14K 15K 16K the
throughput increasing and then decreasing. It seems that there are some
bottleneck here.
*User
**Throughput(Ops)
* *14000* *1843.955*  *13000* *1849.213*  *12000* *1842.368*  *11000* *
1859.053*  *10000* *1969.393*  *9000* *1810.323* My ramp time is 300s while
steady time is 600s and the rampdown is 60s
The client start up:
    Time between starts (ms) :1
    Start simultaneously: No
    Start agents in parallel: No
See my attachment run.xml

But my profiling data shows that the CPU( Highest is about 80%~90% when
#concurrent user is 10000, softirq% is about 14% with *4tx and 4rx* queues )
/ Networks Bandwidth(70% of 1Gb) /Memory Usage/Disk are not the bottleneck.
The Apache error log is very clean with no exception and error. At the same
time I have disabled the static images serving (Just disable all *<img* tag
in the HTML)
>From the pictures in
http://docs.google.com/present/view?id=df7282nf_30x8gwmrch&autoStart=true ,
when 9K concurrent user, the response time is steady enough, when 10K, there
is pulse lasting 600sec (what happen?) and down to very small enough in the
last 300sec.
I want to know what cause the strange pulse when concurrent users reach 10K?

<?xml version="1.0" encoding="UTF-8"?>
<olio>
    <jvmConfig xmlns="http://faban.sunsource.net/ns/fabanharness";>
        <javaHome>/usr/java/jdk1.6.0_11</javaHome>
        <jvmOptions>-Xmx7g -Xms7g -XX:+DisableExplicitGC</jvmOptions>
    </jvmConfig>
    
    <fa:runConfig definition="org.apache.olio.workload.driver.UIDriver" xmlns="http://faban.sunsource.net/ns/fabandriver"; xmlns:fa="http://faban.sunsource.net/ns/faban"; xmlns:fh="http://faban.sunsource.net/ns/fabanharness";>
        <fh:description>Enter description for this run here</fh:description>
        
        <fa:hostConfig>
            <fa:host>jicsr108 jicsr109 jicsr110 jicsr111</fa:host>
            <fh:enabled>true</fh:enabled>
            <fh:cpus>0</fh:cpus>
            <fh:tools></fh:tools>
            <fh:userCommands/>
        </fa:hostConfig>
        
        <fa:scale>10000</fa:scale>
        
        <fa:runControl>
            <fa:rampUp>300</fa:rampUp>
            <fa:steadyState>600</fa:steadyState>
            <fa:rampDown>60</fa:rampDown>
        </fa:runControl>

        <outputDir>/tmp/cloudstress/OlioDriver</outputDir>
        
        <audit>false</audit>
        <threadStart>            
            <delay>1</delay>            
            <simultaneous>false</simultaneous>            
            <parallel>false</parallel>
        </threadStart>
        
        <stats>
            <maxRunTime>6</maxRunTime>
            <interval>30</interval>
        </stats>
        
        <runtimeStats enabled="false">
            <interval>5</interval>
        </runtimeStats>
        
        <driverConfig name="UIDriver">            
            <agents>8</agents>            
            <stats>
                <interval>30</interval>
            </stats>            
            <runtimeStats target="9988"/>
            <properties>
                <property name="serverType">php</property>
            </properties>
        </driverConfig>
    </fa:runConfig>
    
    <webServer>
        <fa:hostConfig xmlns="http://faban.sunsource.net/ns/fabanharness"; xmlns:fa="http://faban.sunsource.net/ns/faban";><fa:host>jicsr117</fa:host>
            <fa:hostPorts>jicsr117:80</fa:hostPorts>
            <enabled>true</enabled>
            <cpus>0</cpus>
            <tools></tools>           
            <userCommands/>            
        </fa:hostConfig>
        <type>apache</type>
        <hostBinPath>/opt/jiclamp/apache/bin</hostBinPath>
        <hostLogPath>/opt/jiclamp/apache/logs</hostLogPath>
        <hostConfPath>/opt/jiclamp/apache/conf</hostConfPath>
        <phpIniPath>/opt/jiclamp/apache/conf</phpIniPath>
    </webServer>
    <dbServer>
        <fa:hostConfig xmlns="http://faban.sunsource.net/ns/fabanharness"; xmlns:fa="http://faban.sunsource.net/ns/faban";>
            <fa:host>jicsr102 jicsr115</fa:host>
            <enabled>true</enabled>
            <cpus>0</cpus>
            <tools></tools>
            <userCommands/>
        </fa:hostConfig>
        <dbDriver>com.mysql.jdbc.Driver</dbDriver>
        <connectURL>jdbc:mysql://localhost/olio?user=olio&amp;password=olio&amp;relaxAutoCommit=true&amp;sessionVariables=FOREIGN_KEY_CHECKS=0</connectURL>
        <autoCommit>true</autoCommit>
        <reloadDB></reloadDB>
        <scale>10000</scale>
        <dbHomePath>/opt/jiclamp/mysql</dbHomePath>
        <dbConfPath>/opt/jiclamp/mysql/etc</dbConfPath>
    </dbServer>
    <dataStorage>
        <fa:hostConfig xmlns="http://faban.sunsource.net/ns/fabanharness"; xmlns:fa="http://faban.sunsource.net/ns/faban";>
            <fa:host>jicsr117</fa:host>
            <enabled>true</enabled>
            <cpus>0</cpus>
            <tools>NONE</tools>
            <userCommands/>
        </fa:hostConfig>
        <reloadMedia>false</reloadMedia>
        <mediaDir>/opt/oliofile</mediaDir>
    </dataStorage>
    <cacheServers>
        <fa:hostConfig xmlns="http://faban.sunsource.net/ns/fabanharness"; xmlns:fa="http://faban.sunsource.net/ns/faban";>
		<fa:host>jicsr117</fa:host>
            <fa:hostPorts>jicsr117:11000</fa:hostPorts>
            <enabled>true</enabled>
            <cpus>0</cpus>
            <tools>NONE</tools>
            <userCommands/>
        </fa:hostConfig>
        <cacheBinPath>/opt/memcached/bin</cacheBinPath>
    </cacheServers>
    
</olio>