Set test.runners based on cores and memory size patch by Jay Zhuang; reviewed by Ariel Weisberg for CASSANDRA-13078
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2400d07b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2400d07b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2400d07b Branch: refs/heads/cassandra-3.11 Commit: 2400d07bf52829b25a7c03c19b22ddd3301899be Parents: 97fb4d1 Author: Jay Zhuang <jay.zhu...@yahoo.com> Authored: Thu Jul 6 18:01:38 2017 -0700 Committer: Ariel Weisberg <aweisb...@apple.com> Committed: Tue Jul 11 14:08:14 2017 -0400 ---------------------------------------------------------------------- CHANGES.txt | 1 + build.xml | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++----- circle.yml | 2 +- 3 files changed, 55 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/2400d07b/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 8095e25..ce2324d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.15 + * Set test.runners based on cores and memory size (CASSANDRA-13078) * Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557) * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606) * Fix secondary index queries on COMPACT tables (CASSANDRA-13627) http://git-wip-us.apache.org/repos/asf/cassandra/blob/2400d07b/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 5eb6572..53c2cea 100644 --- a/build.xml +++ b/build.xml @@ -59,7 +59,6 @@ <property name="test.classlistfile" value="testlist.txt"/> <property name="benchmark.name" value=""/> <property name="test.methods" value=""/> - <property name="test.runners" value="1"/> <property name="test.unit.src" value="${test.dir}/unit"/> <property name="test.long.src" value="${test.dir}/long"/> <property name="test.burn.src" value="${test.dir}/burn"/> @@ -1625,12 +1624,25 @@ <scriptdef name="testparallelhelper" language="javascript"> <attribute name="testdelegate"/> <![CDATA[ - var Integer = java.lang.Integer; sep = project.getProperty("path.separator"); all = project.getProperty("all-test-classes").split(sep); - dir = project.getProperty("test.unit.src"); + runners = project.getProperty("test.runners") + cores = project.getProperty("cores.count") + mem = project.getProperty("mem.size") - numRunners = parseInt(project.getProperty("test.runners")); + numRunners = 1 + if (runners != null) // there's test.runners override + numRunners = parseInt(runners) || 1; + else if (cores != null && mem != null) // only if cores and memory size is set + numRunners = Math.min(Math.floor(Math.sqrt(parseInt(cores) || 1)), + Math.floor((parseInt(mem) || 1)/(4*1024*1024*1024))); + + if (numRunners < 1) + numRunners = 1 + + var echo = project.createTask("echo"); + echo.setMessage("Number of test runners: " + numRunners); + echo.perform(); var p = project.createTask('parallel'); p.setThreadCount(numRunners); @@ -1652,7 +1664,42 @@ ]]> </scriptdef> - <target name="test" depends="build-test" description="Parallel Test Runner"> + <target name="get-cores"> + <property environment="env"/> + <!-- support for Windows --> + <condition property="cores.count" value="${env.NUMBER_OF_PROCESSORS}"> + <os family="windows" /> + </condition> + <!-- support for Linux and Solaris (package SUNWgnu-coreutils is required) --> + <exec executable="nproc" outputproperty="cores.count" os="Linux,SunOS,Solaris" failifexecutionfails="false"> + <arg value="--all"/> + </exec> + <!-- support for Mac OS X --> + <exec executable="sysctl" outputproperty="cores.count" os="Mac,Mac OS X,Darwin" failifexecutionfails="false"> + <arg value="-n"/> + <arg value="hw.ncpu"/> + </exec> + <echo message="Number of cores: ${cores.count}"/> + </target> + + <target name="get-mem"> + <condition property="mem.size" value="unknown"> + <os family="windows" /> + </condition> + <!-- support for Linux and Solaris (package SUNWgnu-coreutils is required) --> + <exec executable="bash" outputproperty="mem.size" os="Linux,SunOS,Solaris" failifexecutionfails="false"> + <arg value="-c"/> + <arg value="free -b | grep Mem: | awk '{print $2}'"/> + </exec> + <!-- support for Mac OS X --> + <exec executable="sysctl" outputproperty="mem.size" os="Mac,Mac OS X,Darwin" failifexecutionfails="false"> + <arg value="-n"/> + <arg value="hw.memsize"/> + </exec> + <echo message="Mem size : ${mem.size}"/> + </target> + + <target name="test" depends="build-test,get-cores,get-mem" description="Parallel Test Runner"> <path id="all-test-classes-path"> <fileset dir="${test.unit.src}" includes="**/${test.name}.java" /> </path> @@ -1933,4 +1980,5 @@ file="${build.dir}/${final.name}-javadoc.jar" classifier="javadoc"/> </target> + </project> http://git-wip-us.apache.org/repos/asf/cassandra/blob/2400d07b/circle.yml ---------------------------------------------------------------------- diff --git a/circle.yml b/circle.yml index 9d31277..f4801b7 100644 --- a/circle.yml +++ b/circle.yml @@ -7,7 +7,7 @@ test: - sudo apt-get update; sudo apt-get install wamerican: parallel: true override: - - case $CIRCLE_NODE_INDEX in 0) ant eclipse-warnings; ant test ;; 1) ant long-test ;; 2) ant test-compression ;; 3) ant stress-test ;;esac: + - case $CIRCLE_NODE_INDEX in 0) ant eclipse-warnings; ant test -Dtest.runners=1;; 1) ant long-test ;; 2) ant test-compression ;; 3) ant stress-test ;;esac: parallel: true post: --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
