Repository: cassandra Updated Branches: refs/heads/cassandra-2.1 1fb88c0b5 -> 23893ead5
Paralellize the unit tests patch by tjake; reviewed by thobbs for CASSANDRA-7442 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/23893ead Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/23893ead Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/23893ead Branch: refs/heads/cassandra-2.1 Commit: 23893ead557989bb74d699e1028c9c6a1b689f3b Parents: 1fb88c0 Author: Jake Luciani <j...@apache.org> Authored: Wed Jun 25 17:09:31 2014 -0400 Committer: Jake Luciani <j...@apache.org> Committed: Wed Jun 25 17:09:31 2014 -0400 ---------------------------------------------------------------------- build.xml | 78 +++++++++++++++++++- .../OffsetAwareConfigurationLoader.java | 63 ++++++++++++++++ test/unit/org/apache/cassandra/cli/CliTest.java | 3 +- 3 files changed, 140 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/23893ead/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 1f975f0..c34bcfc 100644 --- a/build.xml +++ b/build.xml @@ -57,6 +57,7 @@ <property name="test.data" value="${test.dir}/data"/> <property name="test.name" value="*Test"/> <property name="test.methods" value=""/> + <property name="test.runners" value="1"/> <property name="test.unit.src" value="${test.dir}/unit"/> <property name="test.long.src" value="${test.dir}/long"/> <property name="test.pig.src" value="${test.dir}/pig"/> @@ -1060,11 +1061,13 @@ <macrodef name="testmacro"> <attribute name="suitename" /> <attribute name="inputdir" /> - <attribute name="timeout" /> + <attribute name="timeout" default="${test.timeout}" /> <attribute name="forkmode" default="perTest"/> <element name="optjvmargs" implicit="true" optional="true" /> <attribute name="filter" default="**/${test.name}.java"/> <attribute name="exclude" default="" /> + <attribute name="filelist" default="" /> + <attribute name="poffset" default="0"/> <sequential> <echo message="running @{suitename} tests"/> <mkdir dir="${build.test.dir}/cassandra"/> @@ -1080,7 +1083,8 @@ <jvmarg value="-Xss256k"/> <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/> <jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/> - <optjvmargs/> + <jvmarg value="-Dcassandra.test.offsetseed=@{poffset}"/> + <optjvmargs/> <classpath> <path refid="cassandra.classpath" /> <pathelement location="${test.classes}"/> @@ -1092,6 +1096,7 @@ </classpath> <batchtest todir="${build.test.dir}/output"> <fileset dir="@{inputdir}" includes="@{filter}" excludes="@{exclude}"/> + <filelist dir="@{inputdir}" files="@{filelist}"/> </batchtest> </junit> <fail message="Some @{suitename} test(s) failed."> @@ -1134,7 +1139,7 @@ </junit> </target> - <target name="test" depends="build-test" description="Execute unit tests"> + <target name="testold" depends="build-test" description="Execute unit tests"> <testmacro suitename="unit" inputdir="${test.unit.src}" exclude="**/pig/*.java" timeout="${test.timeout}"> <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/> <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/> @@ -1146,6 +1151,17 @@ <exclude name="**/pig/*.java" /> </fileset> </target> + + <target name="testlist"> + <testmacro suitename="${testlist.name}" inputdir="${test.unit.src}" filelist="${test.file.list}" poffset="${testlist.offset}" exclude="**/*.java" timeout="${test.timeout}"> + <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/> + <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/> + <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/> + <jvmarg value="-Dcassandra.ring_delay_ms=1000"/> + <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/> + <jvmarg value="-Dcassandra.config.loader=org.apache.cassandra.OffsetAwareConfigurationLoader"/> + </testmacro> + </target> <target name="testsome" depends="build-test" description="Execute specific unit tests" > <testmacro suitename="unit" inputdir="${test.unit.src}" exclude="**/pig/*.java" timeout="${test.timeout}"> @@ -1334,6 +1350,62 @@ </create-javadoc> </target> + <!-- Split test classes into n buckets and run across processes --> + <target name="test" depends="build-test" description="Parallel Test Runner"> + <path id="all-test-classes-path"> + <fileset dir="${test.unit.src}" excludes="**/pig/*.java" includes="**/${test.name}.java" /> + </path> + <property name="all-test-classes" refid="all-test-classes-path"/> + <script language="javascript"> <![CDATA[ + importClass(java.lang.Integer) + sep = project.getProperty("path.separator"); + all = project.getProperty("all-test-classes").split(sep); + dir = project.getProperty("test.unit.src"); + + numRunners = parseInt(project.getProperty("test.runners")); + + buckets = new Array(numRunners); + for (i = 0; i < all.length; i++) { + bucketNum = i % numRunners; + if (buckets[bucketNum] == undefined) + buckets[bucketNum] = ""; + else + buckets[bucketNum] += ","; + + buckets[bucketNum] += all[i]; + } + + + var p = project.createTask('parallel'); + p.setThreadCount(numRunners); + + for (i = 0; i < buckets.length; i++) { + + if (buckets[i] == undefined) continue; + + task = project.createTask( 'antcall' ); + + task.setTarget("testlist"); + param = task.createParam(); + param.setName("test.file.list"); + param.setValue(buckets[i]); + + param = task.createParam(); + param.setName("testlist.name"); + param.setValue("test bucket "+i); + + param = task.createParam(); + param.setName("testlist.offset"); + param.setValue(i); + + p.addTask(task); + } + + p.perform(); + +]]> </script> + </target> + <!-- Generate Eclipse project description files --> <target name="generate-eclipse-files" depends="build" description="Generate eclipse files"> <echo file=".project"><![CDATA[<?xml version="1.0" encoding="UTF-8"?> http://git-wip-us.apache.org/repos/asf/cassandra/blob/23893ead/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java b/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java new file mode 100644 index 0000000..9023b11 --- /dev/null +++ b/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.YamlConfigurationLoader; +import org.apache.cassandra.exceptions.ConfigurationException; + +import java.io.File; + + +public class OffsetAwareConfigurationLoader extends YamlConfigurationLoader +{ + + static final String OFFSET_PROPERTY = "cassandra.test.offsetseed"; + int offset = 0; + + public OffsetAwareConfigurationLoader() + { + String offsetStr = System.getProperty(OFFSET_PROPERTY); + + if (offsetStr == null) + throw new RuntimeException("offset property is not set: "+OFFSET_PROPERTY); + + offset = Integer.valueOf(offsetStr); + + assert offset >= 0; + } + + @Override + public Config loadConfig() throws ConfigurationException + { + Config config = super.loadConfig(); + + + config.rpc_port += offset; + config.native_transport_port += offset; + config.storage_port += offset; + + config.commitlog_directory += File.pathSeparator + offset; + config.saved_caches_directory += File.pathSeparator + offset; + for (int i = 0; i < config.data_file_directories.length; i++) + config.data_file_directories[i] += File.pathSeparator + offset; + + + return config; + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/23893ead/test/unit/org/apache/cassandra/cli/CliTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/cli/CliTest.java b/test/unit/org/apache/cassandra/cli/CliTest.java index 5d28c70..38f567b 100644 --- a/test/unit/org/apache/cassandra/cli/CliTest.java +++ b/test/unit/org/apache/cassandra/cli/CliTest.java @@ -19,6 +19,7 @@ package org.apache.cassandra.cli; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.service.EmbeddedCassandraService; import org.apache.cassandra.thrift.*; @@ -226,7 +227,7 @@ public class CliTest extends SchemaLoader ByteArrayOutputStream outStream = new ByteArrayOutputStream(); // checking if we can connect to the running cassandra node on localhost - CliMain.connect("127.0.0.1", 9170); + CliMain.connect("127.0.0.1", DatabaseDescriptor.getRpcPort()); // setting new output stream CliMain.sessionState.setOut(new PrintStream(outStream));
