HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45d1b0fd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45d1b0fd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45d1b0fd Branch: refs/heads/HDFS-7240 Commit: 45d1b0fdcc04a86be91a9b72073cdc30bec04d3b Parents: 19292bc Author: Chris Douglas <cdoug...@apache.org> Authored: Mon Mar 12 19:47:42 2018 -0700 Committer: Chris Douglas <cdoug...@apache.org> Committed: Mon Mar 12 20:05:39 2018 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/pom.xml | 26 +---- .../apache/hadoop/test/GenericTestUtils.java | 68 +++++++++---- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 26 +---- .../plugin/paralleltests/CreateDirsMojo.java | 100 +++++++++++++++++++ hadoop-tools/hadoop-aws/pom.xml | 26 +---- 5 files changed, 161 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/45d1b0fd/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 078a943..49d3575 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -979,30 +979,13 @@ <build> <plugins> <plugin> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> - <phase>test-compile</phase> - <configuration> - <target> - <script language="javascript"><![CDATA[ - var baseDirs = [ - "${test.build.data}", - "${test.build.dir}", - "${hadoop.tmp.dir}" ]; - for (var i in baseDirs) { - for (var j = 1; j <= ${testsThreadCount}; ++j) { - var mkdir = project.createTask("mkdir"); - mkdir.setDir(new java.io.File(baseDirs[i], j)); - mkdir.perform(); - } - } - ]]></script> - </target> - </configuration> + <id>parallel-tests-createdir</id> <goals> - <goal>run</goal> + <goal>parallel-tests-createdir</goal> </goals> </execution> </executions> @@ -1015,6 +998,7 @@ <reuseForks>false</reuseForks> <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> + <testsThreadCount>${testsThreadCount}</testsThreadCount> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> http://git-wip-us.apache.org/repos/asf/hadoop/blob/45d1b0fd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index cdde48c..61b0271 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -286,7 +286,7 @@ public abstract class GenericTestUtils { public static void assertExists(File f) { Assert.assertTrue("File " + f + " should exist", f.exists()); } - + /** * List all of the files in 'dir' that match the regex 'pattern'. * Then check that this list is identical to 'expectedMatches'. @@ -294,7 +294,7 @@ public abstract class GenericTestUtils { */ public static void assertGlobEquals(File dir, String pattern, String ... expectedMatches) throws IOException { - + Set<String> found = Sets.newTreeSet(); for (File f : FileUtil.listFiles(dir)) { if (f.getName().matches(pattern)) { @@ -349,7 +349,7 @@ public abstract class GenericTestUtils { StringUtils.stringifyException(t)), t); } - } + } /** * Wait for the specified test to return true. The test will be performed @@ -499,18 +499,18 @@ public abstract class GenericTestUtils { */ public static class DelayAnswer implements Answer<Object> { private final Log LOG; - + private final CountDownLatch fireLatch = new CountDownLatch(1); private final CountDownLatch waitLatch = new CountDownLatch(1); private final CountDownLatch resultLatch = new CountDownLatch(1); - + private final AtomicInteger fireCounter = new AtomicInteger(0); private final AtomicInteger resultCounter = new AtomicInteger(0); - + // Result fields set after proceed() is called. private volatile Throwable thrown; private volatile Object returnValue; - + public DelayAnswer(Log log) { this.LOG = log; } @@ -521,7 +521,7 @@ public abstract class GenericTestUtils { public void waitForCall() throws InterruptedException { fireLatch.await(); } - + /** * Tell the method to proceed. * This should only be called after waitForCall() @@ -529,7 +529,7 @@ public abstract class GenericTestUtils { public void proceed() { waitLatch.countDown(); } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { LOG.info("DelayAnswer firing fireLatch"); @@ -558,7 +558,7 @@ public abstract class GenericTestUtils { resultLatch.countDown(); } } - + /** * After calling proceed(), this will wait until the call has * completed and a result has been returned to the caller. @@ -566,7 +566,7 @@ public abstract class GenericTestUtils { public void waitForResult() throws InterruptedException { resultLatch.await(); } - + /** * After the call has gone through, return any exception that * was thrown, or null if no exception was thrown. @@ -574,7 +574,7 @@ public abstract class GenericTestUtils { public Throwable getThrown() { return thrown; } - + /** * After the call has gone through, return the call's return value, * or null in case it was void or an exception was thrown. @@ -582,20 +582,20 @@ public abstract class GenericTestUtils { public Object getReturnValue() { return returnValue; } - + public int getFireCount() { return fireCounter.get(); } - + public int getResultCount() { return resultCounter.get(); } } - + /** * An Answer implementation that simply forwards all calls through * to a delegate. - * + * * This is useful as the default Answer for a mock object, to create * something like a spy on an RPC proxy. For example: * <code> @@ -606,14 +606,14 @@ public abstract class GenericTestUtils { * ... * </code> */ - public static class DelegateAnswer implements Answer<Object> { + public static class DelegateAnswer implements Answer<Object> { private final Object delegate; private final Log log; - + public DelegateAnswer(Object delegate) { this(null, delegate); } - + public DelegateAnswer(Log log, Object delegate) { this.log = log; this.delegate = delegate; @@ -653,7 +653,7 @@ public abstract class GenericTestUtils { this.minSleepTime = minSleepTime; this.maxSleepTime = maxSleepTime; } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { boolean interrupted = false; @@ -683,11 +683,11 @@ public abstract class GenericTestUtils { " but got:\n" + output, Pattern.compile(pattern).matcher(output).find()); } - + public static void assertValueNear(long expected, long actual, long allowedError) { assertValueWithinRange(expected - allowedError, expected + allowedError, actual); } - + public static void assertValueWithinRange(long expectedMin, long expectedMax, long actual) { Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + "," @@ -842,4 +842,28 @@ public abstract class GenericTestUtils { failf(format, args); } } + + /** + * Retreive the max number of parallel test threads when running under maven. + * @return int number of threads + */ + public static int getTestsThreadCount() { + String propString = System.getProperty("testsThreadCount", "1"); + int threadCount = 1; + if (propString != null) { + String trimProp = propString.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(trimProp); + } + } + return threadCount; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/45d1b0fd/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index d6afed1..909e589 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -515,30 +515,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <build> <plugins> <plugin> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> - <phase>test-compile</phase> - <configuration> - <target> - <script language="javascript"><![CDATA[ - var baseDirs = [ - "${test.build.data}", - "${test.build.dir}", - "${hadoop.tmp.dir}" ]; - for (var i in baseDirs) { - for (var j = 1; j <= ${testsThreadCount}; ++j) { - var mkdir = project.createTask("mkdir"); - mkdir.setDir(new java.io.File(baseDirs[i], j)); - mkdir.perform(); - } - } - ]]></script> - </target> - </configuration> + <id>parallel-tests-createdir</id> <goals> - <goal>run</goal> + <goal>parallel-tests-createdir</goal> </goals> </execution> </executions> @@ -551,6 +534,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <reuseForks>false</reuseForks> <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> + <testsThreadCount>${testsThreadCount}</testsThreadCount> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> http://git-wip-us.apache.org/repos/asf/hadoop/blob/45d1b0fd/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java ---------------------------------------------------------------------- diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java new file mode 100644 index 0000000..4c9b9d7 --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.paralleltests; + +import java.io.File; + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + + +/** + * Goal which creates the parallel-test directories. + */ +@Mojo(name="parallel-tests-createdir", + defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES) +public class CreateDirsMojo extends AbstractMojo { + + /** + * Location of the test.build.dir. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildDir; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildData; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/tmp") + private File hadoopTmpDir; + + /** + * Thread count. + */ + @Parameter(defaultValue="${testsThreadCount}") + private String testsThreadCount; + + public void execute() throws MojoExecutionException { + int numDirs=getTestsThreadCount(); + + mkParallelDirs(testBuildDir, numDirs); + mkParallelDirs(testBuildData, numDirs); + mkParallelDirs(hadoopTmpDir, numDirs); + } + + /** + * Get the real number of parallel threads. + * @return int number of threads + */ + + public int getTestsThreadCount() { + int threadCount = 1; + if (testsThreadCount != null) { + String trimProp = testsThreadCount.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(testsThreadCount); + } + } + return threadCount; + } + + private void mkParallelDirs(File testDir, int numDirs) + throws MojoExecutionException { + for (int i=1; i<=numDirs; i++) { + File newDir = new File(testDir, String.valueOf(i)); + if (!newDir.exists()) { + getLog().info("Creating " + newDir.toString()); + if (!newDir.mkdirs()) { + throw new MojoExecutionException("Unable to create " + + newDir.toString()); + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/45d1b0fd/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index c704a62..24ed11d 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -85,30 +85,13 @@ <build> <plugins> <plugin> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> - <phase>test-compile</phase> - <configuration> - <target> - <script language="javascript"><![CDATA[ - var baseDirs = [ - "${test.build.data}", - "${test.build.dir}", - "${hadoop.tmp.dir}" ]; - for (var i in baseDirs) { - for (var j = 1; j <= ${testsThreadCount}; ++j) { - var mkdir = project.createTask("mkdir"); - mkdir.setDir(new java.io.File(baseDirs[i], j)); - mkdir.perform(); - } - } - ]]></script> - </target> - </configuration> + <id>parallel-tests-createdir</id> <goals> - <goal>run</goal> + <goal>parallel-tests-createdir</goal> </goals> </execution> </executions> @@ -121,6 +104,7 @@ <reuseForks>false</reuseForks> <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> + <testsThreadCount>${testsThreadCount}</testsThreadCount> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
