HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer
(cherry picked from commit 45d1b0fdcc04a86be91a9b72073cdc30bec04d3b) (cherry picked from commit 09940b1eb3b7ed764149f4a993c1857e9c6ad938) (cherry picked from commit a03c8ea61f6e30a9d462571ace23858b6e0fd1c9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/340cd5f1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/340cd5f1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/340cd5f1 Branch: refs/heads/branch-2 Commit: 340cd5f1b137dccc033023ed92cc87b7e47f45c2 Parents: 38c21ec Author: Chris Douglas <cdoug...@apache.org> Authored: Mon Mar 12 19:47:42 2018 -0700 Committer: Chris Douglas <cdoug...@apache.org> Committed: Mon Mar 12 20:08:46 2018 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/pom.xml | 26 +---- .../apache/hadoop/test/GenericTestUtils.java | 68 +++++++++---- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 26 +---- .../plugin/paralleltests/CreateDirsMojo.java | 100 +++++++++++++++++++ hadoop-tools/hadoop-aws/pom.xml | 26 +---- 5 files changed, 161 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/340cd5f1/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 8c9dffc..85bb5d3 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -795,30 +795,13 @@ <build> <plugins> <plugin> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> - <phase>test-compile</phase> - <configuration> - <target> - <script language="javascript"><![CDATA[ - var baseDirs = [ - "${test.build.data}", - "${test.build.dir}", - "${hadoop.tmp.dir}" ]; - for (var i in baseDirs) { - for (var j = 1; j <= ${testsThreadCount}; ++j) { - var mkdir = project.createTask("mkdir"); - mkdir.setDir(new java.io.File(baseDirs[i], j)); - mkdir.perform(); - } - } - ]]></script> - </target> - </configuration> + <id>parallel-tests-createdir</id> <goals> - <goal>run</goal> + <goal>parallel-tests-createdir</goal> </goals> </execution> </executions> @@ -831,6 +814,7 @@ <reuseForks>false</reuseForks> <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> + <testsThreadCount>${testsThreadCount}</testsThreadCount> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> http://git-wip-us.apache.org/repos/asf/hadoop/blob/340cd5f1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index af47d29..2810213 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -287,7 +287,7 @@ public abstract class GenericTestUtils { public static void assertExists(File f) { Assert.assertTrue("File " + f + " should exist", f.exists()); } - + /** * List all of the files in 'dir' that match the regex 'pattern'. * Then check that this list is identical to 'expectedMatches'. @@ -295,7 +295,7 @@ public abstract class GenericTestUtils { */ public static void assertGlobEquals(File dir, String pattern, String ... expectedMatches) throws IOException { - + Set<String> found = Sets.newTreeSet(); for (File f : FileUtil.listFiles(dir)) { if (f.getName().matches(pattern)) { @@ -333,7 +333,7 @@ public abstract class GenericTestUtils { + StringUtils.stringifyException(t), t); } - } + } /** * Wait for the specified test to return true. The test will be performed @@ -483,18 +483,18 @@ public abstract class GenericTestUtils { */ public static class DelayAnswer implements Answer<Object> { private final Log LOG; - + private final CountDownLatch fireLatch = new CountDownLatch(1); private final CountDownLatch waitLatch = new CountDownLatch(1); private final CountDownLatch resultLatch = new CountDownLatch(1); - + private final AtomicInteger fireCounter = new AtomicInteger(0); private final AtomicInteger resultCounter = new AtomicInteger(0); - + // Result fields set after proceed() is called. private volatile Throwable thrown; private volatile Object returnValue; - + public DelayAnswer(Log log) { this.LOG = log; } @@ -505,7 +505,7 @@ public abstract class GenericTestUtils { public void waitForCall() throws InterruptedException { fireLatch.await(); } - + /** * Tell the method to proceed. * This should only be called after waitForCall() @@ -513,7 +513,7 @@ public abstract class GenericTestUtils { public void proceed() { waitLatch.countDown(); } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { LOG.info("DelayAnswer firing fireLatch"); @@ -542,7 +542,7 @@ public abstract class GenericTestUtils { resultLatch.countDown(); } } - + /** * After calling proceed(), this will wait until the call has * completed and a result has been returned to the caller. @@ -550,7 +550,7 @@ public abstract class GenericTestUtils { public void waitForResult() throws InterruptedException { resultLatch.await(); } - + /** * After the call has gone through, return any exception that * was thrown, or null if no exception was thrown. @@ -558,7 +558,7 @@ public abstract class GenericTestUtils { public Throwable getThrown() { return thrown; } - + /** * After the call has gone through, return the call's return value, * or null in case it was void or an exception was thrown. @@ -566,20 +566,20 @@ public abstract class GenericTestUtils { public Object getReturnValue() { return returnValue; } - + public int getFireCount() { return fireCounter.get(); } - + public int getResultCount() { return resultCounter.get(); } } - + /** * An Answer implementation that simply forwards all calls through * to a delegate. - * + * * This is useful as the default Answer for a mock object, to create * something like a spy on an RPC proxy. For example: * <code> @@ -590,14 +590,14 @@ public abstract class GenericTestUtils { * ... * </code> */ - public static class DelegateAnswer implements Answer<Object> { + public static class DelegateAnswer implements Answer<Object> { private final Object delegate; private final Log log; - + public DelegateAnswer(Object delegate) { this(null, delegate); } - + public DelegateAnswer(Log log, Object delegate) { this.log = log; this.delegate = delegate; @@ -637,7 +637,7 @@ public abstract class GenericTestUtils { this.minSleepTime = minSleepTime; this.maxSleepTime = maxSleepTime; } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { boolean interrupted = false; @@ -667,11 +667,11 @@ public abstract class GenericTestUtils { " but got:\n" + output, Pattern.compile(pattern).matcher(output).find()); } - + public static void assertValueNear(long expected, long actual, long allowedError) { assertValueWithinRange(expected - allowedError, expected + allowedError, actual); } - + public static void assertValueWithinRange(long expectedMin, long expectedMax, long actual) { Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + "," @@ -826,4 +826,28 @@ public abstract class GenericTestUtils { failf(format, args); } } + + /** + * Retreive the max number of parallel test threads when running under maven. + * @return int number of threads + */ + public static int getTestsThreadCount() { + String propString = System.getProperty("testsThreadCount", "1"); + int threadCount = 1; + if (propString != null) { + String trimProp = propString.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(trimProp); + } + } + return threadCount; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/340cd5f1/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 88d916f..483be36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -434,30 +434,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <build> <plugins> <plugin> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> - <phase>test-compile</phase> - <configuration> - <target> - <script language="javascript"><![CDATA[ - var baseDirs = [ - "${test.build.data}", - "${test.build.dir}", - "${hadoop.tmp.dir}" ]; - for (var i in baseDirs) { - for (var j = 1; j <= ${testsThreadCount}; ++j) { - var mkdir = project.createTask("mkdir"); - mkdir.setDir(new java.io.File(baseDirs[i], j)); - mkdir.perform(); - } - } - ]]></script> - </target> - </configuration> + <id>parallel-tests-createdir</id> <goals> - <goal>run</goal> + <goal>parallel-tests-createdir</goal> </goals> </execution> </executions> @@ -470,6 +453,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <reuseForks>false</reuseForks> <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> + <testsThreadCount>${testsThreadCount}</testsThreadCount> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> http://git-wip-us.apache.org/repos/asf/hadoop/blob/340cd5f1/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java ---------------------------------------------------------------------- diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java new file mode 100644 index 0000000..4c9b9d7 --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.paralleltests; + +import java.io.File; + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + + +/** + * Goal which creates the parallel-test directories. + */ +@Mojo(name="parallel-tests-createdir", + defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES) +public class CreateDirsMojo extends AbstractMojo { + + /** + * Location of the test.build.dir. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildDir; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildData; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/tmp") + private File hadoopTmpDir; + + /** + * Thread count. + */ + @Parameter(defaultValue="${testsThreadCount}") + private String testsThreadCount; + + public void execute() throws MojoExecutionException { + int numDirs=getTestsThreadCount(); + + mkParallelDirs(testBuildDir, numDirs); + mkParallelDirs(testBuildData, numDirs); + mkParallelDirs(hadoopTmpDir, numDirs); + } + + /** + * Get the real number of parallel threads. + * @return int number of threads + */ + + public int getTestsThreadCount() { + int threadCount = 1; + if (testsThreadCount != null) { + String trimProp = testsThreadCount.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(testsThreadCount); + } + } + return threadCount; + } + + private void mkParallelDirs(File testDir, int numDirs) + throws MojoExecutionException { + for (int i=1; i<=numDirs; i++) { + File newDir = new File(testDir, String.valueOf(i)); + if (!newDir.exists()) { + getLog().info("Creating " + newDir.toString()); + if (!newDir.mkdirs()) { + throw new MojoExecutionException("Unable to create " + + newDir.toString()); + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/340cd5f1/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 659af93..ea49aa4 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -85,30 +85,13 @@ <build> <plugins> <plugin> - <artifactId>maven-antrun-plugin</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> - <phase>test-compile</phase> - <configuration> - <target> - <script language="javascript"><![CDATA[ - var baseDirs = [ - "${test.build.data}", - "${test.build.dir}", - "${hadoop.tmp.dir}" ]; - for (var i in baseDirs) { - for (var j = 1; j <= ${testsThreadCount}; ++j) { - var mkdir = project.createTask("mkdir"); - mkdir.setDir(new java.io.File(baseDirs[i], j)); - mkdir.perform(); - } - } - ]]></script> - </target> - </configuration> + <id>parallel-tests-createdir</id> <goals> - <goal>run</goal> + <goal>parallel-tests-createdir</goal> </goals> </execution> </executions> @@ -121,6 +104,7 @@ <reuseForks>false</reuseForks> <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> + <testsThreadCount>${testsThreadCount}</testsThreadCount> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
