I am just writing at the moment and want to commit that into Ants sandbox
"parallelexecutor" ;)
My basic idea is:
- each target should run in its own thread
- each thread could start if all dependent thread stopped successfully
Ant is relying on Java 1.3, but for better concurrency support I am using Java
1.6.
ATM I am thinking about how to test that ...
Jan
---8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<----
package org.apache.ant.parallelexecutor;
import static org.apache.ant.parallelexecutor.TargetContainerStatus.WAITING;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Executor;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.Target;
/**
* <p>This executors parallelizes the exution of Ant targets.
* Each target will run in its own thread. That thread will be started
* if all dependend targets are finished, the if/unless attributes are
* evaluated and no dependend target failed. When a TargetContainer finishes
* it calls this Executors <tt>targetFinished</tt> so that the Executor could
* restart all the waiting threads.</p>
* <p>This executor is used via Ants magic property
<tt>ant.executor.class</tt></p>
* <pre>ant
-Dant.executor.class=org.apache.ant.parallelexecutor.ParallelExecutor</pre>
*/
public class ParallelExecutor implements Executor {
/**
* Default value for waiting for shutting down the ExecutorService.
* @see ExecutorService#awaitTermination(long, TimeUnit)
* @see #EXECUTOR_SERVICE_SHUTDOWN_TIME_UNIT
*/
public static final long EXECUTOR_SERVICE_SHUTDOWN_TIME_VALUE = 10;
/**
* TimeUnit for the shutdown time.
*/
public static final TimeUnit EXECUTOR_SERVICE_SHUTDOWN_TIME_UNIT =
TimeUnit.MINUTES;
/**
* Targets which should run, wrapped by TargetContainers for threading and
monitoring.
*/
private Set<TargetContainer> targetsToProcess;
/**
* ExecutorService for Thread-creation.
*/
private ExecutorService executorService;
/**
* Entry-point defined in the Executor interface.
* Initializes this Executor and starts the threads.
* @see
org.apache.tools.ant.Executor#executeTargets(org.apache.tools.ant.Project,
java.lang.String[])
*/
public void executeTargets(Project project, String[] targetNames) throws
BuildException {
targetsToProcess = getTargetsToProcess(project, targetNames);
executorService = java.util.concurrent.Executors.newCachedThreadPool();
startWaitingContainers();
}
/**
* Initializes the list of TargetContainers with all targets which should
be started.
* @param project project containing the targets
* @param targetNames list of the targets to start
* @return list of TargetContainers for these targets
*/
private Set<TargetContainer> getTargetsToProcess(Project project, String[]
targetNames) {
Set<TargetContainer> rv = new HashSet<TargetContainer>();
for (String targetName : targetNames) {
Target target = (Target)project.getTargets().get(targetName);
TargetContainer container = new TargetContainer(target, this);
targetsToProcess.add(container);
}
return rv;
}
/**
* not used
* @see org.apache.tools.ant.Executor#getSubProjectExecutor()
*/
public Executor getSubProjectExecutor() {
return null;
}
/**
* Starts all waiting TargetContainers.
* @return <tt>true</tt> if one or more containers were be started,
<tt>false</tt> otherwise.
*/
private boolean startWaitingContainers() {
boolean hasStartedAContainer = false;
for (TargetContainer container : targetsToProcess) {
if (container.getCurrentStatus() == WAITING) {
executorService.execute(container);
hasStartedAContainer = true;
}
}
return hasStartedAContainer;
}
/**
* Call-back method for finishing TargetContainers.
* @param container TargetContainer which finished.
*/
public void targetFinished(TargetContainer container) {
if (!startWaitingContainers()) {
// no more waiting containers, so we have finished
try {
// ... if there were no running targets ...
executorService.awaitTermination(
EXECUTOR_SERVICE_SHUTDOWN_TIME_VALUE,
EXECUTOR_SERVICE_SHUTDOWN_TIME_UNIT);
} catch (InterruptedException e) {
// no-op
}
buildFinished();
}
}
/**
* Collects all the results from the different targets.
*/
private void buildFinished() {
List<String> runningTargets = new ArrayList<String>();
List<BuildException> thrownExceptions = new ArrayList<BuildException>();
// Checks the targets for BuildExceptions and their state.
for (TargetContainer container : targetsToProcess) {
if (!container.getCurrentStatus().hasFinished()) {
runningTargets.add(container.getName());
}
if (container.getBuildExeption()!=null) {
thrownExceptions.add(container.getBuildExeption());
}
}
if (!runningTargets.isEmpty()) {
// There are still running targets - shouldn't be, so add an error.
StringBuilder sb = new StringBuilder();
sb.append("Shutting down while having running targets: ");
for (int i=0; i<runningTargets.size()-1; i++) {
sb.append(runningTargets.get(i)).append(", ");
}
sb.append(runningTargets.get(runningTargets.size()));
thrownExceptions.add(new BuildException(sb.toString()));
}
// throw BuildExceptions if needed
throwCaughtExceptions(thrownExceptions);
}
/**
* Checks the given list of BuildExceptions and <ol>
* <li>throws a composite BuildException with the information provided by
that list</li>
* <li>throws the BuildException if the list does contain only one
exception</li>
* <li>does not throw any Exception if the list is empty</li>
* </ul>
* @param thrownExceptions list of caught exceptions
*/
private void throwCaughtExceptions(List<BuildException> thrownExceptions) {
if (thrownExceptions.isEmpty()) {
return;
}
if (thrownExceptions.size() == 1) {
throw thrownExceptions.get(0);
}
// Collect all BEs into one new
StringBuilder sb = new StringBuilder();
sb.append("Multiple BuildExceptions occured:")
.append(System.getProperty("line.separator"));
for (BuildException be : thrownExceptions) {
sb.append("\t")
.append(be.getLocation())
.append(" : ")
.append(be.getMessage())
.append(System.getProperty("line.separator"));
}
throw new BuildException(sb.toString());
}
/**
* Returns the current status for a given target.
* @param depName name of the target
* @return status of that target
*/
public TargetContainerStatus getStatus(String depName) {
return getContainer(depName).getCurrentStatus();
}
/**
* Returns a TargetContainer by its name.
* @param targetName name of the target to look for
* @return the target container wrapping that target
*/
private TargetContainer getContainer(String targetName) {
for (TargetContainer container : targetsToProcess) {
if (container.getName().equals(targetName)) {
return container;
}
}
return null;
}
}
---8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<----
package org.apache.ant.parallelexecutor;
import java.util.Enumeration;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Target;
import static org.apache.ant.parallelexecutor.TargetContainerStatus.*;
/**
* The TargetContainer wrapps a target for the use in a multithreaded
environment.
* It provides the "management methods" needed by the ParallelExecutor.
*/
public class TargetContainer implements Runnable {
/** Current status of the target. */
private TargetContainerStatus currentStatus;
/** Wrapped target. */
private Target target;
/** Caught exception if the target throws one. */
private BuildException caughtBuildExeption;
/** The calling ParallelExecutor to inform on finished works. */
private ParallelExecutor caller;
/**
* Constructor.
* @param target target to wrap
* @param caller calling exectuor for call back
*/
public TargetContainer(Target target, ParallelExecutor caller) {
this.target = target;
this.caller = caller;
setCurrentStatus(WAITING);
}
/**
* Gets the current status of the TargetContainer.
* @return status
*/
public TargetContainerStatus getCurrentStatus() {
return currentStatus;
}
/**
* Sets the current status of the TargetContainer.
* @param currentStatus new status
*/
private void setCurrentStatus(TargetContainerStatus currentStatus) {
synchronized (this.currentStatus) {
this.currentStatus = currentStatus;
}
}
/**
* Called by the [EMAIL PROTECTED] ParallelExecutor} if the target should
try
* to start.
* @see java.lang.Runnable#run()
*/
@Override
public void run() {
setCurrentStatus(CHECKING);
TargetContainerStatus statusDepends = checkDependentTargets();
if (statusDepends == SUCCESSED) {
if (evaluateIf() && !evaluateUnless()) {
setCurrentStatus(RUNNING);
try {
target.execute();
setCurrentStatus(SUCCESSED);
} catch (BuildException be) {
// Don't handle the be, just store it for handling by the
executor.
caughtBuildExeption = be;
setCurrentStatus(FAILED);
}
} else {
// 'if' or 'unless' attributes failed
setCurrentStatus(PREREQUISITE_FAILED);
}
} else {
// Finishing the run method stops the thread. Status here is
WAITING or PREFAILED
setCurrentStatus(statusDepends);
}
}
/**
* Checks the result of the dependend targets.
* @return <tt>SUCCESSED</tt> if <b>all</b> targets finished successfully,
* <tt>PREREQUISITE_FAILED</tt> if one or more failed or
* <tt>WAITING</tt> otherwise.
*/
private TargetContainerStatus checkDependentTargets() {
for (Enumeration deps = target.getDependencies();
deps.hasMoreElements();) {
TargetContainerStatus status = caller.getStatus((String)
deps.nextElement());
if (status == FAILED || status == PREREQUISITE_FAILED) {
return PREREQUISITE_FAILED;
}
if (status != SUCCESSED) {
return WAITING;
}
}
return SUCCESSED;
}
/**
* Checks if the property specified as <tt>unless</tt> is set.
* @return <tt>true</tt> if set, <tt>false</tt> otherwise
*/
private boolean evaluateUnless() {
return target.getProject().getProperty(target.getUnless()) != null;
}
/**
* Checks if the property specified as <tt>if</tt> is set.
* @return <tt>true</tt> if set, <tt>false</tt> otherwise
*/
private boolean evaluateIf() {
return target.getProject().getProperty(target.getIf()) != null;
}
/**
* Gets the name of the target.
* @return target name
* @see Target#getName()
*/
public String getName() {
return target.getName();
}
/**
* Returns the caught exception thrown by the target.
* @return the exception or <tt>null</tt>
*/
public BuildException getBuildExeption() {
return caughtBuildExeption;
}
}
---8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<----
package org.apache.ant.parallelexecutor;
/**
* TargetContainer can be in the here defined states.
* The transistion between them is:<pre>
* start --> WAITING
* WAITING --> CHECKING
* CHECKING --> RUNNING, PREREQUISITE_FAILED
* RUNNING --> SUCCESSED, FAILED
* SUCCESSED --> end
* FAILED --> end
* PREREQUISITE_FAILED --> end
* </pre>
*/
public enum TargetContainerStatus {
/* Waiting for starting by the Executor. */
WAITING,
/* Checking prerequisites (depends, if, unless). */
CHECKING,
/* Target is currently running. */
RUNNING,
/* Target has finished without any error. */
SUCCESSED,
/* Target has thrown a BuildException. */
FAILED,
/* Target could not run because a prerequisite has not succeeded. */
PREREQUISITE_FAILED;
/**
* Utility method for easier access to a set of states.
* <pre>Finished = [SUCCESSED|FAILED|PREREQUISITE_FAILED]</pre>
* @return computed state
*/
public boolean hasFinished() {
return this == SUCCESSED || this == FAILED || this ==
PREREQUISITE_FAILED;
}
}
---8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<----
> -----Ursprüngliche Nachricht-----
> Von: Klaus Malorny [mailto:[EMAIL PROTECTED]
> Gesendet: Dienstag, 18. Dezember 2007 10:10
> An: Ant Users List
> Betreff: extended parallelism
>
>
> Hi,
>
> with the increased availability of multi-core systems, I am
> wondering whether
> there are ways to improve the capabilities of Ant to
> parallelize the build
> process. While it is possible to execute tasks in parallel
> within a target, it
> seems to not be simple to execute multiple targets in
> parallel. My vision is
> an extension to the "depends" attribute: if a pipe is used
> instead of a comma,
> the target does not care about the order and the targets may
> be executed in
> parallel by Ant. I would also add parenthesis to group
> things. For example, "(a
> | (b, c)), d)" means, c must be executed after b, but both
> may be executed in
> parallel to a, while d must be executed only after completion
> of a, b and c. Of
> course, more constraints may occur in the dependent targets,
> e.g. if c depends
> on a, they can't be executed in parallel. Just an idea, feel
> free to ignore it ;-)
>
> Klaus
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: [EMAIL PROTECTED]
> For additional commands, e-mail: [EMAIL PROTECTED]
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
