ant-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From <Jan.Mate...@rzf.fin-nrw.de>
Subject AW: extended parallelism
Date Tue, 18 Dec 2007 09:19:58 GMT
I am just writing at the moment and want to commit that into Ants sandbox "parallelexecutor"
;)
My basic idea is:
- each target should run in its own thread
- each thread could start if all dependent thread stopped successfully

Ant is relying on Java 1.3, but for better concurrency support I am using Java 1.6.

ATM I am thinking about how to test that ...



Jan



---8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<----
package org.apache.ant.parallelexecutor;

import static org.apache.ant.parallelexecutor.TargetContainerStatus.WAITING;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;

import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Executor;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.Target;

/**
 * <p>This executors parallelizes the exution of Ant targets.
 * Each target will run in its own thread. That thread will be started
 * if all dependend targets are finished, the if/unless attributes are
 * evaluated and no dependend target failed. When a TargetContainer finishes
 * it calls this Executors <tt>targetFinished</tt> so that the Executor could
 * restart all the waiting threads.</p>
 * <p>This executor is used via Ants magic property <tt>ant.executor.class</tt></p>
 * <pre>ant -Dant.executor.class=org.apache.ant.parallelexecutor.ParallelExecutor</pre>
 */
public class ParallelExecutor implements Executor {
    
    /** 
     * Default value for waiting for shutting down the ExecutorService.
     * @see ExecutorService#awaitTermination(long, TimeUnit)
     * @see #EXECUTOR_SERVICE_SHUTDOWN_TIME_UNIT
     */
    public static final long EXECUTOR_SERVICE_SHUTDOWN_TIME_VALUE = 10;

    /**
     * TimeUnit for the shutdown time.
     */
    public static final TimeUnit EXECUTOR_SERVICE_SHUTDOWN_TIME_UNIT = TimeUnit.MINUTES;
    
    
    /**
     * Targets which should run, wrapped by TargetContainers for threading and monitoring.
     */
    private Set<TargetContainer> targetsToProcess;

    
    /**
     * ExecutorService for Thread-creation.
     */
    private ExecutorService executorService;


    /**
     * Entry-point defined in the Executor interface.
     * Initializes this Executor and starts the threads. 
     * @see org.apache.tools.ant.Executor#executeTargets(org.apache.tools.ant.Project, java.lang.String[])
     */
    public void executeTargets(Project project, String[] targetNames) throws BuildException
{
        targetsToProcess = getTargetsToProcess(project, targetNames);
        executorService = java.util.concurrent.Executors.newCachedThreadPool();
        startWaitingContainers();
    }
    
    
    /**
     * Initializes the list of TargetContainers with all targets which should be started.
     * @param project      project containing the targets
     * @param targetNames  list of the targets to start
     * @return             list of TargetContainers for these targets
     */
    private Set<TargetContainer> getTargetsToProcess(Project project, String[] targetNames)
{
        Set<TargetContainer> rv = new HashSet<TargetContainer>();
        for (String targetName : targetNames) {
            Target target = (Target)project.getTargets().get(targetName);
            TargetContainer container = new TargetContainer(target, this);
            targetsToProcess.add(container);
        }
        return rv;
    }
    
    
    /**
     * not used
     * @see org.apache.tools.ant.Executor#getSubProjectExecutor()
     */
    public Executor getSubProjectExecutor() {
        return null;
    }
    
    
    /**
     * Starts all waiting TargetContainers.
     * @return <tt>true</tt> if one or more containers were be started, <tt>false</tt>
otherwise.
     */
    private boolean startWaitingContainers() {
        boolean hasStartedAContainer = false;
        for (TargetContainer container : targetsToProcess) {
            if (container.getCurrentStatus() == WAITING) {
                executorService.execute(container);
                hasStartedAContainer = true;
            }
        }
        return hasStartedAContainer;
    }


    /**
     * Call-back method for finishing TargetContainers. 
     * @param container TargetContainer which finished.
     */
    public void targetFinished(TargetContainer container) {
        if (!startWaitingContainers()) {
            // no more waiting containers, so we have finished
            try {
                // ... if there were no running targets ...
                executorService.awaitTermination(
                        EXECUTOR_SERVICE_SHUTDOWN_TIME_VALUE,
                        EXECUTOR_SERVICE_SHUTDOWN_TIME_UNIT);
            } catch (InterruptedException e) {
                // no-op
            }
            buildFinished();
        }
    }


    /**
     * Collects all the results from the different targets.
     */
    private void buildFinished() {
        List<String> runningTargets = new  ArrayList<String>();
        List<BuildException> thrownExceptions = new ArrayList<BuildException>();
        
        // Checks the targets for BuildExceptions and their state.
        for (TargetContainer container : targetsToProcess) {
            if (!container.getCurrentStatus().hasFinished()) {
                runningTargets.add(container.getName());
            }
            if (container.getBuildExeption()!=null) { 
                thrownExceptions.add(container.getBuildExeption());
            }
        }
        
        if (!runningTargets.isEmpty()) {
            // There are still running targets - shouldn't be, so add an error.
            StringBuilder sb = new StringBuilder();
            sb.append("Shutting down while having running targets: ");
            for (int i=0; i<runningTargets.size()-1; i++) {
                sb.append(runningTargets.get(i)).append(", ");
            }
            sb.append(runningTargets.get(runningTargets.size()));
            thrownExceptions.add(new BuildException(sb.toString()));
        }

        // throw BuildExceptions if needed
        throwCaughtExceptions(thrownExceptions);
    }


    /**
     * Checks the given list of BuildExceptions and <ol>
     * <li>throws a composite BuildException with the information provided by that list</li>
     * <li>throws the BuildException if the list does contain only one exception</li>
     * <li>does not throw any Exception if the list is empty</li>
     * </ul>
     * @param thrownExceptions list of caught exceptions
     */
    private void throwCaughtExceptions(List<BuildException> thrownExceptions) {
        if (thrownExceptions.isEmpty()) {
            return;
        }
        if (thrownExceptions.size() == 1) {
            throw thrownExceptions.get(0);
        }
        // Collect all BEs into one new 
        StringBuilder sb = new StringBuilder();
        sb.append("Multiple BuildExceptions occured:")
          .append(System.getProperty("line.separator"));
        for (BuildException be : thrownExceptions) {
            sb.append("\t")
              .append(be.getLocation())
              .append(" : ")
              .append(be.getMessage())
              .append(System.getProperty("line.separator"));
        }
        throw new BuildException(sb.toString());
    }


    /**
     * Returns the current status for a given target.
     * @param depName name of the target
     * @return status of that target
     */
    public TargetContainerStatus getStatus(String depName) {
        return getContainer(depName).getCurrentStatus();
    }
    
    
    /**
     * Returns a TargetContainer by its name.
     * @param targetName name of the target to look for
     * @return the target container wrapping that target
     */
    private TargetContainer getContainer(String targetName) {
        for (TargetContainer container : targetsToProcess) {
            if (container.getName().equals(targetName)) {
                return container;
            }
        }
        return null;
    }
    
}
---8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<----
package org.apache.ant.parallelexecutor;

import java.util.Enumeration;

import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Target;

import static org.apache.ant.parallelexecutor.TargetContainerStatus.*;


/**
 * The TargetContainer wrapps a target for the use in a multithreaded environment.
 * It provides the "management methods" needed by the ParallelExecutor.
 */
public class TargetContainer implements Runnable {
    
    /** Current status of the target. */
    private TargetContainerStatus currentStatus;
    
    /** Wrapped target. */
    private Target target;
    
    /** Caught exception if the target throws one. */
    private BuildException caughtBuildExeption;
    
    /** The calling ParallelExecutor to inform on finished works. */
    private ParallelExecutor caller;

    
    /**
     * Constructor.
     * @param target target to wrap
     * @param caller calling exectuor for call back
     */
    public TargetContainer(Target target, ParallelExecutor caller) {
        this.target = target;
        this.caller = caller;
        setCurrentStatus(WAITING);
    }

    
    /**
     * Gets the current status of the TargetContainer.
     * @return status
     */
    public TargetContainerStatus getCurrentStatus() {
        return currentStatus;
    }

    
    /**
     * Sets the current status of the TargetContainer.
     * @param currentStatus new status
     */
    private void setCurrentStatus(TargetContainerStatus currentStatus) {
        synchronized (this.currentStatus) {
            this.currentStatus = currentStatus;
        }
    }


    /**
     * Called by the {@link ParallelExecutor} if the target should try
     * to start.
     * @see java.lang.Runnable#run()
     */
    @Override
    public void run() {
        setCurrentStatus(CHECKING);
        TargetContainerStatus statusDepends = checkDependentTargets();
        if (statusDepends == SUCCESSED) {
            if (evaluateIf() && !evaluateUnless()) {
                setCurrentStatus(RUNNING);
                try {
                    target.execute();
                    setCurrentStatus(SUCCESSED);
                } catch (BuildException be) {
                    // Don't handle the be, just store it for handling by the executor.
                    caughtBuildExeption = be;
                    setCurrentStatus(FAILED);
                }
            } else {
                // 'if' or 'unless' attributes failed
                setCurrentStatus(PREREQUISITE_FAILED);
            }
        } else {
            // Finishing the run method stops the thread. Status here is WAITING or PREFAILED
            setCurrentStatus(statusDepends);
        }
    }
    

    /**
     * Checks the result of the dependend targets. 
     * @return <tt>SUCCESSED</tt> if <b>all</b> targets finished successfully,
     *         <tt>PREREQUISITE_FAILED</tt> if one or more failed or
     *         <tt>WAITING</tt> otherwise. 
     */
    private TargetContainerStatus checkDependentTargets() {
        for (Enumeration deps = target.getDependencies(); deps.hasMoreElements();) {
            TargetContainerStatus status = caller.getStatus((String) deps.nextElement());
            if (status == FAILED || status == PREREQUISITE_FAILED) {
                return PREREQUISITE_FAILED;
            }
            if (status != SUCCESSED) {
                return WAITING;
            }
        }
        return SUCCESSED;
    }


    /**
     * Checks if the property specified as <tt>unless</tt> is set.
     * @return <tt>true</tt> if set, <tt>false</tt> otherwise
     */
    private boolean evaluateUnless() {
        return target.getProject().getProperty(target.getUnless()) != null;
    }


    /**
     * Checks if the property specified as <tt>if</tt> is set.
     * @return <tt>true</tt> if set, <tt>false</tt> otherwise
     */
    private boolean evaluateIf() {
        return target.getProject().getProperty(target.getIf()) != null;
    }


    /**
     * Gets the name of the target.
     * @return target name
     * @see Target#getName()
     */
    public String getName() {
        return target.getName();
    }

    
    /**
     * Returns the caught exception thrown by the target. 
     * @return the exception or <tt>null</tt>
     */
    public BuildException getBuildExeption() {
        return caughtBuildExeption;
    }
    
}
---8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<----
package org.apache.ant.parallelexecutor;

/**
 * TargetContainer can be in the here defined states.
 * The transistion between them is:<pre>
 *   start --> WAITING
 *   WAITING --> CHECKING
 *   CHECKING --> RUNNING, PREREQUISITE_FAILED
 *   RUNNING --> SUCCESSED, FAILED
 *   SUCCESSED --> end
 *   FAILED --> end
 *   PREREQUISITE_FAILED --> end
 * </pre>
 */
public enum TargetContainerStatus {
    /* Waiting for starting by the Executor. */
    WAITING,
    /* Checking prerequisites (depends, if, unless). */
    CHECKING,
    /* Target is currently running. */
    RUNNING,
    /* Target has finished without any error. */
    SUCCESSED,
    /* Target has thrown a BuildException. */
    FAILED,
    /* Target could not run because a prerequisite has not succeeded. */
    PREREQUISITE_FAILED;
    
    
    /**
     * Utility method for easier access to a set of states.
     * <pre>Finished = [SUCCESSED|FAILED|PREREQUISITE_FAILED]</pre>
     * @return computed state
     */
    public boolean hasFinished() {
        return this == SUCCESSED || this == FAILED || this == PREREQUISITE_FAILED;
    }
}
---8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<-------8-<----


 

> -----Urspr√ľngliche Nachricht-----
> Von: Klaus Malorny [mailto:Klaus.Malorny@knipp.de] 
> Gesendet: Dienstag, 18. Dezember 2007 10:10
> An: Ant Users List
> Betreff: extended parallelism
> 
> 
> Hi,
> 
> with the increased availability of multi-core systems, I am 
> wondering whether
> there are ways to improve the capabilities of Ant to 
> parallelize the build
> process. While it is possible to execute tasks in parallel 
> within a target, it
> seems to not be simple to execute multiple targets in 
> parallel. My vision is
> an extension to the "depends" attribute: if a pipe is used 
> instead of a comma,
> the target does not care about the order and the targets may 
> be executed in 
> parallel by Ant. I would also add parenthesis to group 
> things. For example, "(a 
> | (b, c)), d)" means, c must be executed after b, but both 
> may be executed in 
> parallel to a, while d must be executed only after completion 
> of a, b and c. Of 
> course, more constraints may occur in the dependent targets, 
> e.g. if c depends 
> on a, they can't be executed in parallel. Just an idea, feel 
> free to ignore it ;-)
> 
> Klaus
> 
> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: user-unsubscribe@ant.apache.org
> For additional commands, e-mail: user-help@ant.apache.org
> 
> 

---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@ant.apache.org
For additional commands, e-mail: user-help@ant.apache.org


Mime
View raw message