Subversion Repositories DevTools

package com.erggroup.buildtool.abt;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Properties;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.Target;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.taskdefs.Execute;
import org.apache.tools.ant.taskdefs.Redirector;

import com.erggroup.buildtool.abt.RemoteExecution.LogOutput;
import com.erggroup.buildtool.smtp.CreateUrls;
import com.erggroup.buildtool.smtp.Smtpsend;
import com.erggroup.buildtool.utilities.CommandBuilder;
import com.erggroup.buildtool.utilities.ElapseTime;
import com.erggroup.buildtool.utilities.utilities;

public class ABT extends Task
{
    private String ls = System.getProperty( "line.separator" );
    private String gbeMachtype = System.getenv("GBE_MACHTYPE");
    private String dpkgArchive = System.getenv("GBE_DPKG");
    private String hostname = System.getenv("GBE_HOSTNAME");
    private String gbeMachClass;          // Escrow Only

    // ant built in
    private Project project;
    private String basedir;

    // global
    private boolean master = false;
    private String gbebuildfilter;
    private String mailServer;
    private String mailSender;

    // project related
    private String rtagId;
    private String daemon;
    private String release;
    private String buildtoolVersion;
    private String family;
    private String oldExtension;
    private String newExtension;
    private ABTData abtData;

    // target related
    private Target target;
    private String packageVersionID;
    private String packageAlias;
    private String packageName;
    private String packageVersion;
    private String packageFullVersion;
    private String packageExtension;
    private String packageLoc;
    private String packageVcsTag;
    private String directChange;
    private String doesNotRequireSourceControlInteraction;
    private boolean isaRipple;
    private int testBuildInstruction = 0;
    private boolean generic = false;
    private boolean packageTarBall = false;
    private boolean useTestArchive = false;
    private String loc;
    private String unittests;
    private ArrayList<Depend> dependCollection = new ArrayList<Depend> ();
    private ArrayList<BuildInfo> buildInfoCollection = new ArrayList<BuildInfo>();
    private StringBuilder owners = new StringBuilder();;
    private Machine mMachine;
    private String logBase;

    // set up in execute
    private BuildStandards packageStandard = BuildStandards.NONE;           // Build standard for the package
    private BuildStandards buildStandard = BuildStandards.NONE;             // Build Standard for this instance
    private JavaVersions javaVersion = JavaVersions.NONE;
    private CompileTargets compileTarget = CompileTargets.NONE;
    // abt task result property
    private String propertyValue = "0";
    // output redirector
    private Redirector re;
    private File output;
    // exec thread
    private CommandBuilder command = new CommandBuilder();
    private String cwd;
    private Execute thread;
    private File wd;
    private String newVcsTag;
    private static final Logger mLogger = LoggerFactory.getLogger(ABT.class);
    private boolean fullyPublished = false;

    private static final char DEBUG = 0;
    private static final char INFO = 1;
    private static final char WARN = 2;
    private static final char ERROR = 3;
    private static final char FATAL = 4;
    
    //  Simple class to contain error information to be reported to the user
    private class ErrorInfo {
        String cmd;
        int result;
        String error;
        
        public ErrorInfo (String error)
        {
            this.error = error;
        }
        
        public ErrorInfo(String cmd, int result, String error)
        {
            this.cmd = cmd;
            this.result = result;
            this.error = error;
        }
        
        public String toString()
        {
            String rv = error;
            if (cmd != null)
            {
                rv += " : " + cmd;
                rv += " returned " + result;
            }
            return rv;
        }

        public String toHtml() {
            String rv = "Error : " + error;
            if (cmd != null)
            {
                rv += "<br>Command : " + cmd;
                rv += "<br>Returned: " + result;
            }
            return rv;
        }
    }

    /** Log a message to two locations
     *  1) log4j application wide log - using facility
     *  2) Ant Command output log - always
     * 
     * @param message   - Message to log
     * @param facility  - Log level
     */
    private void logMsg(String message, char facility)
    {
        if ( daemon != null )
        {
            if ( facility == DEBUG)
            {
                mLogger.debug(message);
            }
            else if ( facility == INFO)
            {
                mLogger.info(message);
            }
            else if ( facility == WARN)
            {
                mLogger.warn(message);
            }
            else if ( facility == ERROR)
            {
                mLogger.error(message);
            }
            else
            {
                mLogger.error(message);
            }
        }

        log( message );
    }

    /** Log two strings as 'info'
     *  The two strings are simply concatenated.
     * 
     * @param message1  - First part of the message
     * @param message2  - Second part of the message
     */
    private void logMsg(String message1, String message2)
    {
        logMsg( message1 + message2, INFO );
    }

    
    /** Add package location options
     *  Options depend on the build standard in use
     */
    private void addCommandLocate()
    {
        if ( packageStandard == BuildStandards.ANT )
        {
            command.add( "-locatefile=" + packageName + ".xml" );
        }
        else
        {
            command.add( "-locatepkg=" + packageLoc );
        }
    }
    

    /** Execute the command currently being assembled
     *  Logs command, exit code and execution duration
     * 
     * @param message           - Null: Don't report errors, otherwise text to prepend to the reported error
     * @return                  - Return code from the executed command
     * @throws Exception
     */
    private int runCommand( String message ) throws Exception
    {
        ElapseTime et = new ElapseTime();

        //  Need to create a cmd array with exactly the required number of arguments
        //  thread.setCommandline() cannot handle null items in the array
        //  Create a single logMessage while processing the array
        String logMessage = command.toString();
        logMsg( logMessage, WARN );
        
        thread.setCommandline(command.getArray());
        int retVal = thread.execute();
        
        //  Report error conditions 
        
        if ( retVal == Execute.INVALID )
        {
            reportErrorException( "Failure to execute thread, internal ABT problem", 256 );
        }
        else
        {
            logMsg( "command returned: "+ retVal  + ", elapsed time (S): " + et.toString(), WARN);

            if ( message != null && retVal != 0 )
            {
                ErrorInfo errorData = new ErrorInfo(command.toString() , retVal, message );
                reportErrorException( errorData, retVal );
            }
        }

        return retVal;
    }

    /**
     * Check that one marker file has been published into dpkg_archive 
     * Assumption - not called in Escrow Mode 
     * 
     * @param archive     - Archive to test
     * @param tag         - Marker suffix
     * 
     * @return true if the package is published 
     */
    private boolean checkPublished(String archive, String tag) throws Exception
    {
        if (archive == null)
        {
            reportErrorException( "GBE_DPKG environment variable not set", 256 );
        }

        String filename = "built." + tag;
        String destination = utilities.catDir(archive, packageName,  packageFullVersion); 

        //
        //  Have been having issues with NFS caching
        //  Current attempt is to flush NFS caches using freshFile(), but at the moment
        //  the code will still retry a number of time. This 'should' not be required if the
        //  flushing code is operational.
        //
        //  Insert log info if we have tried more than once
        //  Idea is to monitor first time failures and then delete this code
        //
        
        boolean found = false;
        int   maxwait = 2 * 4;
        for ( int ii = 0; ii < maxwait && !found; ii++)
        {
            found = utilities.freshFileExists(utilities.catDir(destination, filename));
            if ( !found )
            {
                // Debug - display files in the 
                try
                {
                    String files;
                    File folder = new File(destination);
                    File[] listOfFiles = folder.listFiles();
                    if ( listOfFiles == null )
                    {
                        logMsg( "published. Destination not a dir: " + destination, FATAL );
                    }
                    else
                    {
                        for (int i = 0; i < listOfFiles.length; i++) 
                        {
                            if (listOfFiles[i].isFile())
                            {
                                files = listOfFiles[i].getName();
                                logMsg("published. DirContent:" + files , WARN);
                            }
                        }
                    }
                }
                catch( Exception e )
                {
                    logMsg( "published. Display files caught Exception:" + e.getMessage(), FATAL );
                }


                Thread.sleep(500);
                logMsg("published. Wait for " + filename + ".(" + ii +")" , WARN);
            }
        }

        if ( found )
        {
            logMsg("published. " + filename + " exists", WARN);
        }
        else
        {
            logMsg("published. " + filename + " does not exist", WARN);
        }

        return found;
    }

    /** Print a line of text to a file
     * 
     * @param file  - FileWrite to print to
     * @param msg1  - Text to write. Will append line separator
     * @throws IOException
     */
    private void printPackage( FileWriter file, String msg1 ) throws IOException
    {
        file.write( msg1 + ls );
    }

    /** Copy a file
     * 
     * @param source    - Source File
     * @param dest      - Target File
     */
    private void copyFile( File source, File dest )
    {
        if ( source.exists() )
        {
            try
            {
                FileInputStream fis = new FileInputStream(source);
                FileOutputStream fos = new FileOutputStream(dest);
                byte[] buf = new byte[1024];
                int i = 0;
                while( (i=fis.read(buf)) != -1)
                {
                    fos.write(buf, 0, i);
                }
                fis.close();
                fos.close();
            }
            catch( Exception e )
            {
                logMsg( "copyFile caught Exception", ERROR );
            }
        }
    }

    /** Save the log file
     *  <br>Used on error build
     *  <br>Used on complete build
     * 
     * @return  True: Can log ( Daemon mode)
     */

    private boolean saveLogs()
    {
        if ( daemon == null )
        {
            // do nothing more in escrow
            return false;
        }

        // persist the failure log to GBE_LOG/hostname/rtagid/daemon
        String log = System.getenv("GBE_LOG");

        if ( log == null )
        {
            // cannot happen, GBE_LOG is checked in BuildDaemon at startup
            return false;
        }

        File destinationLog = new File( log );
        File destinationHostname = new File( destinationLog, hostname );
        File destinationRtagId = new File( destinationHostname, rtagId );
        // create directories
        new File( destinationRtagId, daemon).mkdirs();

        File destinationDaemon = new File( destinationRtagId, daemon );

        if ( destinationDaemon.exists() )
        {
            File buildsource = new File(rtagId + "build.xml");
            File buildcopy = new File( destinationDaemon, logBase + ".xml");
            copyFile( buildsource, buildcopy );

            File logsource = new File(rtagId + ".log");
            File logcopy = new File( destinationDaemon, logBase + ".log");
            copyFile( logsource, logcopy );

            logMsg("reportError persisting " + logcopy.getName(), WARN);
        }

        return true;
    }

    private void reportError(String error, int retVal)
    {
        reportError( new ErrorInfo(error), retVal);
    }
    private void reportError(ErrorInfo error, int retVal)
    {
        propertyValue = String.valueOf( retVal );

        logMsg( error.toString(), ABT.ERROR );
        logMsg( "reportError propertyValue = " + propertyValue + ", ", error.toString());

        //  Save the logs
        //      Will return false in situations were we can't proceed
        if ( ! saveLogs() )
        {
            return;
        }

        //  Only send emails if we have all the bits required
        //  Will not get this far unless we are in daemon mode
        if ( owners.length() > 0 && mailServer != null && mailSender != null )
        {

            String href = hostname + "/" + rtagId + "/" + daemon + "/" + logBase + ".log"; 
            String unc =  CreateUrls.generateUncBaseUrl() + "/" + href;

            String body = "Release: " + release + "<p>" +
                   "Package: " + packageName + "<p>" + 
                   "RM link: " + CreateUrls.generateRmUrl(rtagId,packageVersionID) + "<p>" +
                   "Build machine: " + hostname + "<p>" +
                   "Machine type: " + gbeMachtype + "<p>" +
                   error.toHtml() + "<p>" +
                   "Log file: <a href=\"" + unc + "\">" + href + "</a> ... URL expires in 5 working days time<p><hr>";

            try
            {
                Smtpsend.send( mailServer,          // mailServer
                               mailSender,          // source
                               owners.toString(),   // target
                               null,                // cc
                               null,                // bcc
                               "BUILD FAILURE on package " + packageName, // subject
                               body,                // body
                               null                 // attachment
                        );
            }
            catch( Exception e )
            {
                // not fatal
                logMsg( "reportError caught Exception", WARN);
            }
        }

    }

    private void reportErrorException(String error, int retVal) throws Exception
    {
        reportError( error, retVal );
        throw new Exception( hostname );
    }
    
    private void reportErrorException(ErrorInfo error, int retVal) throws Exception
    {
        reportError( error, retVal );
        throw new Exception( hostname );
    }
    
    /**
     * Init the metrics generation process
     * 
     * @exception Exception
     */
    private void metricsBegin() throws Exception
    {
        if ( master )
        {
            // gather metrics
            // jats -locatefile=.jats.packageroot etool jats_metrics -mode=init -rootdir=.
            // -locate=.jats.packageroot causes jats to cd to the folder containing the file .jats.packageroot
            // this file is created in the -path folder by jats release -extract
            // DEVI 51370 it is essential for metrics branch calculation the metrics utility is started within a static view
            //
            command.init("jats" );
            command.add( "-locatefile=.jats.packageroot" );
            command.add( "etool", "jats_metrics" );
            command.add( "-mode=init" );
            command.add( "-rootdir=." );

            runCommand( "Failure to collect metrics" );
        }     
    }
    
    /**
     * Complete the metrics generation process
     * 
     * @exception Exception
     */
    private void metricsEnd() throws Exception
    {
        if ( master )
        {
            File metrics = new File(utilities.catDir(basedir, rtagId + "abtmetrics.txt"));
            if ( metrics.exists() )
            {
                metrics.delete();
            }

            // gather metrics
            // jats -locatefile=.jats.packageroot etool jats_metrics -mode=finish -out=abtmetrics.txt
            command.init("jats" );
            command.add( "-locatefile=.jats.packageroot" );
            command.add( "etool", "jats_metrics" );
            command.add( "-mode=finish" );
            command.add( "-out=" + metrics.getAbsolutePath() );

            runCommand( "Failure to collect metrics" );
        }    
    }
    
    /**
     * Generate the build dependency data file.
     * 
     * @exception Exception
     */
    private void genbuild() throws Exception
    {

        //
        //  Create build files with updated dependency information
        //  In escrow mode : No new dependency information
        //  In daemon mode :
        //      Ripple     : New dependency information
        //      Verify     : Existing dependency information
        //
        //  In all modes  : ANT builds need an auto.xml
        //
        //  In order to create auto.xml/auto.pl with package dependencies need to
        //  create file with dependency data (auto.cfg) only because its
        //  needed by "jats_rewrite"
        //
        File cfg = null;
        boolean  createFile = false;
        if ( daemon != null )
        {
            createFile = true;

            // always generate auto.cfg
            cfg = new File( utilities.catDir(cwd, "auto.cfg"));
            cfg.createNewFile();

            if ( !cfg.exists() || !cfg.canWrite() )
            {
                reportErrorException( "Failed to open auto.cfg", 263 );
            }

            try
            {
                FileWriter cfgfw = new FileWriter(cfg);
                printPackage( cfgfw, packageName + " " + packageFullVersion );

                // provide releasemanager.projectname=<name[0]> and releasemanager.releasename=<name[1]>
                String[] name = release.split( " > ", 2 );

                if ( name.length == 2 )
                {
                    printPackage( cfgfw, "releasemanager.projectname=" + name[0] );
                    printPackage( cfgfw, "releasemanager.releasename=" + name[1] );
                }

                for (Iterator<Depend> it=dependCollection.iterator(); it.hasNext(); )
                {
                    Depend depend = it.next();
                    printPackage( cfgfw, depend.getPackage_Alias() );
                }

                cfgfw.close();
            }
            catch( IOException e )
            {
                reportErrorException( "Failure to write to auto.cfg", 263 );
            }
        }
        else
            if ( packageStandard == BuildStandards.ANT )
            {
                // Have not created an auto.xml with updated dependencies
                // Ant build still would like an auto.xml file
                //
                // ie packages with a line <import file="auto.xml" optional="true"/> are fine
                //    but in escrow and rebuild we still need an auto.xml to be kind
                //
                createFile = true;
            }

        //
        //  Create the auto.pl/auto.xml file if required
        //
        if ( createFile )
        {
            command.init("jats");
            addCommandLocate();
            command.add( "etool", "jats_rewrite" );
            
            if ( packageStandard == BuildStandards.ANT )
            {
                command.add( "-infile=" + packageName + "depends.xml" );
                command.add( "-outfile=auto.xml" );
            }
            else
            {
                command.add( "-infile=build.pl" );
                command.add( "-outfile=auto.pl" );
            }

            if ( cfg != null && daemon != null)
            {
                command.add( "-config=" + cfg.getAbsolutePath() );

                //  Validate the Release Manager versions against those in the build files
                //  Only do this in daemon mode for:
                //      We are rebuilding the package
                //          This is detected as: Not have VCS interaction AND is not a test Build
                boolean isaRebuild = doesNotRequireSourceControlInteraction != null && testBuildInstruction == 0;
                command.add(isaRebuild, "-validate");
                
                //  Ensure that we don't have excessive packages in the Metadata
                //  Only do this in daemon mode, for 'new' or 'test' builds
                //  Don't do for:
                //      Escrow builds - they can't fix it
                //      Ripple builds - it will break a lot of stuff
                //      Rebuilds      - it will break stuff
                // Do for:
                //      New Versions  - User can fix it
                //      Test Builds   - User can fix it
                boolean isaTestBuild = doesNotRequireSourceControlInteraction != null && testBuildInstruction > 0;
                boolean isaNewVersion = !isaRipple && !isaRebuild && !isaTestBuild;
                command.add( isaNewVersion || isaTestBuild, "-errors");
                
            }
            else
            {
                command.add( "-noconfig");
            }

            if ( oldExtension != null )
            {
                command.add( "-oldproject", oldExtension );
                command.add( "-newproject", newExtension );
            }

            runCommand( "Failure to generate build dependency information. Potentially caused by a mismatch between build files and Release Manager database" );
        }
    }

    /**
     * Build the package
     * 
     * @exception Exception
     */
    private void jatsBuildPackage() throws Exception
    {
        //--------------------------------------------------------------------------
        //  Wait for package replication and caching
        //      If there is a GBE_DPKG_REPLICA, then there may be a delay between
        //      a package being built and the package being available on the replica
        //      We need to wait for the required packages to replicate
        //
        //      If there is a GBE_DPKG_CACHE, then we need to cache the package
        //      into the local store - mostly for windows as it seriously speeds
        //      up the builds.
        
        Boolean hasReplica = System.getenv("GBE_DPKG_REPLICA") != null;
        Boolean hasCache   = System.getenv("GBE_DPKG_CACHE") != null;
        
        if (daemon != null && (hasReplica || hasCache)  && !dependCollection.isEmpty())
        {
            /*
             * Limit the command line length provided to the following utility
             * Very very long dependency lists may over fill the command line
             */
            Iterator<Depend> it=dependCollection.iterator();
            while( it.hasNext() )
            {
               
                command.init( "jats", "etool", "cache_dpkg");
                command.add( hasReplica, "-wait" );
                
                // ANT builds require the package to be cached
                // JATS does not - if its needed it will be done later
                command.add( buildStandard != BuildStandards.ANT , "-nocache");
            
                // Add PackageName/PackageVersion
                for (int maxArgs = 0; maxArgs < 10 && it.hasNext(); maxArgs++ )
                {
                    Depend depend = it.next();
                    command.add(depend.getPackageNameVersion());
                }
                
                // Run the command - don't care if it fails
                runCommand( null );
            }
        }
        
        //--------------------------------------------------------------------------
        //  Build Phase
        //  Create build environment

        command.init( "jats" );
        command.add( gbebuildfilter != null ,"-buildfilter", gbebuildfilter);
        addCommandLocate();

        if ( buildStandard == BuildStandards.ANT )
        {
            command.add( "abt" );
            command.add( "-java=" + javaVersion.getJavaVersion() );
            command.add( "-buildfile=" + packageName + ".xml" );
            command.add( "build" );
        }
        else
        {
            command.add( "build", "--package" );

            // Daemon: Sanity check the building of Generic packages
            // Escrow: Allow this error - for backward compatibility
            if ( daemon != null )
            {
                if (generic) 
                    command.add( "-generic" );
                else
                    command.add( "-nogeneric" );
            }
            
            // Cache packages into a machine-local archive
            //  Will only do this if the package is really built
            //
            command.add(hasCache, "--cache" );
        }

        runCommand( "Failure to set up sandbox" );

        //--------------------------------------------------------------------------
        //  Make Package phase
        //  Create the package (make)

        command.init("jats" );
        command.add( gbebuildfilter != null, "-buildfilter", gbebuildfilter);
        addCommandLocate();

        if ( buildStandard == BuildStandards.ANT )
        {
            command.add( "abt" );
            command.add( "-java=" + javaVersion.getJavaVersion() );
            command.add( "-buildfile=" + packageName + ".xml" );
            command.add( "make_package" );
        }
        else
        {
            command.add( "make", "NODEPEND=1" );

            if ( compileTarget == CompileTargets.PROD )
            {
                command.add( "prod", "package_prod" );
            }
            else if ( compileTarget == CompileTargets.DEBUG )
            {
                command.add( "debug", "package_debug" );
            }
            else
            {
                command.add( "all" );
            }
        }

        runCommand( "Failure to generate derived files" );


        //--------------------------------------------------------------------------
        //  Unit tests phase
        //  Run autonomous unit tests

        if ( unittests != null )
        {
            command.init("jats" );
            command.add( gbebuildfilter != null, "-buildfilter", gbebuildfilter);
            addCommandLocate();

            if ( buildStandard == BuildStandards.ANT )
            {
                command.add( "abt" );
                command.add( "-java=" + javaVersion.getJavaVersion() );
                command.add( "-buildfile=" + packageName + ".xml" );
                command.add( "run_tests" );
            }
            else
            {
                command.add( "make" );
                if ( compileTarget == CompileTargets.PROD )
                {
                    command.add( "run_unit_tests_prod" );
                }
                else if ( compileTarget == CompileTargets.DEBUG )
                {
                    command.add( "run_unit_tests_debug" );
                }
                else
                {
                    command.add( "run_unit_tests" );
                }
            }

            runCommand( "Failure to run unit tests" );
        }

        //--------------------------------------------------------------------------
        //  create_dpkg phase
        //  Publish the package to dpkg_archive
        //

        command.init("jats" );
        addCommandLocate();
        command.add( "create_dpkg", "-m" );
        command.add( packageTarBall, "-tarmode");
        command.add( generic, "-generic" );
        command.add( "-pname", packageName );
        command.add( "-pversion", packageFullVersion );

        runCommand( "Failure to publish to archive" );

        insertEscrowMarker();
    }
    
    /** Skip Build an ANT Package 
     *  Only used on ANT based builds in which it has been determined that another machine
     *  is performing the build.
     *  
     *  Insert required marker files to indicate that the build has occurred
     *  Perform this using JATS commands to ensure correct creation of the marker files
     *  Basically, a dummy publish phase
     *  
     *  Note: Skip Builds cannot be generic
     *  
     */
    private void jatsSkipPackage() throws Exception
    {
        command.init( "jats" );
        command.add( "-here" );
        command.add( "create_dpkg", "-m" );
        command.add( packageTarBall, "-tarmode");
        command.add( "-noBuild" );
        command.add( "-pname", packageName );
        command.add( "-pversion", packageFullVersion );

        runCommand( "Failure to (skip)publish to archive" );
    }

    /**   Check if the package has been Published on all the required machines
     *    If its a 'generic' package then a tag of built.generic will have been placed in the package
     *    Otherwise each build machine will place a built.<MachName> file in the package when
     *    its build is completed successfully.
     *  
     *    If we don't have a complete set of 'built' files, then the package failed to build
     *    on one or more platforms. The assumption is that the build failure has been reported
     *    by the build daemon.
     *  
     *    All we know is which MachTypes we build for. This needs to be expanded into
     *    a list of MachineNames. Thus a build-set with multiple machines of the same MachType
     *    can indicate build failures.
     */
    private void puplishPackage() throws Exception
    {  
        //  If the package fragments are being delivered as tarballs then they needs to be reassembled
        if (packageTarBall)
        {
            fullyPublished = assemblePackage();
        }
        
        if (fullyPublished)
        {
            if ( generic )
            {
                //  Generic - only one marker file
                fullyPublished = checkPublished( dpkgArchive, "generic" );
            }
            else
            {
                //  Multi machine package
                //  Scan required target machines artifacts
                //  Expand from MachType to one or more MachName's
                scanBuilds:
                    for (Iterator<BuildInfo> ip=buildInfoCollection.iterator(); ip.hasNext(); )
                    {
                        BuildInfo buildInfo = ip.next();
                        for (Iterator<Machine> it=abtData.machineCollection.iterator(); it.hasNext(); )
                        {
                            Machine ma = it.next();
                            if (ma.getMachtype().compareTo(buildInfo.getMachtype()) == 0 )
                            {
                                fullyPublished = checkPublished( dpkgArchive, ma.getName() );
                                if ( !fullyPublished )
                                {
                                    //  Break out of both loops
                                    //  Once we have seen an error we need to retain it.
                                    break scanBuilds;
                                }
                            }
                        }
                    }
            }
        }

        //  Detect problem with zero length descpkg file
        //      May be caused by write clash from multiple machines
        //      Don't know
        //
        //  Not forcing NFS flush as this will have been done in the previous step
        //
        if ( fullyPublished )
        {
            String destination = utilities.catDir(dpkgArchive, packageName, packageFullVersion, "descpkg");
            File flag = new File( destination);

            //  File Must
            //      Exist
            //      Be a File
            //      Have a length. Problem appears to be zero length file
            //
            boolean found = false;
            logMsg( "published. Descpkg Sanity Test: " + flag.getPath(), DEBUG );
            if (!flag.exists() ) {
                reportErrorException( "No descpkg file. Package may not build with current build filters", 261 );

            } else if ( !flag.isFile()) {
                reportErrorException( "Descpkg Sanity Test: Is not a file", 261 );

            } else if ( flag.length() <= 0 ) {
                reportErrorException( "Descpkg Sanity Test: Zero length File", 261 );

            } else {
                logMsg( "published. Descpkg Sanity Test: Passed", INFO );
                found = true;
            }

            if (!found) {
                fullyPublished = false;
            }
        }

        if ( fullyPublished && daemon != null )
        {
            if ( doesNotRequireSourceControlInteraction == null )
            {
                //
                //  Save modified build files
                //  Label the resultant files
                //  Create a command of the form:
                //      jats etool jats_vcssave_build
                //          -infile     auto.xml/auto.pl
                //          -outfile    xxxdepends.xml/build.pl
                //          -pname      package_name
                //          -pversion   package_version
                //          -infofile   path_to_info_file
                //          -wiplabel   Existing WIP label (optional)
                //          -baselabel  View label

                logMsg("publish save build files", WARN);

                File saveinfo = new File(utilities.catDir(basedir, rtagId + "abtinfo.txt"));
                if ( saveinfo.exists() )
                {
                    saveinfo.delete();
                }

                command.init("jats" );
                addCommandLocate();
                command.add( "etool", "jats_vcssave_build" );

                if ( packageStandard == BuildStandards.ANT )
                {
                    command.add( "-infile", "auto.xml" );
                    command.add( "-outfile", packageName + "depends.xml" );
                }
                else
                {
                    command.add( "-infile", "auto.pl" );
                    command.add( "-outfile", "build.pl" );
                }

                command.add( "-pname", packageName );
                command.add( "-pversion", packageFullVersion );
                command.add( directChange != null, "-isawip" );
                command.add( "-infofile", saveinfo.getAbsolutePath() );
                command.add( "-baselabel", packageVcsTag );

                if ( oldExtension != null && newExtension != null && packageExtension.compareTo( newExtension ) == 0 )
                {
                    // only branch build.pl where its extension is being migrated
                    command.add( "-branch", "project_migration_branch", "-newbranch" );
                }

                runCommand( "Failure to save build file changes into source control" );

                //
                //  The command creates an info file(saveinfo) to pass information back
                //  to the caller (me). This is a 'properties' file
                //  Need to extract the following information
                //    newVcsTag - which is held as VCS.tag
                //
                logMsg("publish read info file", INFO);

                Properties properties = new Properties();
                try
                {
                    properties.load(new FileInputStream(saveinfo));
                }
                catch (IOException e)
                {
                    reportErrorException( "Failed to open save_build info file", 262 );
                }

                newVcsTag = properties.getProperty("VCS.tag" );
                if ( newVcsTag == null )
                {
                    reportErrorException( "Save_build info. Missing property:VCS.tag", 262 );
                }
                logMsg("publish read info file. Label:" + newVcsTag , INFO);
            }
        }
    }

    /** Re-assemble package fragments into a package within the target archive
     *  Used only in non-escrow mode
     *  
     *  This method will invoke a command on the target package server
     *  This is done to overcome several issues:
     *  1) Atomic merging of package fragments
     *  2) Very long times seen in transferring a package with a large number of files
     *     over a non-local network. It may take a second to create a file. For a 3000 file
     *     package we were seeing times of about an hour simply to transfer the files from AWS(Sydney)
     *     to the package server in Perth.
     *     
     *  The down side is that we need to set up an SSH session to the package server
     *  We need:
     *  1) Network name of the machine
     *  2) User on the machine to do the work
     *  3) Credentials for that user
     *  
     *   The existence of these items will have been performed by the daemon at startup
     *   This simplifies the process of reporting errors
     * 
     * @return  true - reassemble occurred correctly
     * @throws Exception 
     */
    private boolean assemblePackage() throws Exception 
    {
        if ( daemon == null )
        {
            // Not to be used in escrow
            return false;
        }
        /*
         *  Setup the credentials
         *  The format 'should' have been checked as the daemon started
         *  The link should have been authenticated before the build was started,
         *  but it may have gone down since.
         */
        LogOutput logger = new LogOutput()
        {
            @Override
            public void data(String message) {
                logMsg( message, INFO );
            }

            @Override
            public void info(String message) {
                logMsg( message, INFO );
            }
            
            @Override
            public void fatal(String message) {
                logMsg( message, FATAL);
            }
        };
        
        ElapseTime et = new ElapseTime();
        RemoteExecution mRemoteExecution = new RemoteExecution(logger);
        
        /*
         * Generate the command string
         * Its being executed over an ssh session so we need to:
         *  1) Set up the required environment
         *  2) Set the location of GBE_DPKG - more for testing
         */
        command.init();
        command.add("sudo","-n", "-u", "buildadm", "/home/releasem/sbin/jatsTool", "assemble_dpkg");
        command.add(useTestArchive, "-testArchive");
        command.add("'-pname=" + packageName + "'");
        command.add("'-pversion=" + packageFullVersion + "'");
       
        String cmd = command.toString();
        logMsg( "assemblePackage. Command:" + cmd, WARN );
        int retCode = mRemoteExecution.execute(cmd);
        logMsg( "command returned: "+ retCode + ", elapsed time (S): " + et.toString(), WARN);
        if (retCode != 0)
        {
            reportErrorException( "Package reassembly error", 267 );
        }
        return 0 == retCode;
    }

    /** determineBuildStandard
     *  Process the current target and set up
     *      buildStandard
     *          compileTarget
     *          javaVersion
     *          
     *      packageStandard
     *          The package standard is used to determine the mode used to process the auto.cfg file
     *          
     */
    private void determineBuildStandard() throws Exception 
    {
        // Scan the platform collection looking for an entry that matches the current machine
        //
        // Just wanting to set up
        //    buildStandard and one of
        //        compileTarget
        //    OR  javaVersion

        for (Iterator<BuildInfo> it = buildInfoCollection.iterator(); it.hasNext();)
        {
            BuildInfo buildInfo = it.next();
            String method = buildInfo.getMethod();
            String arg = buildInfo.getArg();            
            
            //  Capture the first non-null build standard as the packageStandard
            if ( packageStandard == BuildStandards.NONE)
            {
                if (method.equals("jats"))
                {
                    packageStandard = BuildStandards.JATS;
                }
                else if (method.equals("ant"))
                {
                    packageStandard = BuildStandards.ANT;
                }
            }
            
            //  Capture the build standard for the current machine type
            if (gbeMachtype.compareTo(buildInfo.getMachtype()) == 0)
            {
                if (method.equals("jats"))
                {
                    buildStandard = BuildStandards.JATS;
                    if (arg.compareTo("production") == 0)
                    {
                        compileTarget = CompileTargets.PROD;

                    }
                    else if (arg.compareTo("debug") == 0)
                    {
                        compileTarget = CompileTargets.DEBUG;

                    }
                    else if (arg.compareTo("all") == 0)
                    {
                        compileTarget = CompileTargets.ALL;

                    }
                    else
                    {
                        reportErrorException("Failure, unsupported Jats build environment " + arg + ", internal ABT problem", 256);
                    }   
                }
                else if (method.equals("ant"))
                {
                    buildStandard = BuildStandards.ANT;

                    //    Handle version of 'none' and of the form 'n.n'
                    //    Anything else is an error

                    javaVersion = new JavaVersions(arg);
                    if ( javaVersion.isBad())
                    {
                        reportErrorException("Failure, unsupported Ant build environment " + arg + ", internal ABT problem", 256);
                    }
                }

                else if (method.equals("none"))
                {
                    buildStandard = BuildStandards.NONE;
                }
                else
                {
                    reportErrorException("Failure, unsupported build method " + method + ", internal ABT problem", 256);    
                }
            }
        }

        //
        //    Log the build method and arguments
        //
        logMsg("execute packageStandard: ", packageStandard.getBuildStandard());
        logMsg("execute buildStandard: ", buildStandard.getBuildStandard());
        if (buildStandard == BuildStandards.ANT )
        {
            logMsg("execute javaVersion: ", javaVersion.getJavaVersion());    
        }
        else if (buildStandard == BuildStandards.JATS )
        {
            logMsg("execute compileTarget: ", compileTarget.getCompileTarget());    
        }


        //  Daemon Mode Only
        //  If we have an ANT Build then we may have an additional problem.
        //  If there is more than one machine of the same 'Machine Class' in the buildset, then
        //  we need to limit the build to just one of them.
        //  In Escrow this is not an issue as the builds are done serially
        //
        //  The rule is:
        //    If there is only one machine in the class - then use it
        //    If there are multiple machines of the same class - then use the first one that has JAVA in its build filter
        //    If none have JAVA in the build filter, then use the first one in the list
        // Supplementary Rules:
        //    If a machine does no have a buildfilter, then assume that it can build everything
        // 
        // Rule Simplification:
        //    If this machine is building ANT, then exclude ourself from the build if
        //    another machine satisfies the criteria. ie: another machine will determine that
        //    it will build the ANT component.
        //
        if (buildStandard == BuildStandards.ANT && daemon != null)
        {
            int classCount = 0;
            Machine firstWithJava = null;
            Machine firstInMachClass = null;

            for (Iterator<Machine> it = abtData.machineCollection.iterator(); it.hasNext();)
            {
                Machine ma = it.next();
                if (ma.getMachclass().equals(mMachine.getMachclass()) )
                {
                    classCount++;
                    
                    if (firstInMachClass == null)
                        firstInMachClass = ma;

                    if (firstWithJava == null)
                    {
                        String buildFilter = ma.getBuildfilter();
                        if (buildFilter != null)
                        {
                            buildFilter = "," + buildFilter.replace(" ", ",") + ",";
                        }

                        if (buildFilter == null || buildFilter.contains(",JAVA,"))
                        {
                            firstWithJava = ma;
                        }
                    }
                }
            }
            
            // If none have 'JAVA', then used the first machine in the nominated class
            if (firstWithJava == null)
            {
                firstWithJava = firstInMachClass;
            }
                

            //
            //  Apply the rule described above.
            //  Really only need to disqualify ourselves iff there is another machine that will do the build
            //
            if ((classCount > 1) && (firstWithJava != null) && (mMachine != firstWithJava))
            {
                javaVersion = JavaVersions.SKIP;
                logMsg("execute determineBuildStandard: Exclude myself from ANT Build", INFO);
            }
        }
    }


    /** execute
     *  The Ant Task entry point. This is invoked to perform the targets action
     *      Display the start and end time of the action
     */
    @Override
    public void execute()
    {
        ElapseTime et = new ElapseTime();

        //  Setup enough information to enable logging
        project = getProject();
        target = getOwningTarget();
        daemon = project.getProperty("abt_daemon");
        rtagId = project.getProperty("abt_rtag_id");
        release = project.getProperty("abt_release");
        
        logMsg( "execute start: " + ElapseTime.getTimeStamp(), WARN );
        if ( daemon != null ) {
            logMsg("execute daemon: yes", INFO);
        }

        try {
            executeAbtTask();    
        }
        finally         {
            logMsg( "execute complete: " + ElapseTime.getTimeStamp(), INFO );
            logMsg( "execute elapsed time (S): " + et.toString(), WARN);
        }
    }

    /** executeAbtTask
     *  Body of the ABT execution task
     *  Factored out so that a time stamp can be added to the log
     */
    private void executeAbtTask()
    {
        boolean errorReported = false;
        boolean benign = false;
        boolean skipped = false;

        try
        {
            // required by the Execute API on non win32 platforms, don't ask because I don't know
            project.setProperty("ant.home", System.getenv("ANT_HOME"));
            
            // Setup the base name of the output log file
            // Need to do before we report errors
            if ( daemon != null )
            {
                //  Use the packages name and version
                packageName = project.getProperty("abt_package_name");
                logMsg("execute packageName: ", packageName);
                
                packageFullVersion = project.getProperty("abt_package_version");
                logMsg("execute packageFullVersion: ", packageFullVersion);
                
                logBase = packageName + "_" +  packageFullVersion;
            }
            else
            {
                // Use the current target - which will be the packages name and version
                logBase = target.getName();
            }

            // Check the current hostname
            //    Needed early as its used in error reporting
            //    JATS should ensure that this value is set
            // Daemon Mode:
            //    When sending emails
            //    Identify the target machine parameters
            // Escrow Mode:
            //    Determine escrow build components
            //    
            if (hostname == null) {
                hostname = "unknown";
                reportErrorException( "GBE_HOSTNAME environment variable not set", 256 );
            }

            // early target related
            packageAlias = target.getName();
            packageVcsTag = project.getProperty(packageAlias + "packagevcstag");
            logMsg("execute packageVcsTag: ", packageVcsTag);

            if ( packageVcsTag == null )
            {
                // this abt task has no properties, by design
                // future use to estimate build completion time
                logMsg("execute packageAlias: " + packageAlias, INFO);
                return;
            }

            logMsg("execute packageAlias: " + packageAlias, WARN);

            // ant built in
            basedir = project.getProperty("basedir");
            logMsg("execute basedir: ", basedir);

            // global

            if ( gbeMachtype == null ) {
                reportErrorException( "GBE_MACHTYPE environment variable not set", 256 );
            }

            if (dpkgArchive == null) {
                reportErrorException( "GBE_DPKG environment variable not set", 256 );
            }

            //  Locate ABTDATA
            //  This will be referenced by a known ID
            abtData = (ABTData) project.getReference("global-abt-data");
            if (abtData == null) {
                reportErrorException( "ABTData Task not found", 256 );
            }
            
            if ( daemon != null )
            {

                mailServer = project.getProperty("abt_mail_server");
                if ( mailServer == null )
                {
                    reportErrorException( "Missing ANT property: abt_mail_server", 256 );
                }
                logMsg("execute mailServer: ", mailServer);

                mailSender = project.getProperty( "abt_mail_sender" );
                if ( mailSender == null )
                {
                    reportErrorException( "Missing ANT property: abt_mail_sender", 256 );
                }
                logMsg("execute mailSender: ", mailSender);

                for (Iterator<Owner> it=abtData.ownerCollection.iterator(); it.hasNext(); )
                {
                    Owner owner = it.next();

                    if ( owners.length() > 0 )
                    {
                        owners.append(",");
                    }
                    owners.append(owner.getEmail());
                    logMsg("execute owner: ", owner.getEmail());
                }
                logMsg("execute owners: ", owners.toString());
            }

            if ( daemon == null)
            {
                // Locate any entry for my machType and use it to convert into machClass
                // In escrow the hostName cannot be used.
                //    
                for (Iterator<Machine> it=abtData.machineCollection.iterator(); it.hasNext(); )
                {
                    Machine ma = it.next();
                    if (ma.getMachtype().compareTo(gbeMachtype) == 0 )
                    {
                        gbeMachClass = ma.getMachclass();
                        break;
                    }
                }

                if (gbeMachClass == null)
                {
                    reportErrorException( "No MachClass mapping for " + gbeMachtype, 256 );
                }
                logMsg("execute gbeMachClass: ", gbeMachClass);
            }

            // project related
            // Some data has already been collected. Just log it.
            logMsg("execute rtagId: ", rtagId);
            logMsg("execute release: ", release);

            packageVersionID = project.getProperty("abt_package_version_id");
            if (packageVersionID != null)
            {
                logMsg("execute packageVersionID: ", packageVersionID); 
            }

            buildtoolVersion = project.getProperty("abt_buildtool_version");
            logMsg("execute buildtoolVersion: ", buildtoolVersion);

            family = project.getProperty("abt_family");
            logMsg("execute family: ", family);

            oldExtension = project.getProperty("abt_old_extension");
            logMsg("execute oldExtension: ", oldExtension);

            newExtension = project.getProperty("abt_new_extension");
            logMsg("execute newExtension: ", newExtension);

            {
                 String prop = project.getProperty("abt_test_build_instruction");
                 logMsg("execute testBuildInstruction: ", prop);
                 testBuildInstruction = 0;
                 if ( prop != null)
                 {
                     testBuildInstruction = Integer.parseInt(prop);
                 }
            }
            
            {
                String prop = project.getProperty("abt_is_ripple");
                logMsg("execute isRipple: ", prop);
                isaRipple = ( prop != null && Integer.parseInt(prop) > 0);
            }


            if ( project.getProperty("abt_packagetarball") != null ) {
                packageTarBall = true;
                logMsg("execute packageTarBall: yes", INFO);  
            }

            if ( project.getProperty("abt_usetestarchive") != null ) {
                useTestArchive = true;
                logMsg("execute useTestArchive: yes", INFO);  
            }

            // Locate Machine Information for this machine
            //    It contains the build filter for the current machine
            //    Only present in daemon builds
            //
            for (Iterator<Machine> it=abtData.machineCollection.iterator(); it.hasNext(); )
            {
                Machine ma = it.next();
                if (ma.getName().compareTo(hostname) == 0 )
                {
                    mMachine = ma;
                    logMsg("execute machine: ", mMachine.getName() + ", Type:" + mMachine.getMachtype() + ", Class:" + mMachine.getMachclass() + ", Master:" + mMachine.getMaster() ); 
                    break;
                }
            }

            if ( (daemon != null) && (mMachine == null) ) {
                reportErrorException( "Cannot find Machine entry for: " + hostname, 256 );
            }

            //  Setup machine specific information
            //    Build Filter
            //    Master Mode
            if (mMachine != null) 
            {
                if (mMachine.getMaster() != null)
                {
                    master = true;
                    logMsg("execute master: ", "true");
                }

                gbebuildfilter = mMachine.getBuildfilter(); 
                if ( gbebuildfilter != null )
                {
                    //
                    // Ensure the Execute task passes gbebuildfilter as one parameter...
                    // Replace spaces with commas
                    //
                    gbebuildfilter = gbebuildfilter.replace( " ", ",");
                    logMsg("execute gbebuildfilter: ", gbebuildfilter);
                }
            }
            
            // Extract the build standard information from the abt element
            //  Sets up buildStandard and one of
            //            compileTarget
            //        OR  javaVersion

            determineBuildStandard();
            if ( buildStandard == BuildStandards.NONE)
            {
                //  Flag a benign build is good
                //  A benign is a no-operation. The build is not required on the current machine
                //
                benign = true;
            }

            if ( buildStandard == BuildStandards.ANT && javaVersion == JavaVersions.SKIP )
            {
                //  Flag a benign build - we are not building
                //  Flag a skipped build - we need to place a marker in dpkg_archive
                // 
                //  JavaVersions.SKIP is used when a ANT build could be done on multiple machines
                //  It must only be done on one such machine
                // 
                benign = true;
                skipped = true;
            }

            // Do we really need to extract and work with the source
            //      Escrow Mode : yes (Not done here)
            //      Daemon Mode :
            //          If Master : Yes. We need to generate metrics and update the build file
            //          Not Master: May be
            //                      If a benign build : No
            boolean extractSource = true;
            if ( daemon != null && !master && benign ) {
                extractSource = false;
            }
            logMsg("execute extractSource: ", Boolean.toString(extractSource));
            logMsg("execute benign: ", Boolean.toString(benign));
            logMsg("execute skipped: ", Boolean.toString(skipped));

            // redirect all thread (therefore jats) output to the output file in the basedir
            re = new Redirector(this);
            if ( daemon != null )
            {
                // thread logging is done to <rtagId>thread.log and is continually re-used every build
                // full thread and non thread (ie abt logging) is done to the DefaultLogger set up by the build daemons, <rtagId>.log
                // this file is copied on a build failure to cwd (if it exists) and is continually re-used every build

                // capture full logging
                re.setAlwaysLog(true);
            }
            else
            {
                output = new File( logBase + ".log" );
            }

            if (output != null)
            {
                re.setOutput( output );
            }
            thread = new Execute(re.createHandler());
            thread.setAntRun( project );
            thread.setVMLauncher(false);

            // must set up package name and owners before reportError
            packageName = project.getProperty(packageAlias + "packagename");
            logMsg("execute packageName: ", packageName);

            //-----------------------------------------------------------------------
            //    AbtTestPath
            //
            if ( packageAlias.compareTo("AbtTestPath") == 0 )
            {

                // jats -label check pkgVcsTag
                command.init("jats", "label","-check" );
                command.add( packageVcsTag );
                //command.add( "-verbose=2" );

                runCommand( "Failure to validate label in source control" );
                
                return;
            }
            
            //-----------------------------------------------------------------------
            //    AbtSetUp
            //
            if ( packageAlias.compareTo("AbtSetUp") == 0 )
            {
                // create rtag_id directory from scratch
                File rId = new File( rtagId );
                deleteDirectory(rId);
                if (rId.exists())
                {
                    reportErrorException( "Build directory not empty", 264 );
                }
                rId.mkdirs();

                if (extractSource)
                {
                    // jats jats_vcsrelease -extract -label=pkgVcsTag -path=src_path -root=rtag_id/timestamp -tag=timestamp -noprefix
                    command.init("jats", "jats_vcsrelease","-extract" );
                    command.add( "-label=" + packageVcsTag );
                    command.add( "-root=" + rtagId );
                    command.add( "-tag=" + daemon );
                    command.add( "-noprefix" );
                    command.add(!master, "-extractfiles");
                    //command.add( "-verbose=2" );
    
                    runCommand( "Failure to extract source code from source control" );
                }
                return;
            }

            //-----------------------------------------------------------------------
            //    AbtTearDown
            //
            if ( packageAlias.compareTo("AbtTearDown") == 0 )
            {
                // tear the build view down, regardless of build error
                wd = new File( basedir );

                // only now change the thread working directory back
                thread.setWorkingDirectory( wd );

                File buildDirectory = new File(wd + getProject().getProperty("abt_package_location"));

                if ( buildDirectory.exists() )
                {
                    // jats jats_vcsrelease -label pkgTag -root=rtag_id/timestamp -tag=timestamp -noprefix -delete=2
                    command.init("jats", "jats_vcsrelease" );
                    command.add( "-label=" + packageVcsTag );
                    command.add( "-root=" + rtagId );
                    command.add( "-tag=" + daemon );
                    command.add( "-noprefix" );
                    command.add( "-delete=2" );
                    //command.add( "-verbose=2" );

                    runCommand( "Failure to remove source code extraction" );
                }

                //  delete the rtagId directory
                //  If it cannot be deleted then the build has done something ugly
                //  Left over bits will only hurt following builds
                File rId = new File( rtagId );
                deleteDirectory(rId);
                if (rId.exists() )
                {
                    reportErrorException( "Build directory cannot be removed after build", 264 );
                }

                // Always save the log files
                saveLogs();

                return;      
            }

            // target related    
            packageVersion = project.getProperty(packageAlias + "packageversion");
            logMsg("execute packageVersion: ", packageVersion);

            packageExtension = project.getProperty(packageAlias + "packageextension");
            logMsg("execute packageExtension: ", packageExtension);

            packageFullVersion = packageVersion + packageExtension;
            logMsg("execute packageFullVersion: ", packageFullVersion);

            packageLoc = packageName + packageExtension;
            logMsg("execute packageLoc: ", packageLoc);

            if ( project.getProperty(packageAlias + "generic") != null ) {
                generic = true;
                logMsg("execute generic: yes", INFO);  
            }

            directChange = project.getProperty(packageAlias + "directchange");
            if ( directChange != null )
            {
                logMsg("execute directChange: yes", INFO);  
            }

            doesNotRequireSourceControlInteraction = project.getProperty(packageAlias + "doesnotrequiresourcecontrolinteraction");
            if ( doesNotRequireSourceControlInteraction != null )
            {
                logMsg("execute doesNotRequireSourceControlInteraction: true", INFO);  
            }

            //    Display details of the BuildInfo Jats and Ant elements within the
            //    current ABT element
            //
            for (Iterator<BuildInfo> it=buildInfoCollection.iterator(); it.hasNext(); )
            {
                BuildInfo buildInfo = it.next();
                logMsg("execute buildinfo: ", "MachType: " +  buildInfo.getMachtype() + " Method: " + buildInfo.getMethod() + " " + buildInfo.getArg() );
            }


            // Set newVsTag to the existing VcsTag
            // Will be used when rebuilding a package that already exists
            newVcsTag = packageVcsTag;

            //
            //    Determine the location of the package, relative to 'basedir'
            //    Change to this directory for the remainder of the build processing
            loc = project.getProperty(packageAlias + "loc");
            logMsg("execute loc: ", loc);

            cwd = utilities.catDir(basedir, loc);
            logMsg("execute cwd: ", cwd);

            wd = new File( cwd );
            if ( !wd.exists() )
            {
                reportErrorException( "Failure " + cwd + " does not exist", 265 );
            }

            // only now change the thread working directory
            thread.setWorkingDirectory( wd );

            //-----------------------------------------------------------------------
            //    AbtPublish
            //    Note: Have changed directory to the working directory
            //
            if ( packageAlias.compareTo("AbtPublish") == 0 )
            {
                puplishPackage();
                if ( daemon != null )
                {
                    // set properties to drive BuildThread attributes
                    // these are used to determine what ant did
                    getProject().setProperty("abt_fully_published", Boolean.toString(fullyPublished));
                    getProject().setProperty("abt_new_vcstag", newVcsTag);
                }

                return;
            }

            //-----------------------------------------------------------------------
            //    Build Target
            //    Note: Have changed directory to the working directory
            //
            packageVersionID = project.getProperty(packageAlias + "pv_id");
            logMsg("execute packageVersionID: ", packageVersionID);

            unittests = project.getProperty(packageAlias + "unittests");
            if ( unittests != null )
            {
                logMsg("execute unittests: yes", INFO);
            }

            for (Iterator<Depend> it=dependCollection.iterator(); it.hasNext(); )
            {
                Depend depend = it.next();
                logMsg("execute depend: ", depend.getPackage_Alias());
            }

            if ( daemon == null )
            {
                // Escrow Only
                // Recognize an escrow will be run multiple times on a build stage
                //      Do not force continuous rebuilding each time
                //      Has this package version already been published on this platform
                if ( checkEscrowMarker() )
                {
                    logMsg( "execute package has been published on this platform - skipping", INFO);
                    propertyValue = "257";
                    throw new Exception();
                }
            }
            else
            {
                // DEVI 56714 daemon centric check
                String majorV = this.getClass().getPackage().getSpecificationVersion();

                if ( buildtoolVersion.compareTo( majorV ) != 0 )
                {
                    reportErrorException( "Failure, incompatible build.xml not generated with version " + majorV, 256 );
                }
            }


            if (extractSource)
            {
                //  Init the metrics gathering process
                metricsBegin();
                
                //  Generate auto.pl or <PackageName>depends.xml file
                genbuild();
    
                //  Active build
                //      Build the target package unless we are simply doing metrics generation
                //
                if ( !benign )
                {
                    jatsBuildPackage();
                }
    
                //  Complete the metrics gathering
                metricsEnd();
            }
            
            //
            //  Process Skipped builds
            //    Create a built.xxx marker for the publish check
            //
            if (skipped)
            {
                jatsSkipPackage();
            }
            
        }
        catch( SecurityException e)
        {
            errorReported = true;
            reportError( "Failure caught SecurityException", 256 );
        }
        catch( IOException e)
        {
            errorReported = true;
            reportError( "Failure caught IOException", 256 );
        }
        catch( Exception e)
        {
            // assume this condition has been reported on
            errorReported = true;

            if ( e.getMessage() != null )
            {
                logMsg( "execute caught Exception " + e.getMessage(), WARN );
            }
            else
            {
                logMsg( "execute caught Exception", WARN );
            }
        }
        finally
        {
            try
            {
                if ( daemon != null )
                {
                    // do not write to publish.log
                    throw new IOException();
                }

                FileWriter publish = new FileWriter( utilities.catDir(basedir,"publish.log"), true );
                if ( propertyValue.compareTo( "0" ) == 0 )
                {
                    if ( benign )
                    {
                        publish.write( "successfully published " + packageName + " at " + packageFullVersion + ", no action required on this platform" );
                    }
                    else
                    {
                        publish.write( "successfully published " + packageName + " at " + packageFullVersion );
                    }

                    publish.write( ls );
                }
                else if ( propertyValue.compareTo( "256" ) == 0 )
                {
                    publish.write( "failed to publish " + packageName + " at " + packageFullVersion +  ", internal abt issue" + ls );
                }
                else if ( propertyValue.compareTo( "257" ) == 0 )
                {
                    errorReported = false;
                    publish.write( "previously published " + packageName + " at " + packageFullVersion + ls );
                }
                else if ( propertyValue.compareTo( "261" ) == 0 )
                {
                    publish.write( "failed to publish " + packageName + " at " + packageFullVersion + ", missing or bad descpkg file" + ls );
                }
                else if ( propertyValue.compareTo( "262" ) == 0 )
                {
                    publish.write( "failed to publish " + packageName + " at " + packageFullVersion + ", couldn't process save_build info" + ls );
                }
                else if ( propertyValue.compareTo( "263" ) == 0 )
                {
                    publish.write( "failed to publish " + packageName + " at " + packageFullVersion + ", couldn't write to auto.cfg" + ls );
                }
                else if ( propertyValue.compareTo( "265" ) == 0 )
                {
                    publish.write( "failed to publish " + packageName + " at " + packageFullVersion + ", " + cwd + " does not exist" + ls );
                }
                else if ( propertyValue.compareTo( "264" ) == 0 )
                {
                    publish.write( "failed to publish " + packageName + " at " + packageFullVersion + ", " + " Could not delete build directory" + ls );
                }
                else if ( propertyValue.compareTo( "267" ) == 0 )
                {
                    publish.write( "failed to publish " + packageName + " at " + packageFullVersion + ", " + " Could not assemble package fragments" + ls );
                }
                else
                {
                    // nb jats or ant can presumably return a value 1 to 255
                    publish.write( "failed to publish " + packageName + " at " + packageFullVersion + ", jats or ant failed" + ls );
                }

                publish.close();
            }
            catch( IOException e )
            {
                if ( daemon == null )
                {
                    reportError( "Failure to write to publish.log", 266 );
                }
            }
            catch( Exception e )
            {
                if ( daemon == null )
                {
                    reportError( "Failure to write to publish.log", 266 );
                }
            }

            getProject().setProperty(packageAlias + ".res", propertyValue);

            if ( errorReported )
            {
                // designed to prevent subsequent build activity on this build file
                // supply the build failure log file in message with / as file sep
                // takes the form hostname/rtag_id/daemon/logfile
                if ( daemon != null )
                {
                    String message = hostname + "/" + rtagId + "/" + daemon + "/" + logBase + ".log";
                    throw new BuildException( message );
                }
                else
                {
                    throw new BuildException();
                }
            }

        }
    }


    /**
     * Delete a directory and all of its contents
     * Set directories and sub directories writable in case the user has messed with permissions. 
     * This function is called recursively to delete the entire directory tree
     * 
     * Note: Having some issues deleting directories
     *       Try many times. On the second attempt log the files that remained
     * 
     * @param directory The top level directory to delete
     */
    private void deleteDirectory(File directory)
    {
        logMsg("deleteDirectory " + directory.getName(), DEBUG);
       
        int attempts = 0;
        boolean verbose = false;
        while (directory.exists()) {
            attempts++;

            // Use Java native method fist
            // The Jats utility is good, but there is some interaction with VirusScanners/Indexing that a bit strange
            //
            if ( (attempts & 1) == 1) {
                deleteDirectoryInternalViaJava(directory, verbose);
            }  else {
                deleteDirectoryInternalViaJats(directory, verbose);
            }

            if (!directory.exists()) {
                logMsg("deleteDirectory deleted after " + attempts + " attempts " + directory.getName(), FATAL);
                break;
            }
            verbose = true;
            
            if (attempts > 10)
            {
                logMsg("deleteDirectory give up delete after " + attempts + " attempts " + directory.getName(), FATAL);
                break;
            }
            
            logMsg("deleteDirectory not deleted after " + attempts + " attempts " + directory.getName(), FATAL);
            
//          //
//          //  Debugging code
//          //  Display all handles in the system (windows only)
//          //
//          if(System.getProperty("os.name").startsWith("Windows"))
//          {
//              String[] cmd = {"handle"};
//              Log("deleteDirectory Execute handle.", warn);
//              thread.setCommandline(cmd);
//              try {
//                  thread.execute();
//              } catch (IOException e) {
//                  logMsg("deleteDirectory Did not execute handle: " + e.getMessage(), WARN);
//              }
//          }
            
            //  Wait a short while before attempting to delete the directory again
            //  Rational:
            //      - AntiVirus software may be having a sniff, so let it finish
            //      - Program shut down may take a few seconds
            //      - Because nothing else appears to work
            try {
                logMsg("deleteDirectory sleep before attempt:" + attempts, DEBUG);
                Thread.sleep(10 * 1000L);
            } catch (InterruptedException e) {
                logMsg("deleteDirectory sleep exception: " + e.getMessage(), WARN);
                Thread.currentThread().interrupt();
                break;
            }

        }
    }

    /**
     * The body of the deleteDirectory operation
     * Used simply to avoid excessive logging.
     * 
     * @param directory The top level directory to delete
     * @param verbose - display files and folders as they are deleted
     */
    private void deleteDirectoryInternalViaJava(File directory, boolean verbose)
    {
        try
        {
            if ( directory.exists() )
            {
                File[] children = directory.listFiles();
                directory.setWritable(true);
                if ( children != null )
                {
                    for ( int child=0; child < children.length; child++ )
                    {
                        if ( children[ child ].isDirectory() )
                        {
                            deleteDirectoryInternalViaJava( children[ child ], verbose );
                        }
                        else
                        {
                            if(verbose)
                            {
                                logMsg("deleteDirectory File:" + children[ child ].getName(), DEBUG);
                            }
                            children[ child ].setWritable(true);
                            children[ child ].delete();
                        }
                    }
                }
                if (verbose)
                {
                    logMsg("deleteDirectory Dir:" + directory.getName(), DEBUG);
                }
                directory.delete();
            }
        }
        catch( SecurityException e )
        {
            // this can be thrown by exists and delete
            logMsg("deleteDirectory caught SecurityException", WARN);
        }
    }

    
    /**
     * The body of the deleteDirectory operation done with JATS utility
     * Used simply to avoid excessive logging.
     * 
     * @param directory The top level directory to delete
     * @param verbose - display files and folders as they are deleted
     */
    private void deleteDirectoryInternalViaJats(File directory, boolean verbose)
    {
      
        //  Use jats ebin JatsFileUtil T0 TextMsg DirName to delete the directory tree
        //  Address issues that cannot be solved in java
        
        command.init("jats", "ebin","JatsFileUtil" );
        command.add( verbose, "T9" );
        command.add( !verbose, "T0" );
        command.add( "DeleteDir-" + rtagId );
        command.add(directory.getAbsolutePath());
        
        // Run the command - don't care if it fails
        //  It should not generate an exception
        try {
            runCommand(null);
        } catch (Exception e1) {
            logMsg("deleteDirectory caught Exception", ERROR);
        }
    }

    /**
     * Insert a marker into the created package so that the
     * escrow build can detect that the package has been built.
     * 
     * This is used in the following situations:
     * 
     * 1) ANT packages that could be build on multiple machines
     * 2) Restarted build on the same machine
     */
    private void insertEscrowMarker()
    {
        //
        //  Only used in escrow mode
        //
        if ( daemon == null && packageStandard == BuildStandards.ANT)
        {
            //
            //  ANT builds are performed on a machine type basis
            //  Create a 'MachClass' based marker
            //
            String destination = utilities.catDir(dpkgArchive, packageName, packageFullVersion);
            String filename = "built." + gbeMachClass; 

            try
            {
                new File(destination).mkdirs(); 
                new File(destination, filename).createNewFile();
            } 
            catch (IOException ioe)
            {
                logMsg("Cannot insert Escrow Marker", FATAL);
            }
        }
    }


    /**
     * Check for the existence of an Escrow marker
     * Used so that the build system can detect if this
     * package has already been built.
     * 
     * Can occur under two conditions
     * 
     * 1) ANT packages that can be built on multiple machines
     * 2) ReStarted build on same machine 
     *  
     * Assumption: Only called in Escrow Mode 
     * 
     * @return TRUE  - Marker Detected
     *         FALSE - Marker not Detected
     */
    private boolean checkEscrowMarker() throws Exception
    {
        String destination = utilities.catDir(dpkgArchive, packageName, packageFullVersion);
        String filename = "built." + (generic ? "generic" : hostname);
        
        if (new File( destination, filename ).exists())
        {
            return true;
        }

        if ( packageStandard == BuildStandards.ANT)
        {
            //
            //  ANT builds are performed on a machine type basis
            //  Check for a 'MachClass' based marker
            //
            filename = "built." + gbeMachClass; 
            if (new File( destination, filename).exists())
            {
                return true;
            }
        }
        return false;
    }

    //---------------------------------------------------------------------------
    //    Extend the <abt> task with several new elements
    //
    //        <depend package_alias="${debian_dpkg.cots}"/>
    //        <buildinfo gbe_machtype="solaris10_x86" method="jats" arg="debug"/>
    //
    public Depend createDepend()
    {
        Depend depend = new Depend();
        dependCollection.add(depend);
        return depend;
    }

    public BuildInfo createBuildInfo()
    {
        BuildInfo buildInfo = new BuildInfo();
        buildInfoCollection.add(buildInfo);
        return buildInfo;
    }

}