Subversion Repositories DevTools

package com.erggroup.buildtool.abt;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileOutputStream;

import java.io.FileWriter;

import java.io.IOException;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.Properties;

import org.apache.log4j.Logger;

import org.apache.tools.ant.BuildException;

import org.apache.tools.ant.Project;

import org.apache.tools.ant.Target;

import org.apache.tools.ant.Task;

import org.apache.tools.ant.taskdefs.Execute;

import org.apache.tools.ant.taskdefs.Redirector;

import com.erggroup.buildtool.abt.RemoteExecution.LogOutput;

import com.erggroup.buildtool.smtp.CreateUrls;

import com.erggroup.buildtool.smtp.Smtpsend;

import com.erggroup.buildtool.utilities.CommandBuilder;

import com.erggroup.buildtool.utilities.ElapseTime;

import com.erggroup.buildtool.utilities.utilities;

public class ABT extends Task

{

    private String ls = System.getProperty( "line.separator" );

    private String gbeMachtype = System.getenv("GBE_MACHTYPE");

    private String dpkgArchive = System.getenv("GBE_DPKG");

    private String hostname = System.getenv("GBE_HOSTNAME");

    private String gbeMachClass;          // Escrow Only

    // ant built in

    private Project project;

    private String basedir;

    // global

    private boolean master = false;

    private String gbebuildfilter;

    private String mailServer;

    private String mailSender;

    // project related

    private String rtagId;

    private String daemon;

    private String release;

    private String buildtoolVersion;

    private String family;

    private String oldExtension;

    private String newExtension;

    private ABTData abtData;

    // target related

    private Target target;

    private String packageVersionID;

    private String packageAlias;

    private String packageName;

    private String packageVersion;

    private String packageFullVersion;

    private String packageExtension;

    private String packageLoc;

    private String packageVcsTag;

    private String directChange;

    private String doesNotRequireSourceControlInteraction;

    private boolean isaRipple;

    private int testBuildInstruction = 0;

    private boolean generic = false;

    private boolean packageTarBall = false;

    private boolean useTestArchive = false;

    private String loc;

    private String unittests;

    private ArrayList<Depend> dependCollection = new ArrayList<Depend> ();

    private ArrayList<BuildInfo> buildInfoCollection = new ArrayList<BuildInfo>();

    private StringBuilder owners = new StringBuilder();;

    private Machine mMachine;

    private String logBase;

    // set up in execute

    private BuildStandards packageStandard = BuildStandards.NONE;           // Build standard for the package

    private BuildStandards buildStandard = BuildStandards.NONE;             // Build Standard for this instance

    private JavaVersions javaVersion = JavaVersions.NONE;

    private CompileTargets compileTarget = CompileTargets.NONE;

    // abt task result property

    private String propertyValue = "0";

    // output redirector

    private Redirector re;

    private File output;

    // exec thread

    private CommandBuilder command = new CommandBuilder();

    private String cwd;

    private Execute thread;

    private File wd;

    private String newVcsTag;

    private static final Logger mLogger = Logger.getLogger(ABT.class);

    private boolean fullyPublished = false;

    private static final char DEBUG = 0;

    private static final char INFO = 1;

    private static final char WARN = 2;

    private static final char ERROR = 3;

    private static final char FATAL = 4;

    //  Simple class to contain error information to be reported to the user

    private class ErrorInfo {

        String cmd;

        int result;

        String error;

        public ErrorInfo (String error)

        {

            this.error = error;

        }

        public ErrorInfo(String cmd, int result, String error)

        {

            this.cmd = cmd;

            this.result = result;

            this.error = error;

        }

        public String toString()

        {

            String rv = error;

            if (cmd != null)

            {

                rv += " : " + cmd;

                rv += " returned " + result;

            }

            return rv;

        }

        public String toHtml() {

            String rv = "Error : " + error;

            if (cmd != null)

            {

                rv += "<br>Command : " + cmd;

                rv += "<br>Returned: " + result;

            }

            return rv;

        }

    }

    /** Log a message to two locations

     *  1) log4j application wide log - using facility

     *  2) Ant Command output log - always

     * 

     * @param message   - Message to log

     * @param facility  - Log level

     */

    private void logMsg(String message, char facility)

    {

        if ( daemon != null )

        {

            if ( facility == DEBUG)

            {

                mLogger.debug(message);

            }

            else if ( facility == INFO)

            {

                mLogger.info(message);

            }

            else if ( facility == WARN)

            {

                mLogger.warn(message);

            }

            else if ( facility == ERROR)

            {

                mLogger.error(message);

            }

            else

            {

                mLogger.fatal(message);

            }

        }

        log( message );

    }

    /** Log two strings as 'info'

     *  The two strings are simply concatenated.

     * 

     * @param message1  - First part of the message

     * @param message2  - Second part of the message

     */

    private void logMsg(String message1, String message2)

    {

        logMsg( message1 + message2, INFO );

    }

    /** Add package location options

     *  Options depend on the build standard in use

     */

    private void addCommandLocate()

    {

        if ( packageStandard == BuildStandards.ANT )

        {

            command.add( "-locatefile=" + packageName + ".xml" );

        }

        else

        {

            command.add( "-locatepkg=" + packageLoc );

        }

    }

    /** Execute the command currently being assembled

     *  Logs command, exit code and execution duration

     * 

     * @param message           - Null: Don't report errors, otherwise text to prepend to the reported error

     * @return                  - Return code from the executed command

     * @throws Exception

     */

    private int runCommand( String message ) throws Exception

    {

        ElapseTime et = new ElapseTime();

        //  Need to create a cmd array with exactly the required number of arguments

        //  thread.setCommandline() cannot handle null items in the array

        //  Create a single logMessage while processing the array

        String logMessage = command.toString();

        logMsg( logMessage, WARN );

        thread.setCommandline(command.getArray());

        int retVal = thread.execute();

        //  Report error conditions 

        if ( retVal == Execute.INVALID )

        {

            reportErrorException( "Failure to execute thread, internal ABT problem", 256 );

        }

        else

        {

            logMsg( "command returned: "+ retVal  + ", elapsed time (S): " + et.toString(), WARN);

            if ( message != null && retVal != 0 )

            {

                ErrorInfo errorData = new ErrorInfo(command.toString() , retVal, message );

                reportErrorException( errorData, retVal );

            }

        }

        return retVal;

    }

    /**

     * Check that one marker file has been published into dpkg_archive 

     * Assumption - not called in Escrow Mode 

     * 

     * @param archive     - Archive to test

     * @param tag         - Marker suffix

     * 

     * @return true if the package is published 

     */

    private boolean checkPublished(String archive, String tag) throws Exception

    {

        if (archive == null)

        {

            reportErrorException( "GBE_DPKG environment variable not set", 256 );

        }

        String filename = "built." + tag;

        String destination = utilities.catDir(archive, packageName,  packageFullVersion); 

        //

        //  Have been having issues with NFS caching

        //  Current attempt is to flush NFS caches using freshFile(), but at the moment

        //  the code will still retry a number of time. This 'should' not be required if the

        //  flushing code is operational.

        //

        //  Insert log info if we have tried more than once

        //  Idea is to monitor first time failures and then delete this code

        //

        boolean found = false;

        int   maxwait = 2 * 4;

        for ( int ii = 0; ii < maxwait && !found; ii++)

        {

            found = utilities.freshFileExists(utilities.catDir(destination, filename));

            if ( !found )

            {

                // Debug - display files in the 

                try

                {

                    String files;

                    File folder = new File(destination);

                    File[] listOfFiles = folder.listFiles();

                    if ( listOfFiles == null )

                    {

                        logMsg( "published. Destination not a dir: " + destination, FATAL );

                    }

                    else

                    {

                        for (int i = 0; i < listOfFiles.length; i++) 

                        {

                            if (listOfFiles[i].isFile())

                            {

                                files = listOfFiles[i].getName();

                                logMsg("published. DirContent:" + files , WARN);

                            }

                        }

                    }

                }

                catch( Exception e )

                {

                    logMsg( "published. Display files caught Exception:" + e.getMessage(), FATAL );

                }

                Thread.sleep(500);

                logMsg("published. Wait for " + filename + ".(" + ii +")" , WARN);

            }

        }

        if ( found )

        {

            logMsg("published. " + filename + " exists", WARN);

        }

        else

        {

            logMsg("published. " + filename + " does not exist", WARN);

        }

        return found;

    }

    /** Print a line of text to a file

     * 

     * @param file  - FileWrite to print to

     * @param msg1  - Text to write. Will append line separator

     * @throws IOException

     */

    private void printPackage( FileWriter file, String msg1 ) throws IOException

    {

        file.write( msg1 + ls );

    }

    /** Copy a file

     * 

     * @param source    - Source File

     * @param dest      - Target File

     */

    private void copyFile( File source, File dest )

    {

        if ( source.exists() )

        {

            try

            {

                FileInputStream fis = new FileInputStream(source);

                FileOutputStream fos = new FileOutputStream(dest);

                byte[] buf = new byte[1024];

                int i = 0;

                while( (i=fis.read(buf)) != -1)

                {

                    fos.write(buf, 0, i);

                }

                fis.close();

                fos.close();

            }

            catch( Exception e )

            {

                logMsg( "copyFile caught Exception", ERROR );

            }

        }

    }

    /** Save the log file

     *  <br>Used on error build

     *  <br>Used on complete build

     * 

     * @return  True: Can log ( Daemon mode)

     */

    private boolean saveLogs()

    {

        if ( daemon == null )

        {

            // do nothing more in escrow

            return false;

        }

        // persist the failure log to GBE_LOG/hostname/rtagid/daemon

        String log = System.getenv("GBE_LOG");

        if ( log == null )

        {

            // cannot happen, GBE_LOG is checked in BuildDaemon at startup

            return false;

        }

        File destinationLog = new File( log );

        File destinationHostname = new File( destinationLog, hostname );

        File destinationRtagId = new File( destinationHostname, rtagId );

        // create directories

        new File( destinationRtagId, daemon).mkdirs();

        File destinationDaemon = new File( destinationRtagId, daemon );

        if ( destinationDaemon.exists() )

        {

            File buildsource = new File(rtagId + "build.xml");

            File buildcopy = new File( destinationDaemon, logBase + ".xml");

            copyFile( buildsource, buildcopy );

            File logsource = new File(rtagId + ".log");

            File logcopy = new File( destinationDaemon, logBase + ".log");

            copyFile( logsource, logcopy );

            logMsg("reportError persisting " + logcopy.getName(), WARN);

        }

        return true;

    }

    private void reportError(String error, int retVal)

    {

        reportError( new ErrorInfo(error), retVal);

    }

    private void reportError(ErrorInfo error, int retVal)

    {

        propertyValue = String.valueOf( retVal );

        logMsg( error.toString(), ABT.ERROR );

        logMsg( "reportError propertyValue = " + propertyValue + ", ", error.toString());

        //  Save the logs

        //      Will return false in situations were we can't proceed

        if ( ! saveLogs() )

        {

            return;

        }

        //  Only send emails if we have all the bits required

        //  Will not get this far unless we are in daemon mode

        if ( owners.length() > 0 && mailServer != null && mailSender != null )

        {

            String href = hostname + "/" + rtagId + "/" + daemon + "/" + logBase + ".log"; 

            String unc =  CreateUrls.generateUncBaseUrl() + "/" + href;

            String body = "Release: " + release + "<p>" +

                   "Package: " + packageName + "<p>" + 

                   "RM link: " + CreateUrls.generateRmUrl(rtagId,packageVersionID) + "<p>" +

                   "Build machine: " + hostname + "<p>" +

                   "Machine type: " + gbeMachtype + "<p>" +

                   error.toHtml() + "<p>" +

                   "Log file: <a href=\"" + unc + "\">" + href + "</a> ... URL expires in 5 working days time<p><hr>";

            try

            {

                Smtpsend.send( mailServer,          // mailServer

                               mailSender,          // source

                               owners.toString(),   // target

                               null,                // cc

                               null,                // bcc

                               "BUILD FAILURE on package " + packageName, // subject

                               body,                // body

                               null                 // attachment

                        );

            }

            catch( Exception e )

            {

                // not fatal

                logMsg( "reportError caught Exception", WARN);

            }

        }

    }

    private void reportErrorException(String error, int retVal) throws Exception

    {

        reportError( error, retVal );

        throw new Exception( hostname );

    }

    private void reportErrorException(ErrorInfo error, int retVal) throws Exception

    {

        reportError( error, retVal );

        throw new Exception( hostname );

    }

    /**

     * Init the metrics generation process

     * 

     * @exception Exception

     */

    private void metricsBegin() throws Exception

    {

        if ( master )

        {

            // gather metrics

            // jats -locatefile=.jats.packageroot etool jats_metrics -mode=init -rootdir=.

            // -locate=.jats.packageroot causes jats to cd to the folder containing the file .jats.packageroot

            // this file is created in the -path folder by jats release -extract

            // DEVI 51370 it is essential for metrics branch calculation the metrics utility is started within a static view

            //

            command.init("jats" );

            command.add( "-locatefile=.jats.packageroot" );

            command.add( "etool", "jats_metrics" );

            command.add( "-mode=init" );

            command.add( "-rootdir=." );

            runCommand( "Failure to collect metrics" );

        }     

    }

    /**

     * Complete the metrics generation process

     * 

     * @exception Exception

     */

    private void metricsEnd() throws Exception

    {

        if ( master )

        {

            File metrics = new File(utilities.catDir(basedir, rtagId + "abtmetrics.txt"));

            if ( metrics.exists() )

            {

                metrics.delete();

            }

            // gather metrics

            // jats -locatefile=.jats.packageroot etool jats_metrics -mode=finish -out=abtmetrics.txt

            command.init("jats" );

            command.add( "-locatefile=.jats.packageroot" );

            command.add( "etool", "jats_metrics" );

            command.add( "-mode=finish" );

            command.add( "-out=" + metrics.getAbsolutePath() );

            runCommand( "Failure to collect metrics" );

        }    

    }

    /**

     * Generate the build dependency data file.

     * 

     * @exception Exception

     */

    private void genbuild() throws Exception

    {

        //

        //  Create build files with updated dependency information

        //  In escrow mode : No new dependency information

        //  In daemon mode :

        //      Ripple     : New dependency information

        //      Verify     : Existing dependency information

        //

        //  In all modes  : ANT builds need an auto.xml

        //

        //  In order to create auto.xml/auto.pl with package dependencies need to

        //  create file with dependency data (auto.cfg) only because its

        //  needed by "jats_rewrite"

        //

        File cfg = null;

        boolean  createFile = false;

        if ( daemon != null )

        {

            createFile = true;

            // always generate auto.cfg

            cfg = new File( utilities.catDir(cwd, "auto.cfg"));

            cfg.createNewFile();

            if ( !cfg.exists() || !cfg.canWrite() )

            {

                reportErrorException( "Failed to open auto.cfg", 263 );

            }

            try

            {

                FileWriter cfgfw = new FileWriter(cfg);

                printPackage( cfgfw, packageName + " " + packageFullVersion );

                // provide releasemanager.projectname=<name[0]> and releasemanager.releasename=<name[1]>

                String[] name = release.split( " > ", 2 );

                if ( name.length == 2 )

                {

                    printPackage( cfgfw, "releasemanager.projectname=" + name[0] );

                    printPackage( cfgfw, "releasemanager.releasename=" + name[1] );

                }

                for (Iterator<Depend> it=dependCollection.iterator(); it.hasNext(); )

                {

                    Depend depend = it.next();

                    printPackage( cfgfw, depend.getPackage_Alias() );

                }

                cfgfw.close();

            }

            catch( IOException e )

            {

                reportErrorException( "Failure to write to auto.cfg", 263 );

            }

        }

        else

            if ( packageStandard == BuildStandards.ANT )

            {

                // Have not created an auto.xml with updated dependencies

                // Ant build still would like an auto.xml file

                //

                // ie packages with a line <import file="auto.xml" optional="true"/> are fine

                //    but in escrow and rebuild we still need an auto.xml to be kind

                //

                createFile = true;

            }

        //

        //  Create the auto.pl/auto.xml file if required

        //

        if ( createFile )

        {

            command.init("jats");

            addCommandLocate();

            command.add( "etool", "jats_rewrite" );

            if ( packageStandard == BuildStandards.ANT )

            {

                command.add( "-infile=" + packageName + "depends.xml" );

                command.add( "-outfile=auto.xml" );

            }

            else

            {

                command.add( "-infile=build.pl" );

                command.add( "-outfile=auto.pl" );

            }

            if ( cfg != null && daemon != null)

            {

                command.add( "-config=" + cfg.getAbsolutePath() );

                //  Validate the Release Manager versions against those in the build files

                //  Only do this in daemon mode for:

                //      We are rebuilding the package

                //          This is detected as: Not have VCS interaction AND is not a test Build

                boolean isaRebuild = doesNotRequireSourceControlInteraction != null && testBuildInstruction == 0;

                command.add(isaRebuild, "-validate");

                //  Ensure that we don't have excessive packages in the Metadata

                //  Only do this in daemon mode, for 'new' or 'test' builds

                //  Don't do for:

                //      Escrow builds - they can't fix it

                //      Ripple builds - it will break a lot of stuff

                //      Rebuilds      - it will break stuff

                // Do for:

                //      New Versions  - User can fix it

                //      Test Builds   - User can fix it

                boolean isaTestBuild = doesNotRequireSourceControlInteraction != null && testBuildInstruction > 0;

                boolean isaNewVersion = !isaRipple && !isaRebuild && !isaTestBuild;

                command.add( isaNewVersion || isaTestBuild, "-errors");

            }

            else

            {

                command.add( "-noconfig");

            }

            if ( oldExtension != null )

            {

                command.add( "-oldproject", oldExtension );

                command.add( "-newproject", newExtension );

            }

            runCommand( "Failure to generate build dependency information. Potentially caused by a mismatch between build files and Release Manager database" );

        }

    }

    /**

     * Build the package

     * 

     * @exception Exception

     */

    private void jatsBuildPackage() throws Exception

    {

        //--------------------------------------------------------------------------

        //  Wait for package replication and caching

        //      If there is a GBE_DPKG_REPLICA, then there may be a delay between

        //      a package being built and the package being available on the replica

        //      We need to wait for the required packages to replicate

        //

        //      If there is a GBE_DPKG_CACHE, then we need to cache the package

        //      into the local store - mostly for windows as it seriously speeds

        //      up the builds.

        Boolean hasReplica = System.getenv("GBE_DPKG_REPLICA") != null;

        Boolean hasCache   = System.getenv("GBE_DPKG_CACHE") != null;

        if (daemon != null && (hasReplica || hasCache)  && !dependCollection.isEmpty())

        {

            /*

             * Limit the command line length provided to the following utility

             * Very very long dependency lists may over fill the command line

             */

            Iterator<Depend> it=dependCollection.iterator();

            while( it.hasNext() )

            {

                command.init( "jats", "etool", "cache_dpkg");

                command.add( hasReplica, "-wait" );

                // ANT builds require the package to be cached

                // JATS does not - if its needed it will be done later

                command.add( buildStandard != BuildStandards.ANT , "-nocache");

                // Add PackageName/PackageVersion

                for (int maxArgs = 0; maxArgs < 10 && it.hasNext(); maxArgs++ )

                {

                    Depend depend = it.next();

                    command.add(depend.getPackageNameVersion());

                }

                // Run the command - don't care if it fails

                runCommand( null );

            }

        }

        //--------------------------------------------------------------------------

        //  Build Phase

        //  Create build environment

        command.init( "jats" );

        command.add( gbebuildfilter != null ,"-buildfilter", gbebuildfilter);

        addCommandLocate();

        if ( buildStandard == BuildStandards.ANT )

        {

            command.add( "abt" );

            command.add( "-java=" + javaVersion.getJavaVersion() );

            command.add( "-buildfile=" + packageName + ".xml" );

            command.add( "build" );

        }

        else

        {

            command.add( "build", "--package" );

            // Daemon: Sanity check the building of Generic packages

            // Escrow: Allow this error - for backward compatibility

            if ( daemon != null )

            {

                if (generic) 

                    command.add( "-generic" );

                else

                    command.add( "-nogeneric" );

            }

            // Cache packages into a machine-local archive

            //  Will only do this if the package is really built

            //

            command.add(hasCache, "--cache" );

        }

        runCommand( "Failure to set up sandbox" );

        //--------------------------------------------------------------------------

        //  Make Package phase

        //  Create the package (make)

        command.init("jats" );

        command.add( gbebuildfilter != null, "-buildfilter", gbebuildfilter);

        addCommandLocate();

        if ( buildStandard == BuildStandards.ANT )

        {

            command.add( "abt" );

            command.add( "-java=" + javaVersion.getJavaVersion() );

            command.add( "-buildfile=" + packageName + ".xml" );

            command.add( "make_package" );

        }

        else

        {

            command.add( "make", "NODEPEND=1" );

            if ( compileTarget == CompileTargets.PROD )

            {

                command.add( "prod", "package_prod" );

            }

            else if ( compileTarget == CompileTargets.DEBUG )

            {

                command.add( "debug", "package_debug" );

            }

            else

            {

                command.add( "all" );

            }

        }

        runCommand( "Failure to generate derived files" );

        //--------------------------------------------------------------------------

        //  Unit tests phase

        //  Run autonomous unit tests

        if ( unittests != null )

        {

            command.init("jats" );

            command.add( gbebuildfilter != null, "-buildfilter", gbebuildfilter);

            addCommandLocate();

            if ( buildStandard == BuildStandards.ANT )

            {

                command.add( "abt" );

                command.add( "-java=" + javaVersion.getJavaVersion() );

                command.add( "-buildfile=" + packageName + ".xml" );

                command.add( "run_tests" );

            }

            else

            {

                command.add( "make" );

                if ( compileTarget == CompileTargets.PROD )

                {

                    command.add( "run_unit_tests_prod" );

                }

                else if ( compileTarget == CompileTargets.DEBUG )

                {

                    command.add( "run_unit_tests_debug" );

                }

                else

                {

                    command.add( "run_unit_tests" );

                }

            }

            runCommand( "Failure to run unit tests" );

        }

        //--------------------------------------------------------------------------

        //  create_dpkg phase

        //  Publish the package to dpkg_archive

        //

        command.init("jats" );

        addCommandLocate();

        command.add( "create_dpkg", "-m" );

        command.add( packageTarBall, "-tarmode");

        command.add( generic, "-generic" );

        command.add( "-pname", packageName );

        command.add( "-pversion", packageFullVersion );

        runCommand( "Failure to publish to archive" );

        insertEscrowMarker();

    }

    /** Skip Build an ANT Package 

     *  Only used on ANT based builds in which it has been determined that another machine

     *  is performing the build.

     *  

     *  Insert required marker files to indicate that the build has occurred

     *  Perform this using JATS commands to ensure correct creation of the marker files

     *  Basically, a dummy publish phase

     *  

     *  Note: Skip Builds cannot be generic

     *  

     */

    private void jatsSkipPackage() throws Exception

    {

        command.init( "jats" );

        command.add( "-here" );

        command.add( "create_dpkg", "-m" );

        command.add( packageTarBall, "-tarmode");

        command.add( "-noBuild" );

        command.add( "-pname", packageName );

        command.add( "-pversion", packageFullVersion );

        runCommand( "Failure to (skip)publish to archive" );

    }

    /**   Check if the package has been Published on all the required machines

     *    If its a 'generic' package then a tag of built.generic will have been placed in the package

     *    Otherwise each build machine will place a built.<MachName> file in the package when

     *    its build is completed successfully.

     *  

     *    If we don't have a complete set of 'built' files, then the package failed to build

     *    on one or more platforms. The assumption is that the build failure has been reported

     *    by the build daemon.

     *  

     *    All we know is which MachTypes we build for. This needs to be expanded into

     *    a list of MachineNames. Thus a build-set with multiple machines of the same MachType

     *    can indicate build failures.

     */

    private void puplishPackage() throws Exception

    {  

        //  If the package fragments are being delivered as tarballs then they needs to be reassembled

        if (packageTarBall)

        {

            fullyPublished = assemblePackage();

        }

        if (fullyPublished)

        {

            if ( generic )

            {

                //  Generic - only one marker file

                fullyPublished = checkPublished( dpkgArchive, "generic" );

            }

            else

            {

                //  Multi machine package

                //  Scan required target machines artifacts

                //  Expand from MachType to one or more MachName's

                scanBuilds:

                    for (Iterator<BuildInfo> ip=buildInfoCollection.iterator(); ip.hasNext(); )

                    {

                        BuildInfo buildInfo = ip.next();

                        for (Iterator<Machine> it=abtData.machineCollection.iterator(); it.hasNext(); )

                        {

                            Machine ma = it.next();

                            if (ma.getMachtype().compareTo(buildInfo.getMachtype()) == 0 )

                            {

                                fullyPublished = checkPublished( dpkgArchive, ma.getName() );

                                if ( !fullyPublished )

                                {

                                    //  Break out of both loops

                                    //  Once we have seen an error we need to retain it.

                                    break scanBuilds;

                                }

                            }

                        }

                    }

            }

        }

        //  Detect problem with zero length descpkg file

        //      May be caused by write clash from multiple machines

        //      Don't know

        //

        //  Not forcing NFS flush as this will have been done in the previous step

        //

        if ( fullyPublished )

        {

            String destination = utilities.catDir(dpkgArchive, packageName, packageFullVersion, "descpkg");

            File flag = new File( destination);

            //  File Must

            //      Exist

            //      Be a File

            //      Have a length. Problem appears to be zero length file

            //

            boolean found = false;

            logMsg( "published. Descpkg Sanity Test: " + flag.getPath(), DEBUG );

            if (!flag.exists() ) {

                reportErrorException( "No descpkg file. Package may not build with current build filters", 261 );

            } else if ( !flag.isFile()) {

                reportErrorException( "Descpkg Sanity Test: Is not a file", 261 );

            } else if ( flag.length() <= 0 ) {

                reportErrorException( "Descpkg Sanity Test: Zero length File", 261 );

            } else {

                logMsg( "published. Descpkg Sanity Test: Passed", INFO );

                found = true;

            }

            if (!found) {

                fullyPublished = false;

            }

        }

        if ( fullyPublished && daemon != null )

        {

            if ( doesNotRequireSourceControlInteraction == null )

            {

                //

                //  Save modified build files

                //  Label the resultant files

                //  Create a command of the form:

                //      jats etool jats_vcssave_build

                //          -infile     auto.xml/auto.pl

                //          -outfile    xxxdepends.xml/build.pl

                //          -pname      package_name

                //          -pversion   package_version

                //          -infofile   path_to_info_file

                //          -wiplabel   Existing WIP label (optional)

                //          -baselabel  View label

                logMsg("publish save build files", WARN);

                File saveinfo = new File(utilities.catDir(basedir, rtagId + "abtinfo.txt"));

                if ( saveinfo.exists() )

                {

                    saveinfo.delete();

                }

                command.init("jats" );

                addCommandLocate();

                command.add( "etool", "jats_vcssave_build" );

                if ( packageStandard == BuildStandards.ANT )

                {

                    command.add( "-infile", "auto.xml" );

                    command.add( "-outfile", packageName + "depends.xml" );

                }

                else

                {

                    command.add( "-infile", "auto.pl" );

                    command.add( "-outfile", "build.pl" );