Subversion Repositories DevTools

package com.erggroup.buildtool.daemon;

import com.erggroup.buildtool.abt.BuildToolException;

import com.erggroup.buildtool.abt.RemoteExecution;

import com.erggroup.buildtool.abt.RemoteExecution.LogOutput;

import com.erggroup.buildtool.daemon.BuildThread;

import com.erggroup.buildtool.ripple.*;

import com.erggroup.buildtool.ripple.BuildFile.BuildFileState;

import com.erggroup.buildtool.ripple.Package;

import com.erggroup.buildtool.ripple.ReleaseManager.BuildResult;

import com.erggroup.buildtool.ripple.RunLevel.BuildState;

import com.erggroup.buildtool.smtp.CreateUrls;

import com.erggroup.buildtool.smtp.Smtpsend;

import com.erggroup.buildtool.utilities.*;

import java.io.BufferedReader;

import java.io.DataInputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.IOException;

import java.io.InputStreamReader;

import java.sql.SQLException;

import java.util.Iterator;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

/**Master Thread sub component

 */

public class MasterThread extends BuildThread

{

    /**Logger

     * @attribute

     */

    private static final Logger mLogger = LoggerFactory.getLogger(MasterThread.class);

    /** Current BuildFile

     * 

     */

    private BuildFile mMasterBuildFile = new BuildFile();

    /** Measure the duration of the current build

     * 

     */

    private ElapseTime mBuildDuration = new ElapseTime();

    /**constructor

     * @param rtagId          - Identifies the Release

     * @param rconId          - Identifies the Controlling entry

     * @param releaseManager  - A ReleaseManager entry to use

     * @param unitTest        - Unit Test data 

     */

    public MasterThread(int rtagId, int rconId, ReleaseManager releaseManager, String unitTest)

    {

        super(rtagId, rconId, releaseManager, unitTest);

        mLogger.warn("MasterThread rtag_id " + rtagId + " rcon_id " + rconId);

    }

    /**implements the sequence diagrams coordinate slave threads, generate build files, allowed to proceed, check environment

     */

    public void run()

    {

        Integer id = Integer.valueOf(mRtagId);

        setName(id.toString());

        mLogger.warn("Master Thread run");

        boolean exit = false;

        mRippleEngine = new RippleEngine(mReleaseManager, mRtagId, true);

        while(!exit)

        {

            try

            {

                mLogger.warn("run calling sleepCheck");

                mPhase.setPhase("sleepCheck");

                sleepCheck();

                mLogger.warn("run calling rippleEngine.collectMetaData");

                mPhase.setPhase("collectMetaData", 600);

                mRippleEngine.collectMetaData();

                if ( Thread.currentThread().isInterrupted() )

                {

                    mLogger.warn("run is interrupted");

                    // unit test technique

                    throw new ExitException();

                }

                if ( mUnitTest.compareTo("unit test spawn thread") == 0)

                {

                    throw new Exception();

                }

                if ( !mUnitTest.startsWith("unit test check environment"))

                {

                    if ( mUnitTest.compareTo("unit test generate build files") != 0)

                    {

                        if ((mUnitTest.compareTo("unit test allowed to proceed") != 0) &&

                                (mUnitTest.compareTo("unit test not allowed to proceed") != 0) &&

                                (mUnitTest.compareTo("unit test exit") != 0))

                        {

                            //---------------------------------------------------------------

                            //  Wait for all Slaves to enter waiting or paused state

                            //      They may be building

                            //      They may be starting up

                            //

                            mLogger.warn("run changing run level to WAITING (New) for rcon_id {}", mRconId);

                            mPhase.setPhase("Wait for Slave Threads");

                            mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_WAITING);

                            while (true) 

                            {

                                StringBuilder waitList = new StringBuilder();

                                mLogger.info("run calling mReleaseManager.queryRunLevel");

                                mReleaseManager.queryRunLevel(mRtagId);

                                //	Process all run entries

                                for (Iterator<RunLevelData> it = mReleaseManager.mRunLevelCollection.iterator(); it.hasNext(); )

                                {

                                    RunLevelData rl = it.next();

                                    //

                                    // Only examine Slaves.

                                    // This is being run on a Master and at this point the Master is not waiting

                                    // If there are multiple (misconfigured) Masters, then we can get confused

                                    //

                                    // If I am no longer a Master we will process till the end of the build cycle

                                    // at which point it will be detected.

                                    //

                                    // If there are other Masters, then we assume that they are not waiting

                                    //

                                    // Ignore myself. I am not waiting. I am asking the questions

                                    if ( rl.get_rcon_id() == mRconId) {

                                        continue;

                                    }

                                    // Ignore other Master. They should not be waiting.

                                    if ( rl.isMaster()) {

                                        continue;

                                    }

                                    //

                                    // A slave that is 'Paused' is not processing a package

                                    //      It will have been included in the build-set, so we must wait for

                                    //      the pause to be removed.

                                    //

                                    // A slave that is 'Waiting' must also have no BuildFile present

                                    //      This signals that the buildfile has been taken by the slave

                                    // 

                                    if (! (rl.isAt(BuildState.DB_WAITING) && !rl.hasBuildFile() )  )

                                    {

                                        // A non-waiting slave has been found

                                        waitList.append(" ");

                                        waitList.append(rl.get_rcon_id());

                                    }

                                }

                                if ( waitList.length() > 0 )

                                {

                                    // One or more slaves still processing ...

                                    if ( mUnitTest.compareTo("unit test coordinate slave threads") == 0 )

                                    {

                                        Thread.currentThread().interrupt();

                                    }

                                    else

                                    {

                                        mLogger.warn("Waiting 3 seconds for slaves:" + waitList.toString() );

                                        Thread.sleep(3000);

                                        mLogger.info("run sleep returned");

                                    }

                                    continue;

                                }

                                else

                                {

                                    // All slaves have completed there tasks

                                    break;

                                }

                            }

                            if ( mUnitTest.compareTo("unit test coordinate slave threads") == 0 )

                            {

                                throw new ExitException();

                            }

                            //---------------------------------------------------------------

                            //    Publish Package Build Results

                            //        Indicate to RM that we are publishing

                            //        Ensure that all build machines have placed a marker in dpkg_archive

                            //        Save updated build files into VCS

                            // 

                            mLogger.warn("run changing run level to PUBLISHING for rcon_id {}", mRconId);

                            mPhase.setPhase("Publishing");

                            mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_PUBLISHING);

                            mLogger.info("run calling deliverChange on AbtPublish");

                            deliverChange("AbtPublish", true, false);

                            //---------------------------------------------------------------

                            //    TearDown the build workspace

                            //        Remove the build workspace

                            //        Save build log files

                            //        Don't update the Reporting Information as it will null out

                            //        some essential information that has been collected.

                            //

                            mLogger.info("run calling deliverChange on AbtTearDown");

                            mPhase.setPhase("AbtTearDown");

                            deliverChange("AbtTearDown", false, true);

                            // Delete the build file now

                            //  This will prevent any chance of multiply publishing a package version

                            deleteBuildFile();

                            //---------------------------------------------------------------

                            //    Update Release Manager (publish to Release Manager)

                            //

                            mPhase.setPhase("Update Release Manager");

                            if ( mReporting.packageName != null )

                            {

                                // A dummy build file did not apply

                                // Publishing is done outside ant by design

                                //      Do ALL Release Manager work outside ant (and JATS)

                                boolean buildErrorOccurred = true;

                                boolean publishError = true;

                                try

                                {

                                    //  Create a build instance entry in the Release Manager database

                                    //        Create one per build

                                    //        mReporting.packageVersionId will have been updated if this is a new package-version

                                    //        Save the build id

                                    mPhase.setPhase("Create Build Instance");

                                    mReporting.buildId = mReleaseManager.createBuildInstance(

                                            mRtagId, 

                                            mReporting.packageVersionId,

                                            mReporting.buildReason,

                                            mReporting.buildRef);

                                    //

                                    //  Locate and collect the unit test results

                                    //

                                    mPhase.setPhase("Parse Unit Test Results");

                                    BuildTestResults btr = new BuildTestResults(System.getenv("GBE_DPKG"), mReporting, mLogger);

                                    mReleaseManager.insertTestResults(mReporting.buildId, btr);

                                    if ( mReporting.isaTestBuild )

                                    {

                                        //    Test Build

                                        //    Notify the user requesting the build

                                        //        Build Failure emails have already been sent

                                        //

                                        mLogger.info("run completing test build on " + mReporting.packageName + mReporting.packageExtension);

                                        mPhase.setPhase("Report Test Build Complete");

                                        completeTestBuild(mRippleEngine, mReporting.isFullyPublished);

                                        //

                                        //  Determine the total time taken by the build

                                        //      Done as late a possible in the process

                                        //      Does not include planning time

                                        //      Does not handle case where build cycle is broken

                                        //      Only record for successful tests. Don't want to record short build times on failure

                                        //      in particular if the package has been released. Would be OK for a WIP, but how to identify these

                                        if ( mReporting.isFullyPublished)

                                        {

                                            int buildDuration = mBuildDuration.toIntSecs();

                                            mLogger.warn("Build Duration:{}", buildDuration);

                                            mReleaseManager.updateBuildDuration(mReporting.packageVersionId, buildDuration);

                                        }

                                        mPhase.setPhase("Update Build Instance for Test");

                                        mReleaseManager.updateBuildInstance(mReporting.buildId, 0, mReporting.isFullyPublished ? BuildResult.Complete :BuildResult.BuildError );

                                        mPhase.setPhase("Insert BuildInfo for Test");

                                        insertBuildInfo(true);

                                        // ... and clean up the mess in the archive

                                        publishError = false;

                                        throw new Exception();                    

                                    }

                                    //    Package not fully published

                                    //    An error has occurred in the building of the package

                                    if ( !mReporting.isFullyPublished )

                                    {

                                        mLogger.info("run build error occurred on " + mReporting.packageName + mReporting.packageVersion);

                                        // ... and clean up the mess in the archive

                                        publishError = false;

                                        throw new Exception();

                                    }

                                    //---------------------------------------------------------------

                                    //  Package was successfully built and published into dpkg_archive by all daemons

                                    //    Update Release Manager - formally release the package

                                    //    Insert Package Metrics

                                    //    Insert Unit Test Results

                                    buildErrorOccurred = false;

                                    //

                                    //    Detect Badly formed VCS label returned by the builder

                                    //      Have had zero length values returned

                                    //      Treat as a publishing error

                                    //

                                    if ( mReporting.newVcsTag == null || mReporting.newVcsTag.isEmpty() )

                                    {

                                        mLogger.error("run package not labelled in Version Control on {}{}",mReporting.packageName, mReporting.packageVersion);

                                        mReporting.errMsg = "Error publishing to Version Control System";     // Max 50 Characters

                                        throw new Exception();

                                    }

                                    // Publish to release manager

                                    //    On error mReporting.errMsg will contain error string

                                    //    Publishing errors only affect the current package

                                    //

                                    mLogger.warn("run calling mReleaseManager.autoMakeRelease");

                                    mPhase.setPhase("autoMakeRelease");

                                    publishError =  mReleaseManager.autoMakeRelease(mReporting);

                                    if ( publishError )

                                    {

                                        mLogger.error("autoMakeRelease publishError: {}", mReporting.errMsg);

                                        throw new Exception();

                                    }

                                    //

                                    //  Determine the total time taken by the build

                                    //      Done as late a possible in the process

                                    //      Does not include planning time

                                    //      Does not handle case where build cycle is broken

                                    //

                                    int buildDuration = mBuildDuration.toIntSecs();

                                    mLogger.warn("Build Duration:{}", buildDuration);

                                    mReleaseManager.updateBuildDuration(mReporting.packageVersionId, buildDuration);

                                    //

                                    //    Update the Build Instance information

                                    //    If this has been a ripple then the pv_id of the build needs to be associated with

                                    //    the newly created version

                                    mPhase.setPhase("Update Build Instance");

                                    mReleaseManager.updateBuildInstance(mReporting.buildId, mReporting.packageVersionId, BuildResult.Complete );

                                    //

                                    //    Insert package metrics into Release Manager

                                    //        Metrics have been placed in a file in a format suitable for RM

                                    //

                                    mReporting.errMsg = "Error publishing Package Metrics";     // Max 50 Characters

                                    mPhase.setPhase("Insert Metrics");

                                    insertBuildMetrics();

                                    //  Insert BuildInfo

                                    //      New package PVID has been updated

                                    mReporting.errMsg = "Error inserting BuildInfo";            // Max 50 Characters

                                    mPhase.setPhase("Insert BuildInfo");

                                    insertBuildInfo(false);

                                    //

                                    //    Info reporting

                                    //    Send an email to track build system usage - may be noise

                                    //

                                    mReporting.errMsg = null;

                                    mLogger.warn("run calling emailBuildComplete");

                                    mPhase.setPhase("Post Build Email");

                                    emailPostBuild(mRippleEngine, false);

                                    //  Cleanup a few files that make have been created

                                    deleteMiscBuildFiles();

                                    //---------------------------------------------------------------

                                    //    All done

                                    //    The package has been built and released

                                    mPhase.setPhase("Build Complete");

                                }

                                catch( Exception e)

                                {

                                    // Some form of error has occurred. Types include

                                    //    Test build (Treated as an error so that dpkg_archive gets removed)

                                    //        buildErrorOccurred - True

                                    //        publishError - False    

                                    //    Build Error - Package was not fully published to dpkg_archive

                                    //        buildErrorOccurred - True

                                    //        publishError - False

                                    //     

                                    //    Badly formed VCS tag (Error Publishing to Version Control)

                                    //        buildErrorOccurred - False

                                    //        publishError - True    

                                    //    Error Publishing to Release Manager

                                    //        buildErrorOccurred - False

                                    //        publishError - True    

                                    //    Error inserting package metrics

                                    //        buildErrorOccurred - False

                                    //        publishError - True

                                    //    Error inserting unit test results

                                    //        buildErrorOccurred - False

                                    //        publishError - True

                                    // 

                                    // Uses

                                    //    buildErrorOccurred      - Package not in dpkg_archive as expected (or was a test build)

                                    //                              Email should have been sent to the user

                                    //    publishError            - Could not publish to Release Manager

                                    //                              Post build error. Email has not been sent to user

                                    //                              Takes precedence over buildErrorOccurred

                                    // Insert generic message if we don't have one

                                    //     Should only occur under unexpected conditions

                                    if (mReporting.errMsg == null || mReporting.errMsg.isEmpty())

                                    {

                                        mReporting.errMsg = "Error publishing to Release Manager";

                                        mReporting.errMsgDetail = mPhase.sText;

                                    }

                                    // Indicate some form of build error

                                    if ( ! mReporting.isaTestBuild )

                                    {

                                        mPhase.setPhase("Update Build Instance");

                                        mReleaseManager.updateBuildInstance(mReporting.buildId, 0, publishError ? BuildResult.SystemError :BuildResult.BuildError );

                                    }

                                    // 

                                    // Delete the dpkg_archive entry - the package is a dud

                                    //

                                    mPhase.setPhase("Delete package from archive");

                                    mLogger.warn("Delete package from archive");

                                    deletePackageVersion();

                                    //    A publishing error. The error occurred, after the build, in the RM publishing

                                    //        We won't have a log file, but we will have a rootCause

                                    //    Exclude package from the build

                                    //        Unless its a test build (handled within mReleaseManager.excludeFromBuild) 

                                    //

                                    if ( publishError  )

                                    {

                                        mLogger.error("an error occurred publishing to Version Control or Release Manager");

                                        //  Post Build Error

                                        //  eMails have not been sent by the ABT class as the error has occurred outside of that process

                                        mPhase.setPhase("Post Build Failure Email");

                                        emailPostBuild(mRippleEngine, true);

                                        mLogger.warn("run calling excludeFromBuild");

                                        mPhase.setPhase("Excluded from build");

                                        mReleaseManager.excludeFromBuild(false, 

                                                mReporting.packageVersionId, 

                                                null, 

                                                mReporting.rtagId, 

                                                null, 

                                                mReporting.errMsg, 

                                                null, 

                                                false, mReporting.isaTestBuild);

                                    }

                                    else if ( buildErrorOccurred )

                                    {

                                        //    Build Error Occurred

                                        //        Package has not been built on all configured platforms

                                        //        Under normal circumstances, this root cause will be masked by a previously reported error

                                        //        Under abnormal circumstances amidst build activity such as:

                                        //            - slave build machine reboots

                                        //            - slave build machines with a read only file system

                                        //        The rootCause is required to keep the end user informed

                                        //

                                        //    Use excludeFromBuild such that this call will not supersede existing entries

                                        //    In this manner the log file/cause provided here will only be used if we haven't

                                        //    already excluded this package from the build

                                        //    If we have a log file, then provide it to the user, else insert a generic message

                                        mReporting.errMsg = (mReporting.buildFailureLogFile == null) ? "Buildtool env error occurred. Attempt build again" : null;

                                        mLogger.warn("run calling excludeFromBuild"); 

                                        mPhase.setPhase("Excluded from build");

                                        mReleaseManager.excludeFromBuild(false, 

                                                mReporting.packageVersionId, 

                                                mReporting.packageVersion,

                                                mReporting.rtagId, 

                                                null, 

                                                mReporting.errMsg, 

                                                mReporting.buildFailureLogFile, 

                                                false, mReporting.isaTestBuild);

                                    }

                                    else

                                    {

                                        //  Not a buildError and not a publishing Error

                                        //      Don't know how we got here - I don't think we can

                                        //      Will cause an indefinite pause !

                                        throw new Exception("an error occurred publishing to Version Control or Release Manager");

                                    }

                                } // End Catch

                            }

                            if ( mUnitTest.compareTo("unit test coordinate slave threads") == 0 )            

                            {

                                throw new ExitException();

                            }

                            mLogger.info("run coordinate slave threads returned");

                        }

                        //---------------------------------------------------------------

                        //  Start of a new build cycle

                        //      Determine if the daemon has been paused.

                        //      Set RunLevel.IDLE (within allowedToProceed)

                        //

                        mLogger.warn("run calling allowedToProceed");

                        mPhase.setPhase("allowedToProceed");

                        mReleaseManager.discardVersion();

                        mReleaseManager.clearCurrentPackageBeingBuilt(mRconId);

                        deleteMiscBuildFiles();

                        allowedToProceed(true);

                        mLogger.info("run allowedToProceed returned");

                        if ( mUnitTest.compareTo("unit test allowed to proceed") == 0 )            

                        {

                            throw new ExitException();

                        }

                        //  Set up mReleaseConfigCollection

                        //  This will be used when creating a build file

                        mLogger.warn("Collect fresh MetaData");

                        mPhase.setPhase("collectMetaData");

                        mRippleEngine.collectMetaData();

                        //---------------------------------------------------------------

                        //  Plan the next build

                        //      Snap current Release Sequence Number

                        //      Determine what is to be built

                        //      Generate build files

                        mLogger.warn("run calling planRelease");

                        mPhase.setPhase("Plan Release", 600);

                        mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_PLANNING);

                        mReleaseSeqNum =  mReleaseManager.queryReleaseSeqNum(mRtagId, mRconId, BuildDaemon.mHostname); 

                        mRippleEngine.planRelease(!mLastBuildWasBenign);

                        // get the build file from the ripple engine

                        mPhase.setPhase("Get Build File", 10);

                        mMasterBuildFile = mRippleEngine.getFirstBuildFileContent();

                    }

                }

                //---------------------------------------------------------------

                //  Check that we are good to proceed with the build

                //      Check environment

                //      Ensure we have enough disk space

                //     

                mLogger.warn("run calling checkEnvironment"); 

                mPhase.setPhase("checkEnvironment");

                checkEnvironment();

                mLogger.info("run checkEnvironment returned");

                if ( mUnitTest.startsWith("unit test check environment") ) 

                {

                    throw new ExitException();

                }

                mLogger.info("run generated build files");

                // Start of a build cycle. Set new log file to capture entire build log

                mPhase.setPhase("Start Build Cycle");

                flagStartBuildCycle();

                mBuildDuration.reset();

                mReporting.resetData();

                mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_ACTIVE);

                mReleaseManager.setCurrentPackageBeingBuilt(mRconId, mMasterBuildFile.mPkgId, mMasterBuildFile.mPvId);

                // Save the buildfile

                mLogger.warn("Save buildfile");

                mPhase.setPhase("Save buildfile");

                saveBuildFile(mMasterBuildFile.content);

                // Quick Test to validate the provided Version Control tag/label

                mPhase.setPhase("Validate Version Tag");

                if ( deliverChange("AbtTestPath", true, false) )

                {

                    // Master will extract the package and determine the buildInfo before

                    // starting up slaves

                    //

                    mLogger.warn("Setup Package {}_{}", mReporting.packageName, mReporting.packageVersion);

                    mPhase.setPhase("Setup View for build");

                    deliverChange("AbtSetUp", true, false);

                    if ( !mMasterBuildFile.isaDummy() ) {

                        mPhase.setPhase("Generate buildInfo");

                        deliverChange("AbtBuildInfo", false, false);

                        processBuildInfo();

                    }

                    //---------------------------------------------------------------

                    //    Notify all slaves that there is work to be done

                    //      This is done by publishing a buildfile for them to consume

                    //      A NonGeneric build will be done with a real build file on all machines

                    //      Otherwise do a Dummy build on all Machines

                    //

                    mPhase.setPhase("Notify slaves of work", 10);

                    if ( mMasterBuildFile.state != BuildFileState.NonGeneric)

                    {

                        // publish a dummy build file for either generic or dummy cases

                        // this results in the slave not running ant

                        mLogger.warn("run calling publishBuildFile on dummy");

                        mMasterBuildFile.content = mDummyBuildFileContent;

                    }

                    else

                    {

                     // publish the build file

                        mLogger.warn("run calling publishBuildFile");    

                    }

                    mReleaseManager.publishBuildFile(mRtagId, mMasterBuildFile);                    

                    if ( mUnitTest.compareTo("unit test generate build files") == 0 )

                    {

                        throw new ExitException();

                    }

                    //---------------------------------------------------------------

                    //  Build the package

                    //      Build the package - deliver change to product baseline

                    //

                    mLogger.warn("Build Package {}_{}", mReporting.packageName, mReporting.packageVersion);

                    mLogger.info("run calling deliverChange - the actual build");

                    mPhase.setPhase("Build Package", 3 * 60 * 60);

                    deliverChange(null, true, false);

                    mLogger.info("run deliverChange returned");

                }

                mLastBuildWasBenign = false;

                if ( mMasterBuildFile.state == BuildFileState.Dummy)

                {

                    // no build requirement, so let things settle

                    mLogger.warn("run no build requirement, so let things settle");

                    mLastBuildWasBenign = true;

                    // If there are are no packages that are in error, then we can snapshot the

                    // Release as all packages will be built against each other

                    mReleaseManager.takeSnapshot( mRtagId );

                }        

            }

            //---------------------------------------------------------------

            //    Build loop exception handling (only)

            //

            catch( SQLException e )

            {

                //  Oracle connection issues

                //  Request a prolonged sleep at the start of the build loop

                mLogger.error("run oracle connection issues");

                mException = true;

            }

            catch( ExitException e )

            {

                mLogger.warn("run ExitException");

                exit = true;

            }

            catch( InterruptedException e )

            {

                mLogger.warn("run InterruptedException");

                Thread.currentThread().interrupt();

            }

            catch (BuildSystemException e) {

                // BuildSystem exception - a (possibly) correctable error

                // Flag as recoverable

                //    - Can't create XML build file

                //    - 

                // 

                mLogger.error("run BuildSystemException");

                mPhase.setPhase("BuildSystemException indefinitePause");

                notifyIndefinitePause(e.getMessage());

                mReleaseManager.indefinitePause(false);

                mException = true;

                mRecoverable = true;

            }

            catch( BuildToolException e)

            {

                // Buildtool exception - an uncorrectable error

                //    - Can't parse XML build file

                // 

                mLogger.error("run BuildToolException");

                mPhase.setPhase("BuildToolException indefinitePause");

                notifyIndefinitePause(e.getMessage());

                mReleaseManager.indefinitePause(false);

                mException = true;

            }

            catch( Exception e )

            {

                //  Uncaptured exception

                //  These are not good. Current mechanism to handle this condition is:

                //      1) Email build system administrator

                //      2) Pause the build system

                //  

                mLogger.error("run indefinitePause {}", e);

                // DEVI 51366 force sleep at beginning of while loop

                mException = true;

                String cause = e.getMessage();

                if ( cause == null || cause.compareTo("null") == 0 )

                {

                    cause = "Internal Error: " + e.toString();

                }

                try

                {

                    // notify first

                    // many reasons for indefinite pause, including database related, so do database last

                    mRecoverable = false;

                    if ( cause.compareTo(Package.mRecoverable) == 0 )

                    {

                        mRecoverable = true;

                    }

                    mPhase.setPhase("Notify indefinitePause");

                    notifyIndefinitePause(cause);

                    mReleaseManager.indefinitePause(mRecoverable);

                    mReleaseManager.updateBuildInstance(mReporting.buildId, 0, BuildResult.SystemError );

                }

                catch( Exception f )

                {

                    mLogger.error("run indefinitePause failed");

                }

            }

        }

        try {

            mReleaseManager.clearCurrentPackageBeingBuilt(mRconId);

        } catch (Exception e) {

            mLogger.error("Exception in clearCurrentPackageBeingBuilt: {}", e.getMessage());

        }

        mPhase.setPhase("Exit Thread");

    }

    /** Insert the build metrics into the Release Manager database

     *  Metrics have been generated by the build system ( master only )

     * @throws SQLException

     * @throws Exception

     */

    private void insertBuildMetrics() throws SQLException, Exception {

        try {

            mAbtMetricsFile = new File( String.valueOf(mRtagId) + "abtmetrics.txt");

            FileInputStream abtmetrics = new FileInputStream( mAbtMetricsFile);

            DataInputStream din = new DataInputStream( abtmetrics );

            InputStreamReader isr = new InputStreamReader( din );

            BufferedReader br = new BufferedReader( isr );

            String metrics = br.readLine();

            mLogger.warn( "execute read metrics string {}", metrics );

            br.close();

            isr.close();

            din.close();

            mLogger.warn("run calling mReleaseManager.insertPackageMetrics");                      

            mReleaseManager.insertPackageMetrics(mRtagId, mReporting.packageName, mReporting.packageExtension, metrics );

        } catch ( FileNotFoundException e) {

            mLogger.error("insertBuildMetrics: File not found {}", mAbtMetricsFile);

        } catch (IOException e) {

            mLogger.error("insertBuildMetrics: IOException {}", e);

        }

    }

    /** Process the generated buildInfo and determine which build machines need to be used

     *  in the current build.

     *  

     *  ***********************************************************************

     *  This work has been interrupted

     *  It is incomplete

     *  Need to determine the machines that we NEED to build the package on

     *  Only activate those machines, not all of the slaves

     *  This is only a first step. The next step is to allocate a slave from a pool of

     *  suitable machines.

     *  ***********************************************************************

     *  

     *  

     * @throws IOException 

     */

    private void processBuildInfo() throws IOException {

        mBuildInfoFile = new File (mRtagId + "abtBuildInfo.properties");

        if ( mBuildInfoFile.exists() )

        {

            BuildInfo bi = new BuildInfo(mBuildInfoFile);

            mLogger.warn("Can build for: {}", bi.mPlatforms);

            mLogger.warn("Built for: {}", bi.mBuildPlatforms);

        }

        else

        {

            mLogger.error("No BuildInfo file. {}", mBuildInfoFile);

            mErrorReported = true;

        }

    }

    /**

     * Locate the <rtagId>abtBuildInfo.properties file and insert data into Release Manager

     * Insert mPlatforms and mBuildPlatforms

     * We can only do this after a new package version has been created.

     * @param bIsaTestBuild - True. Is a test build

     * @throws Exception 

     */

    private void insertBuildInfo(boolean bIsaTestBuild) throws Exception {

        mBuildInfoFile = new File (mRtagId + "abtBuildInfo.properties");

        if ( mBuildInfoFile.exists() )

        {

            BuildInfo bi = new BuildInfo(mBuildInfoFile);

            mLogger.warn("Can build for: {}", bi.mPlatforms);

            mLogger.warn("Built for: {}", bi.mBuildPlatforms);

            mReleaseManager.insertBuildInfo(bIsaTestBuild, mReporting, bi);

        }

        else

        {

            mLogger.error("No BuildInfo file. {}", mBuildInfoFile);

        }

    }

    /**   Delete a specified version of a package from dpkg_archive

     *    Used during error processing to cleanup packages that did not build correctly

     *    

     *    The package-version is in dpkg_archive. This may be on a remote server

     *    Execute the deletion command on the remote server as it will be a lot faster

     *    for large packages.

     *    

     *    Ensure that the file system cache is cleared before we attempt to look

     *    for the directory

     */

    private void deletePackageVersion()

    {

        //    Setup a Remote Execution environment

        //    The logging interface is is hard part

        RemoteExecution mRemoteExecution = new RemoteExecution(new LogOutput(){

            @Override

            public void data(String message) {

                mLogger.info(message);

            }

            @Override

            public void fatal(String message) {

                mLogger.error(message);

            }

            @Override

            public void info(String message) {

                mLogger.info(message);

            }});

        //    Execute the remote command

        //    This is a best effort deletion - can't really handle conditions

        //    where the command fails.

        CommandBuilder cmd = new CommandBuilder();

        cmd.add("sudo","-n", "-u", "pkgadm", "/home/releasem/sbin/jatsTool", "assemble_dpkg");

        cmd.add("-DeleteVersion");

        cmd.add(!ReleaseManager.getUseMutex(), "-testArchive");

        cmd.add("'-pname=" + mReporting.packageName + "'");

        cmd.add("'-pversion=" + mReporting.packageVersion + "'");

        mLogger.warn("run remote command to delete {}, {}", mReporting.packageName, mReporting.packageVersion);

        int rv = mRemoteExecution.execute(cmd.toString());

        mLogger.info("run remote command to delete. Returned {}", rv);

        //    Belt and Braces

        //    Ensure that the package-version has been deleted, 

        //    and if not then try to delete it the slow way.

        //

        //    Note: Need to flush NFS cache before we check existence 

        String dpkgArchiveEntry = utilities.catDir(Package.mGbeDpkg, mReporting.packageName, mReporting.packageVersion);

        File dpkgArchiveEntryFile = utilities.freshFile(dpkgArchiveEntry);

        mLogger.debug("run checking existence of {}",dpkgArchiveEntry);

        if ( dpkgArchiveEntryFile.exists() )

        {

            if ( dpkgArchiveEntryFile.isDirectory() )

            {

                mLogger.warn("run calling deleteDirectory on {}", dpkgArchiveEntryFile);                      

                deleteDirectory(dpkgArchiveEntryFile);

            }

        }

    }

    /**   Sends email notification and mark the successful completion of a build

     *    Used simply to keep an email trail of build operations

     */

    public void emailPostBuild( RippleEngine rippleEngine, boolean isaError )

    {

        mLogger.debug("emailPostBuild");

        String subject;

        String emailTo;

        String mailBody = "";

        if (isaError)

        {

            subject = "BUILD FAILURE on package " + mReporting.packageName + " " + mReporting.packageVersion;

            emailTo = mReporting.packageOwners;

            mailBody="An error occured within the build system after the package had been built<p><hr>";

        }

        else

        {

            subject = "BUILD SUCCESS on package " + mReporting.packageName + " " + mReporting.packageVersion;

            emailTo = rippleEngine.getMailSender();

        }

        //  Cleanup the dependency string

        //      Expecting: 'junit','3.7.0.cots';'ant-ejbdoclet','1.0000.cr'

        String indent = "<br>&nbsp;&nbsp;&nbsp;&nbsp;";

        String depends = mReporting.packageDepends.replace("','", " ");

        depends = depends.replace("';'",indent);

        depends = depends.replace("'","");

        //  Cleanup the build standard string

        //      Expecting win32: jats all;linux_i386: jats prod

        String buildStandards = mReporting.packageBuildInfo;

        buildStandards = buildStandards.replace(";",indent);

        mailBody += "Release      : " + rippleEngine.mBaselineName + "<br>" +

                "Package      : " + mReporting.packageName + "<br>" +

                "Version      : " + mReporting.packageVersion + "<br>";

        if (isaError)

        {

            mailBody += "Cause        : " + mReporting.errMsg + "<br>";

            if (mReporting.errMsgDetail != null && mReporting.errMsgDetail.length() > 0)

            {

                mailBody += "Detail       : " + mReporting.errMsgDetail + "<br>";

            }

        }

        mailBody += "RM Link      : " + CreateUrls.generateRmUrl(mRtagId,mReporting.packageVersionId) + "<br>" +

                "PVID         : " + mReporting.packageVersionId + "<br>" +

                "VcsTag       : " + mReporting.newVcsTag + "<br>" +

                "Ripple       : " + (mReporting.isRipple ? "Yes" : "No") + "<br>" +

                "Dependencies : " + indent + depends + "<br>" +

                "Build Standards : " + indent + buildStandards;

        mailBody += "<p><hr>";

        try

        {

            Smtpsend.send( rippleEngine.getMailServer(),    // mailServer

                    rippleEngine.getMailSender(),           // source

                    emailTo,                                // target - send to package owners, or myself

                    rippleEngine.getMailSender(),           // cc

                    null,                                   // bcc

                    subject,                                // subject

                    mailBody,                               // body

                    null                                    // attachment

                    );

        }

        catch( Exception e )

        {

            mLogger.warn("Email Failure: emailPostBuild:" + e.getMessage());

        }

        mLogger.debug("emailPostBuild. Returning");

    }

    /**

     * End of Processing for a Test Build

     *     - sends email notification

     *     - marks the instruction complete in the database

     */

    public void completeTestBuild(RippleEngine rippleEngine, boolean success) throws SQLException, Exception

    {

        mLogger.debug("completeTestBuild");

        if (!mReporting.isaTestBuild)

        {

            mLogger.info("completeTestBuild. Not Build Instruction");

            return;

        }

        //  Email Subject

        String subject = (success ? "TEST BUILD COMPLETED SUCCESSFULLY" : "TEST BUILD FAILED") 

                + " on package "

                + mReporting.packageName +  mReporting.packageExtension;

        //  Cleanup the dependency string

        //      Expecting: 'junit','3.7.0.cots';'ant-ejbdoclet','1.0000.cr'

        String indent = "<br>&nbsp;&nbsp;&nbsp;&nbsp;";

        String depends = mReporting.packageDepends.replace("','", " ");

        depends = depends.replace("';'",indent);

        depends = depends.replace("'","");