Rev 4280 | Blame | Last modification | View Log | RSS feed
package com.erggroup.buildtool.daemon;import com.erggroup.buildtool.daemon.BuildThread;import com.erggroup.buildtool.ripple.MutableString;import com.erggroup.buildtool.ripple.Package;import com.erggroup.buildtool.ripple.ReleaseConfig;import com.erggroup.buildtool.ripple.ReleaseManager;import com.erggroup.buildtool.ripple.RippleEngine;import com.erggroup.buildtool.ripple.RunLevelData;import com.erggroup.buildtool.smtp.Smtpsend;import java.io.BufferedReader;import java.io.DataInputStream;import java.io.File;import java.io.FileInputStream;import java.io.InputStreamReader;import java.sql.SQLException;import java.util.Iterator;import java.lang.System;import org.apache.log4j.Logger;/**Master Thread sub component*/public class MasterThreadextends BuildThread{/**Logger* @attribute*/private static final Logger mLogger = Logger.getLogger(MasterThread.class);/**Nagios Monitoring* @attribute*/protected long mPhaseStartTime = 0;/**constructor*/public MasterThread(int rtag_id, int rcon_id, String unitTest){mLogger.warn("MasterThread rtag_id " + rtag_id + " rcon_id " + rcon_id);mRtagId = rtag_id;mRconId = rcon_id;if ( unitTest == null ){unitTest = new String();}mUnitTest = unitTest;}/**implements the sequence diagrams coordinate slave threads, generate build files, allowed to proceed, check environment*/public void run(){Integer id = new Integer(mRtagId);setName(id.toString());mLogger.warn("Master Thread run");boolean exit = false;RippleEngine rippleEngine = new RippleEngine(mReleaseManager, mRtagId, true);MutableString addendum = new MutableString();while(!exit){try{mPhaseStartTime = 0;mLogger.fatal("run calling sleepCheck");sleepCheck();mLogger.fatal("run calling rippleEngine.collectMetaData");mPhaseStartTime = System.currentTimeMillis();rippleEngine.collectMetaData();mPhaseStartTime = 0;if ( Thread.currentThread().isInterrupted() ){mLogger.warn("run is interrupted");// unit test techniquethrow new ExitException();}if ( mUnitTest.compareTo("unit test spawn thread") == 0){throw new Exception();}MutableString buildFileContent = new MutableString();if ( mUnitTest.compareTo("unit test check environment") != 0){if ( mUnitTest.compareTo("unit test generate build files") != 0){if ((mUnitTest.compareTo("unit test allowed to proceed") != 0) &&(mUnitTest.compareTo("unit test not allowed to proceed") != 0) &&(mUnitTest.compareTo("unit test exit") != 0)){//---------------------------------------------------------------// Wait for all Slaves to enter waiting or paused state// They may be building// They may be starting up//mLogger.warn("run coordinate slave threads");mRunLevel = RunLevel.WAITING;mLogger.warn("run changing run level to WAITING for rcon_id " + mRconId);mLogger.fatal("run calling mRunLevel.persistNew to set WAITING");mRunLevel.persistNew(mReleaseManager, mRconId);boolean allSlaveThreadsWaiting = false;boolean logWarning = true;boolean logWarning2 = true;while ( !allSlaveThreadsWaiting ){mLogger.fatal("run calling mReleaseManager.queryRunLevel");mReleaseManager.queryRunLevel(mRtagId);// Process all run entriesallSlaveThreadsWaiting = true;for (Iterator<RunLevelData> it = mReleaseManager.mRunLevelCollection.iterator(); it.hasNext(); ){RunLevelData rl = it.next();int runLevel = rl.get_current_run_level();// DEVI 53065 Slave indication that they are done with a package includes// having a paused run levelif ((runLevel != ReleaseManager.DB_WAITING) &&(runLevel != ReleaseManager.DB_PAUSED)){if ( logWarning ){mLogger.warn("run waiting for rcon id " + rl.get_rcon_id());logWarning = false;}else{mLogger.info("run waiting for rcon id " + rl.get_rcon_id());}allSlaveThreadsWaiting = false;break;}}if ( !allSlaveThreadsWaiting ){// to do, sleep for periodicMsif ( mUnitTest.compareTo("unit test coordinate slave threads") == 0 ){Thread.currentThread().interrupt();}else{if ( logWarning2 ){mLogger.warn("run sleeping 3 secs waiting for slave threads");logWarning2 = false;}else{mLogger.info("run sleeping 3 secs waiting for slave threads");}mLogger.fatal("run calling Thread.sleep for 3 secs");Thread.sleep(3000);mLogger.info("run sleep returned");}}}if ( mUnitTest.compareTo("unit test coordinate slave threads") == 0 ){throw new ExitException();}//---------------------------------------------------------------// Publish Package Build Results// Indicate to RM that we are publishing// Ensure that all build machines have placed placed marker in dpkg_archive// Save updated build files into VCS//mRunLevel = RunLevel.PUBLISHING;mLogger.warn("run changing run level to PUBLISHING for rcon_id " + mRconId);mLogger.fatal("run calling mRunLevel.persist to set PUBLISHING");mRunLevel.persist(mReleaseManager, mRconId);mLogger.fatal("run calling deliverChange on AbtPublish");deliverChange(null, "AbtPublish", true);// preserve, as they are reset by deliverChangeString reportingFullyPublished = mReportingFullyPublished;String reportingNewVcsTag = mReportingNewVcsTag;//---------------------------------------------------------------// TearDown the build workspace// Remove the build workspace// Save build log files//mLogger.fatal("run calling deliverChange on AbtTearDown");deliverChange(null, "AbtTearDown", true);// DEVI 47395 delete the build file now// this will prevent any chance of multiply publishing a package versionFile buildFile = new File(mRtagId + "build.xml");if ( buildFile.exists() ){buildFile.delete();}if ( mReportingPackageName != null ){// a dummy build file did not apply// Publishing is done outside ant by design// The preference is to do all release manager work outside antboolean buildErrorOccurred = true;boolean publishError = false;// max 50 charactersString rootCause = "Error publishing to Release Manager";try{if ( mReportingTestBuild.compareTo("0") != 0 ){mLogger.info("run completing test build on " + mReportingPackageName + mReportingPackageExtension);// build failure emails will have been sent// just left to notify the user of the test build completionPackage p = rippleEngine.findPackage(mReportingPackageName + mReportingPackageExtension);if (p != ReleaseManager.NULL_PACKAGE){boolean success = false;if ( reportingFullyPublished != null && reportingFullyPublished.compareTo("yes") == 0 ){success = true;}mLogger.fatal("run calling p.completeTestBuild");p.completeTestBuild(rippleEngine.mMailServer, rippleEngine.mMailSender, mReleaseManager, rippleEngine.mBaselineName, success);}// ... and clean up the mess in the archivethrow new Exception();}if ( reportingFullyPublished != null && reportingFullyPublished.compareTo("yes") == 0 ){buildErrorOccurred = false;if ( reportingNewVcsTag.length() > 0 ){Integer rtagId = new Integer(mRtagId);if ( mGbeGatherMetricsOnly == null ){// publish to release manager// On error mAutoMakeReleaseCause will contain error stringmLogger.fatal("run calling mReleaseManager.autoMakeRelease");publishError = mReleaseManager.autoMakeRelease(rtagId.toString(),mReportingPackageName,mReportingPackageExtension,mReportingPackageVersion,reportingNewVcsTag,mReportingPackageDepends,mReportingIsRipple);// publishErrors only affect one packageif ( publishError ){rootCause = mReleaseManager.mAutoMakeReleaseCause;mLogger.fatal("autoMakeRelease publishError: " + rootCause);throw new Exception();}}FileInputStream abtmetrics = new FileInputStream( rtagId.toString() + "abtmetrics.txt" );DataInputStream din = new DataInputStream( abtmetrics );InputStreamReader isr = new InputStreamReader( din );BufferedReader br = new BufferedReader( isr );String metrics = br.readLine();mLogger.warn( "execute read metrics string " + metrics );br.close();isr.close();din.close();mLogger.fatal("run calling mReleaseManager.insertPackageMetrics");mReleaseManager.insertPackageMetrics(rtagId.toString(), mReportingPackageName, mReportingPackageExtension, metrics );//// Info reportinging// Send an email to track build system usage - may be noise//mLogger.fatal("run calling emailBuildComplete");emailBuildComplete(rippleEngine, reportingNewVcsTag);// All done// The package has been built and released}else{mLogger.info("run package not labelled in Version Control on " + mReportingPackageName + mReportingPackageVersion);// max 50 charactersrootCause = "Error publishing to Version Control System";throw new Exception();}}else{mLogger.info("run build error occurred on " + mReportingPackageName + mReportingPackageVersion);throw new Exception();}}catch( Exception e){// a build error occurred or// an error occurred publishing to Version Control or Release Manager// take out the archive entry firstString fs = System.getProperty( "file.separator" );String dpkgArchiveEntry = new String(Package.mGbeDpkg + fs + mReportingPackageName + fs + mReportingPackageVersion);File dpkgArchiveEntryFile = new File(dpkgArchiveEntry);mLogger.info("run checking existence of " + dpkgArchiveEntry);if ( dpkgArchiveEntryFile.exists() ){if ( dpkgArchiveEntryFile.isDirectory() ){mLogger.warn("run deleting " + dpkgArchiveEntryFile.getName());mLogger.fatal("run calling deleteDirectory on " + dpkgArchiveEntryFile);deleteDirectory(dpkgArchiveEntryFile);}}Integer rtagId = new Integer(mRtagId);if ( ! buildErrorOccurred ){mLogger.fatal("an error occurred publishing to Version Control or Release Manager");// force exclusion by defaultint testBuildInstruction = 0;try{testBuildInstruction = Integer.parseInt( mReportingTestBuild );}catch( NumberFormatException nfe ){}mLogger.fatal("run calling excludeFromBuild");mReleaseManager.excludeFromBuild(mReportingPackageVersionId, null, rtagId.toString(), null, rootCause, null, false, testBuildInstruction);if ( ! publishError ){throw new Exception("an error occurred publishing to Version Control or Release Manager");}}// DEVI 55364// Package has not been built on all configured platforms// Under normal circumstances, this root cause will be masked by a previously reported error// Under abnormal circumstances amidst build activity such as:// - slave build machine reboots// - slave build machines with a read only file system// this root cause is required to keep the end user informed// max 50 charsString cause = "Buildtool env error occurred. Attempt build again";// force exclusion by defaultint testBuildInstruction = 0;try{testBuildInstruction = Integer.parseInt( mReportingTestBuild );}catch( NumberFormatException nfe ){}mLogger.fatal("run calling excludeFromBuild");mReleaseManager.excludeFromBuild(mReportingPackageVersionId, mReportingPackageVersion, rtagId.toString(), null, mReportingBuildFailureLogFile == null ? cause : null, mReportingBuildFailureLogFile, false, testBuildInstruction);}}if ( mUnitTest.compareTo("unit test coordinate slave threads") == 0 ){throw new ExitException();}mLogger.info("run coordinate slave threads returned");}//---------------------------------------------------------------// Start of a new build cycle// Determine if the daemon has been paused.//mLogger.warn("run checking allowedToProceed");mLogger.fatal("run calling allowedToProceed");allowedToProceed(true);mLogger.info("run allowedToProceed returned");if ( mUnitTest.compareTo("unit test allowed to proceed") == 0 ){throw new ExitException();}// Set up Release Config in mReleaseConfigCollection// This will be used when creating a build filemLogger.fatal("run calling queryReleaseConfig");mReleaseManager.queryReleaseConfig(mRtagId);//---------------------------------------------------------------// Plan the next build// Determine what is to be built// Generate build filesmLogger.warn("run generating build files");mLogger.fatal("run calling planRelease");mPhaseStartTime = System.currentTimeMillis();rippleEngine.planRelease();mPhaseStartTime = 0;mPhaseStartTime = System.currentTimeMillis();mLogger.fatal("run calling reportChange");rippleEngine.reportChange();mPhaseStartTime = 0;// get the build file from the ripple enginerippleEngine.getFirstBuildFileContent(buildFileContent, addendum);if ( addendum.value.compareTo("non generic") == 0 ){// publish the build filemLogger.warn("run publishing build file");mLogger.fatal("run calling publishBuildFile");mReleaseManager.publishBuildFile(mRtagId, buildFileContent.value);}else{// publish a dummy build file for either generic or dummy cases// this results in the slave not running antmLogger.warn("run publishing dummy build file");mLogger.fatal("run calling publishBuildFile on dummy");mReleaseManager.publishBuildFile(mRtagId, mDummyBuildFileContent);}}else{// UTF: Set up Release Config in mReleaseConfigCollectionmLogger.fatal("run calling queryReleaseConfig for UTF");mReleaseManager.queryReleaseConfig(mRtagId);}mLogger.info("run generated build files");// Start of a build cycle. Set new log file to capture entire build logflagStartBuildCycle();//---------------------------------------------------------------// SetUp BuildSpace// Extract source from version control//if ( mGbeGatherMetricsOnly != null ){// set view up early for metrics gathering to ensure build file is fully written before letting the slave loosemLogger.fatal("run calling setViewUp");setViewUp(buildFileContent.value, true);}//---------------------------------------------------------------// Notify all slaves that there is work to be done// Change the run level for all threads in associated with the baseline//mLogger.warn("run change the run level for all threads in associated with the baseline");mRunLevel = RunLevel.ACTIVE;for (Iterator<ReleaseConfig> it = mReleaseManager.mReleaseConfigCollection.iterator(); it.hasNext(); ){ReleaseConfig rc = it.next();mLogger.warn("run changing run level to ACTIVE for rcon_id " + rc.get_rcon_id() );mLogger.fatal("run calling mRunLevel.persist to set ACTIVE");mRunLevel.persist(mReleaseManager, rc.get_rcon_id());}if ( mUnitTest.compareTo("unit test generate build files") == 0 ){throw new ExitException();}}mLogger.info("run changed run levels");//---------------------------------------------------------------// Check that we are good to proceed with the build// Check environment// Ensure we have enough disk space//mLogger.warn("run checkEnvironment");mLogger.fatal("run calling checkEnvironment");checkEnvironment();mLogger.info("run checkEnvironment returned");if ( mUnitTest.compareTo("unit test check environment") == 0 ){throw new ExitException();}//---------------------------------------------------------------// Build the package// Setup the workspace// Build the package - deliver change to product baseline//mLogger.warn("run deliverChange");if ( mGbeGatherMetricsOnly == null ){mLogger.fatal("run calling setViewUp metrics only");setViewUp(buildFileContent.value, true);}mLogger.fatal("run calling deliverChange - the actual build " + mReportingPackageName + "_" + mReportingPackageVersion);deliverChange(null, null, true);mLogger.info("run deliverChange returned");mSleep = false;if ( addendum.value.compareTo("dummy") == 0 ){// no build requirement, so let things settlemLogger.warn("run no build requirement, so let things settle");mSleep = true;}}//---------------------------------------------------------------// Build loop exception handling (only)//catch( SQLException e ){// oracle connection issuesmLogger.warn("run oracle connection issues");mException = true;}catch( ExitException e ){mLogger.warn("run ExitException");exit = true;}catch( InterruptedException e ){mLogger.warn("run InterruptedException");}catch( Exception e ){mLogger.error("run indefinitePause " + e.toString());String cause = e.getMessage();if ( cause == null ){cause = e.toString();}try{// notify first// many reasons for indefinite pause, including database related, so do database lastmRecoverable = false;if ( cause.compareTo(Package.mRecoverable) == 0 ){mRecoverable = true;}indefinitePause(rippleEngine, cause);mReleaseManager.indefinitePause();// DEVI 51366 force sleep at beginning of while loopmException = true;}catch( Exception f ){mLogger.error("run indefinitePause failed");}}}}/** Sends email notification and mark the successful completion of a build* Used simply to keep an email trail of build operations* Send only the the buildadm group** Uses globals for data*/public void emailBuildComplete( RippleEngine rippleEngine, String reportingNewVcsTag ){mLogger.debug("emailBuildComplete");String subject = "BUILD SUCCESS on package " + mReportingPackageName + " " + mReportingPackageVersion;String mailBody="Release : " + rippleEngine.mBaselineName + "<br>" +"Package : " + mReportingPackageName + "<br>" +"Version : " + mReportingPackageVersion + "<br>" +"PVID : " + mReportingPackageVersionId + "<br>" +"VcsTag : " + reportingNewVcsTag + "<br>" +"Ripple : " + mReportingIsRipple + "<br>" +"Dependencies : " + mReportingPackageDepends + "<br>";mailBody += "<p><hr>";try{Smtpsend.send( rippleEngine.mMailServer, // mailServerrippleEngine.mMailSender, // sourcerippleEngine.mMailSender, // target - send to myselfnull, // ccnull, // bccsubject, // subjectmailBody, // bodynull // attachment);}catch( Exception e ){mLogger.warn("Email Failure: emailBuildComplete:" + e.getMessage());}mLogger.debug("emailBuildComplete. Returning");}/**returns 'M'*/protected char getMode(){mLogger.debug("getMode");return 'M';}/*** Nagios interface extension* This method should be overriden by classes that extend this class* If not overriden then the test indicates OK** Returns true if the thread looks OK*/boolean checkThreadExtended(){boolean retVal = true;mLogger.info("Master checkThreadExtended");if ( mPhaseStartTime > 0 ){/*** Report if we appear to be stuck in the current phase** At the moment the phases that we monitor should complete** within 5 minutes and 10 minutes is real error.** We do need to allow time for interlocking with other tasks.*/if ( ((System.currentTimeMillis() - mPhaseStartTime) / 1000) > (10 * 60) ){mLogger.info("Master checkThreadExtended: " + ((System.currentTimeMillis() - mPhaseStartTime) / 1000));retVal = false;}}return retVal;}}