Subversion Repositories DevTools

package com.erggroup.buildtool.daemon;

import com.erggroup.buildtool.daemon.ResumeTimerTask;

import com.erggroup.buildtool.ripple.ReleaseManager;

import com.erggroup.buildtool.smtp.Smtpsend;

import com.erggroup.buildtool.ripple.RippleEngine;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.FileWriter;

import java.io.FilenameFilter;

import java.io.IOException;

import java.io.PrintStream;

import java.io.StringReader;

import java.sql.SQLException;

import java.util.Date;

import java.util.Timer;

import org.apache.log4j.Logger;

import org.apache.tools.ant.BuildException;

import org.apache.tools.ant.DefaultLogger;

import org.apache.tools.ant.Project;

import org.apache.tools.ant.ProjectHelper;

/**Build Thread sub component

 */

public abstract class BuildThread

  extends Thread

{

  /**baseline identifier (which release manager release this BuildThread is dealing with)

   * @attribute

   */

  protected int mRtagId = 0;

  /**unique identifier of this BuildThread

   * @attribute

   */

  protected int mRconId = 0;

  /**

   * @aggregation composite

   */

  protected RunLevel mRunLevel;

  /**

   * @aggregation composite

   */

  protected ReleaseManager mReleaseManager;

  /**unit test support

   * @attribute

   */

  protected String mUnitTest = "";

  /**

   * @aggregation composite

   */

  protected static ResumeTimerTask mResumeTimerTask;

  /**

   * @aggregation composite

   */

  private static Timer mTimer;

  /**Synchroniser object

   * Use to Synchronize on by multiple threads

   * @attribute

   */

  static final Object mSynchroniser = new Object();

  /**BuildThread group

   * @attribute

   */

  public static final ThreadGroup mThreadGroup = new ThreadGroup("BuildThread");

  /**the advertised build file content when either

   * a) no package in the baseline has a build requirement

   * b) the next package in the baseline with a build requirement is generic

   * @attribute

   */

  protected static final String mDummyBuildFileContent = new String("<dummy/>");

  /**Set true when last build cycle was benign.

   * @attribute

   */

  protected boolean mSleep;

  /**Set true when last build cycle caught SQLException or Exception.

   * @attribute

   */

  protected boolean mException;

  /**Set true when ant error reported on any target.

   * @attribute

   */

  private boolean mErrorReported;

  /**Logger

   * @attribute

   */

  private static final Logger mLogger = Logger.getLogger(BuildThread.class);

  /**Package name for reporting purposes.

   * @attribute

   */

  protected String mReportingPackageName;

  /**Package version for reporting purposes.

   * @attribute

   */

  protected String mReportingPackageVersion;

  /**Package extension for reporting purposes.

   * @attribute

   */

  protected String mReportingPackageExtension;

  /**Package location for reporting purposes.

   * @attribute

   */

  protected String mReportingPackageLocation;

  /**Package dependencies for reporting purposes.

   * @attribute

   */

  protected String mReportingPackageDepends;

  /**Is ripple flag for reporting purposes.

   * @attribute

   */

  protected String mReportingIsRipple;

  /**Package version identifier for reporting purposes.

   * @attribute

   */

  protected String mReportingPackageVersionId;

  /**Fully published flag for reporting purposes.

   * @attribute

   */

  protected String mReportingFullyPublished;

  /**New label for reporting purposes.

   * @attribute

   */

  protected String mReportingNewVcsTag;

  /**Source control interaction for reporting purposes.

   * @attribute

   */

  protected String mReportingDoesNotRequireSourceControlInteraction;

  /**Source control interaction for reporting purposes.

   * @attribute

   */

  protected String mReportingTestBuild;

  /**Log file location for reporting purposes.

   * @attribute

   */

  protected String mReportingBuildFailureLogFile;

  /**Non null determines to only gather metrics

   * @attribute

   */

  protected static final String mGbeGatherMetricsOnly = System.getenv("GBE_GATHER_METRICS");

  /**Non null determines to only gather metrics

   * @attribute

   */

  protected boolean mRecoverable;

  /**Logger for the entire build process

  * @attribute

  */

  DefaultLogger mBuildLogger ;

  /**constructor

   */

  BuildThread()

  {

    super(mThreadGroup, "");

    mLogger.debug("BuildThread");

    mSleep = false;

    mException = false;

    mErrorReported = false;

    mReleaseManager = new ReleaseManager();

    mRecoverable = false;

    // no need to be synchronized - BuildThreads are instantiated in a single thread

    if ( mResumeTimerTask == null )

    {

      mResumeTimerTask = new ResumeTimerTask();

      mTimer = new Timer();

      mResumeTimerTask.setTimer(mTimer);

    }

  }

  /**Flags that a new build cycle is about to start

  *  Create a new logger to capture all the complete build log

  *  even though its done in stages we want a complete log.

  */

  void flagStartBuildCycle()

  {

    try

    {

      mBuildLogger = new DefaultLogger();

      PrintStream ps = new PrintStream(mRtagId + ".log");

      mBuildLogger.setOutputPrintStream(ps);

      mBuildLogger.setMessageOutputLevel(Project.MSG_INFO);

    }

    catch( FileNotFoundException e )

    {

      mLogger.error("BuildThread caught FileNotFoundException");

    }

  }

  /**sleeps when mException is set

   */

  protected void sleepCheck()

  {

    if (mException)

    {

      try

      {

        Integer sleepTime = 300000;

        mLogger.warn("sleepCheck sleep " + sleepTime.toString() + " secs");

        Thread.sleep(sleepTime);

        mLogger.info("sleepCheck sleep returned");

      }

      catch(InterruptedException e)

      {

        mLogger.warn("sleepCheck sleep caught InterruptedException");

      }

    }

    mException = false;

  }

  /**initially changes the run level to IDLE

   * determines if the BuildThread is still configured

   * a) determines if the BuildThread is running in scheduled downtime

   * b) determines if the BuildThread is directed to pause

   * changes the run level to PAUSED if a) or b) are true

   * throws ExitException when not configured

   * implements the sequence diagrams allowed to proceed, not allowed to proceed, exit

   */

  protected void allowedToProceed(boolean master) throws ExitException, SQLException, Exception

  {

    mLogger.debug("allowedToProceed");

    try

    {

      mRunLevel = RunLevel.IDLE;

      mLogger.warn("allowedToProceed changing run level to IDLE for rcon_id " + mRconId);

      mLogger.fatal("allowedToProceed calling mRunLevel.persist on IDLE");                      

      mRunLevel.persist(mReleaseManager, mRconId);

      if ( master )

      {

        mLogger.fatal("allowedToProceed calling mReleaseManager.discardVersion");                      

        mReleaseManager.discardVersion();

      }

      mLogger.fatal("allowedToProceed calling mReleaseManager.clearCurrentPackageBeingBuilt");                      

      mReleaseManager.clearCurrentPackageBeingBuilt(mRconId);      

    }

    catch(SQLException e)

    {

      mLogger.warn("allowedToProceed caught SQLException");

    }

    if (mSleep)

    {

      try

      {

        Integer sleepTime = 300000;

        if ( !master )

        {

          // sleep only 3 secs on slave

          sleepTime = 3000;

        }

        mLogger.warn("allowedToProceed sleep " + sleepTime.toString() + " secs no build requirement");

        mLogger.fatal("allowedToProceed calling Thread.sleep for 3 secs");                      

        Thread.sleep(sleepTime);

        mLogger.info("allowedToProceed sleep returned");

      }

      catch(InterruptedException e)

      {

        mLogger.warn("allowedToProceed sleep caught InterruptedException");

      }

    }

    boolean proceed = false;

    try

    {

      while ( !proceed )

      {

        mLogger.fatal("allowedToProceed calling mReleaseManager.connect");                      

        mReleaseManager.connect();

        mLogger.fatal("allowedToProceed calling mReleaseManager.queryReleaseConfig");                      

        if ( !mReleaseManager.queryReleaseConfig(mRtagId, mRconId, BuildDaemon.mHostname, getMode()) )

        {

          mReleaseManager.disconnect();

          mLogger.warn("allowedToProceed queryReleaseConfig failed");

          throw new ExitException();

        }

        Date resumeTime = new Date( 0 );

        mLogger.fatal("allowedToProceed calling mReleaseManager.queryRunLevelSchedule");                      

        if ( !mReleaseManager.queryRunLevelSchedule(resumeTime, mRecoverable) )

        {

          mLogger.info("allowedToProceed scheduled downtime");

          mReleaseManager.disconnect();

          mRunLevel = RunLevel.PAUSED;

          mLogger.warn("allowedToProceed changing run level to PAUSED for rcon_id " + mRconId);

          mRunLevel.persist(mReleaseManager, mRconId);

          synchronized(mSynchroniser)

          {

            // contain the schedule and wait in the same synchronized block to prevent a deadlock

            // eg this thread calls schedule, timer thread calls notifyall, this thread calls wait (forever)

            try

            {

              if (mResumeTimerTask.isCancelled())

              {

                mResumeTimerTask = new ResumeTimerTask();

                mTimer = new Timer();

                mResumeTimerTask.setTimer(mTimer);

              }

              mLogger.warn("allowedToProceed schedule passed " + resumeTime.getTime());

              mTimer.schedule(mResumeTimerTask, resumeTime);

            }

            catch( IllegalStateException e )

            {

              // this may be thrown by schedule if already scheduled

              // it signifies another BuildThread has already scheduled the ResumeTimerTask

               mLogger.warn("allowedToProceed already scheduled");

            }

            try

            {

              mLogger.warn("allowedToProceed wait");

              mSynchroniser.wait();

              mLogger.warn("allowedToProceed wait returned");

              if ( mUnitTest.compareTo("unit test not allowed to proceed") == 0 )

              {

                throw new ExitException();

              }

            }

            catch( InterruptedException e )

            {

              mLogger.warn("allowedToProceed caught InterruptedException");

            }

          }

        }

        else

        {

          mLogger.fatal("allowedToProceed calling mReleaseManager.queryDirectedRunLevel");                      

          if ( !mReleaseManager.queryDirectedRunLevel(mRconId) )

          {

            mLogger.info("allowedToProceed downtime");

            mReleaseManager.disconnect();

            mRunLevel = RunLevel.PAUSED;

            mLogger.warn("allowedToProceed changing run level to PAUSED for rcon_id " + mRconId);

            mRunLevel.persist(mReleaseManager, mRconId);

            try

            {

              // to do, sleep for periodicMs

              mLogger.warn("allowedToProceed sleep 5 mins directed downtime");

              Thread.sleep(300000);

              mLogger.info("allowedToProceed sleep returned");

            }

            catch (InterruptedException e)

            {

              mLogger.warn("allowedToProceed caught InterruptedException");

            }

          }

          else

          {

            mReleaseManager.disconnect();

            proceed = true;

          }

        }

      }

    }

    finally

    {

      // this block is executed regardless of what happens in the try block

      // even if an exception is thrown

      // ensure disconnect

      mLogger.fatal("allowedToProceed calling mReleaseManager.disconnect");                      

      mReleaseManager.disconnect();

    }

    mRecoverable = false;

  }

  /**periodically 

   * a) performs disk housekeeping

   * b) determines if a minimum threshold of disk space is available

   * c) determines if a file can be touched

   * changes the run level to CANNOT_CONTINUE if insufficient disk space

   * otherwise changes the run level to ACTIVE and returns

   * implements the sequence diagram check environment

   */

  protected void checkEnvironment() throws Exception

  {

    mLogger.debug("checkEnvironment");

    boolean exit = false;

    while( !exit )

    {

      housekeep();

      // attempt to exit

      exit = true;

      if ( !hasSufficientDiskSpace() || !touch() )

      {

        mLogger.warn("checkEnvironment below disk free threshold or read only file system detected");

        exit = false;

        mRunLevel = RunLevel.CANNOT_CONTINUE;

        mLogger.warn("checkEnvironment changing run level to CANNOT_CONTINUE for rcon_id " + mRconId);

        mRunLevel.persist(mReleaseManager, mRconId);

        try

        {

          // to do, sleep for periodicMs

          if ( mUnitTest.compareTo("unit test check environment") != 0 )

          {

            mLogger.warn("checkEnvironment sleep 5 mins below disk free threshold");

            Thread.sleep(300000);

            mLogger.info("checkEnvironment sleep returned");

          }

        }

        catch (InterruptedException e)

        {

          mLogger.warn("checkEnvironment caught InterruptedException");

        }

      }

    }

    mRunLevel = RunLevel.ACTIVE;    

    mLogger.warn("checkEnvironment changing run level to ACTIVE for rcon_id " + mRconId);

    mRunLevel.persist(mReleaseManager, mRconId);

  }

  /**performs disk housekeeping which involves deleting build directories > 5 days old

   * refer to the sequence diagram check environment

   */

  private void housekeep()

  {

    mLogger.debug("housekeep");

    FilenameFilter filter = new FilenameFilter()

    {

      public boolean accept(File file, String name)

      {

        mLogger.debug("accept " + name);

        boolean retVal = false;

        if ( file.isDirectory() && !name.startsWith( "." ) )

        {

          retVal = true;

        }

        mLogger.info("accept returned " + retVal);

        return retVal;

      }

    };

    try

    {

      // DEVI 46729, 46730, solaris 10 core dumps implicate deleteDirectory

      // let each BuildThread look after its own housekeeping

      File ocwd = new File( BuildDaemon.mGbeLog );

      File hcwd = new File( ocwd, BuildDaemon.mHostname );

      File cwd = new File( hcwd, String.valueOf( mRtagId ) );

      File[] children = cwd.listFiles( filter );

      if ( children != null )

      {

        for ( int child=0; child < children.length; child++ )

        {

          // child is named uniquely to encapsulate a build

          // 5 days = 432,000,000 milliseconds

          if ( ( System.currentTimeMillis() - children[ child ].lastModified() ) > 432000000 )

          {

            // the directory is over 5 days old

            mLogger.warn("housekeep deleting directory " + children[ child ].getName());

            if ( mUnitTest.compareTo("unit test check environment") != 0 )

            {

              deleteDirectory( children[ child ] );

            }

          }

        }

      }

    }

    catch( SecurityException e )

    {

      // this can be thrown by lastModified

      mLogger.warn("housekeep caught SecurityException");

    }

  }

  /**returns true if a file exists and can be deleted,

   * created and exists in the file system

   * this is to guard against read-only file systems

   */

  private boolean touch()

  {

    mLogger.debug("touch");

    boolean retVal = true;

    try

    {

      File touch = new File( String.valueOf( mRtagId ) + "touch" );

      if ( touch.exists() )

      {

        // delete it

        retVal = touch.delete();

      }

      if ( retVal )

      {

        // file does not exist

        retVal = touch.createNewFile();

      }

    }

    catch( SecurityException e )

    {

      // this can be thrown by exists, delete, createNewFile

      retVal = false;

      mLogger.warn("touch caught SecurityException");

    }

    catch( IOException e )

    {

      // this can be thrown by createNewFile

      retVal = false;

      mLogger.warn("touch caught IOException");

    }

    mLogger.info("touch returned " + retVal);

    return retVal;

  }

  /**returns true if free disk space > 10G

   * this may become configurable if the need arises

   * refer to the sequence diagram check environment

   */

  private boolean hasSufficientDiskSpace()

  {

    mLogger.debug("hasSufficientDiskSpace");

    boolean retVal = true;

    long freeSpace = 0;

    try

    {

      File cwd = new File( "." );

      // 5G = 5368709120 bytes

      // 1G = 1073741824 bytes - useful for testing

      if ( mUnitTest.compareTo("unit test check environment") == 0 )

      {

        if ( ReleaseManager.mPersistedRunLevelCollection.size() == 0 )

        {

          retVal = false;

        }

        else

        {

          retVal = true;

        }

      }

      else

      {

        freeSpace = cwd.getUsableSpace();

        if ( freeSpace < 5368709120L )

        {

          mLogger.warn("hasSufficientDiskSpace on " + cwd.getAbsolutePath() + " freeSpace " + freeSpace);

          retVal = false;

        }

      }

    }

    catch( SecurityException e )

    {

      // this can be thrown by getFreeSpace

       mLogger.warn("hasSufficientDiskSpace caught SecurityException");

    }

    mLogger.info("hasSufficientDiskSpace returned " + retVal + " " + freeSpace);

    return retVal;

  }

  /**abstract method

   */

  public abstract void run();

  /**deletes directory and all its files

   */

  protected void deleteDirectory(File directory)

  {

    mLogger.debug("deleteDirectory " + directory.getName());

    try

    {

      if ( directory.exists() )

      {

        FilenameFilter filter = new FilenameFilter()

        {

          public boolean accept(File file, String name)

          {

            mLogger.debug("accept " + name);

            boolean retVal = false;

            if ( name.compareTo(".") != 0 && ( name.compareTo("..") != 0 ) )

            {

              retVal = true;

            }

            mLogger.info("accept returned " + retVal);

            return retVal;

          }

        };

        File[] children = directory.listFiles( filter );

        if ( children != null )

        {

          for ( int child=0; child < children.length; child++ )

          {

            if ( children[ child ].isDirectory() )

            {

              deleteDirectory( children[ child ] );

            }

            else

            {

              children[ child ].delete();

            }

          }

        }

        directory.delete();

      }

    }

    catch( SecurityException e )

    {

      // this can be thrown by exists and delete

       mLogger.warn("deleteDirectory caught SecurityException");

    }

  }

  /**abstract method

   */

  protected abstract char getMode();

  /**

   * builds a buildFile from the buildFileContent

   * triggers ant to operate on the buildFile

   */

  protected void deliverChange(String buildFileContent, String target, boolean master)

  {

    mLogger.debug("deliverChange");

    // always perform a AbtSetUp and AbtTearDown

    if ( ( target == null && mErrorReported ) || ( target == "AbtPublish" && mErrorReported ) )

    {

      // AbtSetUp or the build failed

      // the default target will inevitably fail and will generate further email if allowed to proceed

      // do not mask the root cause

      return;

    }

    File buildFile = new File(mRtagId + "build.xml");

    boolean logError = true;

    Project p = new Project();

    try

    {

      //    AbtSetUp

      //    Create the build's xml file

      //

      if ( buildFileContent != null && target != null && target.compareTo("AbtSetUp") == 0 )

      {

        FileOutputStream buildFileOutputStream = new FileOutputStream(buildFile, false);

        buildFileOutputStream.close();

        StringReader buildFileContentStringReader = new StringReader(buildFileContent);

        BufferedReader buildFileBufferedReader = new BufferedReader(buildFileContentStringReader);

        // sanitise the buildFileContent

        //      it may contain line.separators of "\n", "\r", or "\r\n" variety, 

        //      depending on the location of the ripple engine

        String sanitisedBFC = new String();

        String lf = new String( System.getProperty("line.separator") );

        String line = new String();

        while( ( line = buildFileBufferedReader.readLine() ) != null)

        {

          sanitisedBFC += line + lf;

        }

        buildFileBufferedReader.close();

        FileWriter buildFileWriter = new FileWriter(buildFile);

        buildFileWriter.write(sanitisedBFC);

        buildFileWriter.close();

      }

      mReportingPackageName = null;

      mReportingPackageVersion = null;

      mReportingPackageExtension = null;

      mReportingPackageLocation = null;

      mReportingPackageDepends = null;

      mReportingIsRipple = null;

      mReportingPackageVersionId = null;

      mReportingDoesNotRequireSourceControlInteraction = null;

      mReportingTestBuild = null;

      mReportingFullyPublished = null;

      mReportingNewVcsTag = null;

      if ( buildFile.exists() )

      {

        p.setProperty("ant.file", buildFile.getAbsolutePath());

        // Add listener for logging the complete build process

        // If the daemon has been restarted, then the listener will not have been

        // set up - do don't add it. Perhaps we need a way to open an existing

        // logfile to append.

        if ( mBuildLogger != null )

        {

          p.addBuildListener(mBuildLogger);

        }

        p.init();

        ProjectHelper pH = ProjectHelper.getProjectHelper();

        p.addReference("ant.projectHelper", pH);

        // parse can throw BuildException, this is serious

        pH.parse(p, buildFile);

        mLogger.warn("deliverChange ant launched on " + buildFile.getAbsolutePath());

        if ( target == null )

        {

          target = p.getDefaultTarget();

        }

        mLogger.warn("deliverChange ant launched against target " + target);

        // executeTarget can throw BuildException, this is not serious

        logError = false;

        // set up project properties for reporting purposes

        // this first group are hard coded in the build file

        mReportingPackageName = p.getProperty("abt_package_name");

        mReportingPackageVersion = p.getProperty("abt_package_version");

        mReportingPackageExtension = p.getProperty("abt_package_extension");

        mReportingPackageLocation = p.getProperty("basedir") + p.getProperty("abt_package_location");

        mReportingPackageDepends = p.getProperty("abt_package_depends");

        mReportingIsRipple = p.getProperty("abt_is_ripple");

        mReportingPackageVersionId = p.getProperty("abt_package_version_id");

        mReportingDoesNotRequireSourceControlInteraction = p.getProperty("abt_does_not_require_source_control_interaction");

        mReportingTestBuild = p.getProperty("abt_test_build_instruction");

        p.executeTarget(target);

        mLogger.warn("deliverChange ant returned");

      }

    }

    //

    //  Catch exceptions

    //  Do not catch ALL exceptions. The MasterThread::run and SlaveThread::run

    //  relies on exceptions propergating upwards for the indefinite pause feature

    //

    catch( BuildException e )

    {

      if ( logError )

      {

        mLogger.error("deliverChange caught BuildException, the build failed " + e.getMessage());

      }

      else

      {

        if ( mReportingBuildFailureLogFile == null )

        {

          mReportingBuildFailureLogFile = e.getMessage();

        }

        mLogger.debug("deliverChange caught BuildException, big deal, the build failed " + mReportingBuildFailureLogFile);

      }

      mErrorReported = true;

    }

    catch( FileNotFoundException e )

    {

      mLogger.error("deliverChange caught FileNotFoundException");

    }

    catch( IOException e )

    {

      mLogger.error("deliverChange caught IOException");

    }

    // this group are set at run time (by the AbtPublish target only)

    // they will be null for every other target,

    // and null if an error occurs in the AbtPublish target

    mReportingFullyPublished = p.getProperty("abt_fully_published");

    mReportingNewVcsTag = p.getProperty("abt_new_vcstag");

  }

  /**Extract source from Version Control

   */

  protected void setViewUp(String content, boolean master) throws SQLException, Exception

  {

    mLogger.debug("setViewUp");

    mReportingBuildFailureLogFile = null;

    mErrorReported = false;

    if ( !master && mGbeGatherMetricsOnly != null )

    {

      // do not run AbtSetUp on slave in metrics gathering mode

      return;

    }

    // run ant on the AbtSetUp target

    deliverChange(content, "AbtSetUp", master);

  }

  /**

   * indefinite pause notification

  */

   protected void indefinitePause(RippleEngine rippleEngine, String cause)

   {

     mLogger.debug("indefinitePause");

     String body =

     "Hostname: " + BuildDaemon.mHostname + "<p>" +

     "Release: " + rippleEngine.mBaselineName + "<p>" +

     "Cause: " + cause + "<p><hr>";

     try

     {

       Smtpsend.send(

       rippleEngine.mMailServer,        // mailServer

       rippleEngine.mMailSender,        // source

       rippleEngine.mGlobalTarget,      // target (list)

       null,                            // cc

       null,                            // bcc

       "BUILD DAEMON INDEFINITE PAUSE", // subject

       body,                            // body

       null                             // attachment

       );

     }

     catch( Exception e )

     {

     }

   }

    /**

     *  Nagios interface

     *      Returns true if the thread looks OK

    */

    boolean checkThread()

    {

      boolean retVal = true;

      if ( mRunLevel == RunLevel.CANNOT_CONTINUE )

      {

        retVal = false;

      }

      else

      {

        retVal = checkThreadExtended();

      }

      mLogger.warn("checkThread returned " + retVal);

      return retVal;

    }

    /**

     * Nagios interface extension

     * This method should be overriden by classes that extend this class

     * If not overriden then the test indicates OK

     *

     *      Returns true if the thread looks OK

    */

    boolean checkThreadExtended()

    {

      return true;

    }

}