Subversion Repositories DevTools

package com.erggroup.buildtool.daemon;

import com.erggroup.buildtool.daemon.ResumeTimerTask;
import com.erggroup.buildtool.ripple.ReleaseManager;
import com.erggroup.buildtool.smtp.Smtpsend;
import com.erggroup.buildtool.ripple.RippleEngine;


import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.FilenameFilter;

import java.io.IOException;
import java.io.PrintStream;
import java.io.StringReader;

import java.sql.SQLException;

import java.util.Date;
import java.util.Timer;

import org.apache.log4j.Logger;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.DefaultLogger;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.ProjectHelper;

/**Build Thread sub component
 */
public abstract class BuildThread
  extends Thread
{
  /**baseline identifier (which release manager release this BuildThread is dealing with)
   * @attribute
   */
  protected int mRtagId = 0;

  /**unique identifier of this BuildThread
   * @attribute
   */
  protected int mRconId = 0;

  /**
   * @aggregation composite
   */
  protected RunLevel mRunLevel;

  /**
   * @aggregation composite
   */
  protected ReleaseManager mReleaseManager;

  /**unit test support
   * @attribute
   */
  protected String mUnitTest = "";

  /**
   * @aggregation composite
   */
  protected static ResumeTimerTask mResumeTimerTask;

  /**
   * @aggregation composite
   */
  private static Timer mTimer;

  /**Synchroniser object
   * Use to Synchronize on by multiple threads
   * @attribute
   */
  static final Object mSynchroniser = new Object();

  /**BuildThread group
   * @attribute
   */
  public static final ThreadGroup mThreadGroup = new ThreadGroup("BuildThread");

  /**the advertised build file content when either
   * a) no package in the baseline has a build requirement
   * b) the next package in the baseline with a build requirement is generic
   * @attribute
   */
  protected static final String mDummyBuildFileContent = new String("<dummy/>");

  /**Set true when last build cycle was benign.
   * @attribute
   */
  protected boolean mSleep;

  /**Set true when last build cycle caught SQLException or Exception.
   * @attribute
   */
  protected boolean mException;

  /**Set true when ant error reported on any target.
   * @attribute
   */
  private boolean mErrorReported;

  /**Logger
   * @attribute
   */
  private static final Logger mLogger = Logger.getLogger(BuildThread.class);

  /**Package name for reporting purposes.
   * @attribute
   */
  protected String mReportingPackageName;

  /**Package version for reporting purposes.
   * @attribute
   */
  protected String mReportingPackageVersion;

  /**Package extension for reporting purposes.
   * @attribute
   */
  protected String mReportingPackageExtension;

  /**Package location for reporting purposes.
   * @attribute
   */
  protected String mReportingPackageLocation;

  /**Package dependencies for reporting purposes.
   * @attribute
   */
  protected String mReportingPackageDepends;

  /**Is ripple flag for reporting purposes.
   * @attribute
   */
  protected String mReportingIsRipple;

  /**Package version identifier for reporting purposes.
   * @attribute
   */
  protected String mReportingPackageVersionId;

  /**Fully published flag for reporting purposes.
   * @attribute
   */
  protected String mReportingFullyPublished;
  
  /**New label for reporting purposes.
   * @attribute
   */
  protected String mReportingNewVcsTag;

  /**Source control interaction for reporting purposes.
   * @attribute
   */
  protected String mReportingDoesNotRequireSourceControlInteraction;

  /**Source control interaction for reporting purposes.
   * @attribute
   */
  protected String mReportingTestBuild;

  /**Log file location for reporting purposes.
   * @attribute
   */
  protected String mReportingBuildFailureLogFile;

  /**Non null determines to only gather metrics
   * @attribute
   */
  protected static final String mGbeGatherMetricsOnly = System.getenv("GBE_GATHER_METRICS");

  /**Non null determines to only gather metrics
   * @attribute
   */
  protected boolean mRecoverable;

  /**Logger for the entire build process
  * @attribute
  */
  DefaultLogger mBuildLogger ;

  /**constructor
   */
  BuildThread()
  {
    super(mThreadGroup, "");
    mLogger.debug("BuildThread");
    mSleep = false;
    mException = false;
    mErrorReported = false;
    mReleaseManager = new ReleaseManager();
    mRecoverable = false;
    
    // no need to be synchronized - BuildThreads are instantiated in a single thread
    if ( mResumeTimerTask == null )
    {
      mResumeTimerTask = new ResumeTimerTask();
      mTimer = new Timer();
      mResumeTimerTask.setTimer(mTimer);
    }
  }

  /**Flags that a new build cycle is about to start
  *  Create a new logger to capture all the complete build log
  *  even though its done in stages we want a complete log.
  */
  void flagStartBuildCycle()
  {
    try
    {
      mBuildLogger = new DefaultLogger();
      PrintStream ps = new PrintStream(mRtagId + ".log");
      mBuildLogger.setOutputPrintStream(ps);
      mBuildLogger.setMessageOutputLevel(Project.MSG_INFO);
    }
    catch( FileNotFoundException e )
    {
      mLogger.error("BuildThread caught FileNotFoundException");
    }
  }

  /**sleeps when mException is set
   */
  protected void sleepCheck()
  {
    if (mException)
    {
      try
      {
        Integer sleepTime = 300000;
        mLogger.warn("sleepCheck sleep " + sleepTime.toString() + " secs");
        Thread.sleep(sleepTime);
        mLogger.info("sleepCheck sleep returned");
      }
      catch(InterruptedException e)
      {
        mLogger.warn("sleepCheck sleep caught InterruptedException");
      }
      
    }
    mException = false;
  }
  
  /**initially changes the run level to IDLE
   * determines if the BuildThread is still configured
   * a) determines if the BuildThread is running in scheduled downtime
   * b) determines if the BuildThread is directed to pause
   * changes the run level to PAUSED if a) or b) are true
   * throws ExitException when not configured
   * implements the sequence diagrams allowed to proceed, not allowed to proceed, exit
   */
  protected void allowedToProceed(boolean master) throws ExitException, SQLException, Exception
  {
    mLogger.debug("allowedToProceed");
    
    try
    {
      mRunLevel = RunLevel.IDLE;
      mLogger.warn("allowedToProceed changing run level to IDLE for rcon_id " + mRconId);
      mLogger.fatal("allowedToProceed calling mRunLevel.persist on IDLE");                      
      mRunLevel.persist(mReleaseManager, mRconId);
      if ( master )
      {
        mLogger.fatal("allowedToProceed calling mReleaseManager.discardVersion");                      
        mReleaseManager.discardVersion();
      }
      mLogger.fatal("allowedToProceed calling mReleaseManager.clearCurrentPackageBeingBuilt");                      
      mReleaseManager.clearCurrentPackageBeingBuilt(mRconId);      
    }
    catch(SQLException e)
    {
      mLogger.warn("allowedToProceed caught SQLException");
    }
  
    if (mSleep)
    {
      try
      {
        Integer sleepTime = 300000;
        
        if ( !master )
        {
          // sleep only 3 secs on slave
          sleepTime = 3000;
        }
        mLogger.warn("allowedToProceed sleep " + sleepTime.toString() + " secs no build requirement");
        mLogger.fatal("allowedToProceed calling Thread.sleep for 3 secs");                      
        Thread.sleep(sleepTime);
        mLogger.info("allowedToProceed sleep returned");
      }
      catch(InterruptedException e)
      {
        mLogger.warn("allowedToProceed sleep caught InterruptedException");
      }
    }
    
    boolean proceed = false;
    
    try
    {
      while ( !proceed )
      {
        mLogger.fatal("allowedToProceed calling mReleaseManager.connect");                      
        mReleaseManager.connect();
        mLogger.fatal("allowedToProceed calling mReleaseManager.queryReleaseConfig");                      
        if ( !mReleaseManager.queryReleaseConfig(mRtagId, mRconId, BuildDaemon.mHostname, getMode()) )
        {
          mReleaseManager.disconnect();
          mLogger.warn("allowedToProceed queryReleaseConfig failed");
          throw new ExitException();
        }
        
        Date resumeTime = new Date( 0 );
        mLogger.fatal("allowedToProceed calling mReleaseManager.queryRunLevelSchedule");                      
        if ( !mReleaseManager.queryRunLevelSchedule(resumeTime, mRecoverable) )
        {
          mLogger.info("allowedToProceed scheduled downtime");
          mReleaseManager.disconnect();
          mRunLevel = RunLevel.PAUSED;
          mLogger.warn("allowedToProceed changing run level to PAUSED for rcon_id " + mRconId);
          mRunLevel.persist(mReleaseManager, mRconId);
          
          synchronized(mSynchroniser)
          {
            // contain the schedule and wait in the same synchronized block to prevent a deadlock
            // eg this thread calls schedule, timer thread calls notifyall, this thread calls wait (forever)
            try
            {
              if (mResumeTimerTask.isCancelled())
              {
                mResumeTimerTask = new ResumeTimerTask();
                mTimer = new Timer();
                mResumeTimerTask.setTimer(mTimer);
              }
              mLogger.warn("allowedToProceed schedule passed " + resumeTime.getTime());
              mTimer.schedule(mResumeTimerTask, resumeTime);
            }
            catch( IllegalStateException e )
            {
              // this may be thrown by schedule if already scheduled
              // it signifies another BuildThread has already scheduled the ResumeTimerTask
               mLogger.warn("allowedToProceed already scheduled");
            }
  
            try
            {
              mLogger.warn("allowedToProceed wait");
              mSynchroniser.wait();
              mLogger.warn("allowedToProceed wait returned");
               
              if ( mUnitTest.compareTo("unit test not allowed to proceed") == 0 )
              {
                throw new ExitException();
              }
            }
            catch( InterruptedException e )
            {
              mLogger.warn("allowedToProceed caught InterruptedException");
            }
          }
          
        }
        else
        {
          mLogger.fatal("allowedToProceed calling mReleaseManager.queryDirectedRunLevel");                      
          if ( !mReleaseManager.queryDirectedRunLevel(mRconId) )
          {
            mLogger.info("allowedToProceed downtime");
            mReleaseManager.disconnect();
            mRunLevel = RunLevel.PAUSED;
            mLogger.warn("allowedToProceed changing run level to PAUSED for rcon_id " + mRconId);
            mRunLevel.persist(mReleaseManager, mRconId);
            try
            {
              // to do, sleep for periodicMs
              mLogger.warn("allowedToProceed sleep 5 mins directed downtime");
              Thread.sleep(300000);
              mLogger.info("allowedToProceed sleep returned");
            }
            catch (InterruptedException e)
            {
              mLogger.warn("allowedToProceed caught InterruptedException");
            }
          }
          else
          {
            mReleaseManager.disconnect();
            proceed = true;
          }
        }
      }
    }
    finally
    {
      // this block is executed regardless of what happens in the try block
      // even if an exception is thrown
      // ensure disconnect
      mLogger.fatal("allowedToProceed calling mReleaseManager.disconnect");                      
      mReleaseManager.disconnect();
    }
    
    mRecoverable = false;
  }

  /**periodically 
   * a) performs disk housekeeping
   * b) determines if a minimum threshold of disk space is available
   * c) determines if a file can be touched
   * changes the run level to CANNOT_CONTINUE if insufficient disk space
   * otherwise changes the run level to ACTIVE and returns
   * implements the sequence diagram check environment
   */
  protected void checkEnvironment() throws Exception
  {
    mLogger.debug("checkEnvironment");
    boolean exit = false;
    
    while( !exit )
    {
      housekeep();
      
      // attempt to exit
      exit = true;
    
      if ( !hasSufficientDiskSpace() || !touch() )
      {
        mLogger.warn("checkEnvironment below disk free threshold or read only file system detected");
        exit = false;
        mRunLevel = RunLevel.CANNOT_CONTINUE;
        mLogger.warn("checkEnvironment changing run level to CANNOT_CONTINUE for rcon_id " + mRconId);
        mRunLevel.persist(mReleaseManager, mRconId);
        try
        {
          // to do, sleep for periodicMs
          if ( mUnitTest.compareTo("unit test check environment") != 0 )
          {
            mLogger.warn("checkEnvironment sleep 5 mins below disk free threshold");
            Thread.sleep(300000);
            mLogger.info("checkEnvironment sleep returned");
          }
        }
        catch (InterruptedException e)
        {
          mLogger.warn("checkEnvironment caught InterruptedException");
        }
      }
    }

    mRunLevel = RunLevel.ACTIVE;    
    mLogger.warn("checkEnvironment changing run level to ACTIVE for rcon_id " + mRconId);
    mRunLevel.persist(mReleaseManager, mRconId);
  }

  /**performs disk housekeeping which involves deleting build directories > 5 days old
   * refer to the sequence diagram check environment
   */
  private void housekeep()
  {
    mLogger.debug("housekeep");
    FilenameFilter filter = new FilenameFilter()
    {
      public boolean accept(File file, String name)
      {
        mLogger.debug("accept " + name);
        boolean retVal = false;
        
        if ( file.isDirectory() && !name.startsWith( "." ) )
        {
          retVal = true;
        }
        
        mLogger.info("accept returned " + retVal);
        return retVal;
      }
    };
    
    try
    {
      // DEVI 46729, 46730, solaris 10 core dumps implicate deleteDirectory
      // let each BuildThread look after its own housekeeping
      File ocwd = new File( BuildDaemon.mGbeLog );
      File hcwd = new File( ocwd, BuildDaemon.mHostname );
      File cwd = new File( hcwd, String.valueOf( mRtagId ) );

      File[] children = cwd.listFiles( filter );
      
      if ( children != null )
      {
        for ( int child=0; child < children.length; child++ )
        {
          // child is named uniquely to encapsulate a build
          // 5 days = 432,000,000 milliseconds
          if ( ( System.currentTimeMillis() - children[ child ].lastModified() ) > 432000000 )
          {
            // the directory is over 5 days old
            mLogger.warn("housekeep deleting directory " + children[ child ].getName());
            if ( mUnitTest.compareTo("unit test check environment") != 0 )
            {
              deleteDirectory( children[ child ] );
            }
          }
        }
      }
    }
    catch( SecurityException e )
    {
      // this can be thrown by lastModified
      mLogger.warn("housekeep caught SecurityException");
    }
     
  }

  /**returns true if a file exists and can be deleted,
   * created and exists in the file system
   * this is to guard against read-only file systems
   */
  private boolean touch()
  {
    mLogger.debug("touch");
    boolean retVal = true;
    
    try
    {
      File touch = new File( String.valueOf( mRtagId ) + "touch" );
      
      if ( touch.exists() )
      {
        // delete it
        retVal = touch.delete();
      }
      
      if ( retVal )
      {
        // file does not exist
        retVal = touch.createNewFile();
      }
    }
    catch( SecurityException e )
    {
      // this can be thrown by exists, delete, createNewFile
      retVal = false;
      mLogger.warn("touch caught SecurityException");
    }
    catch( IOException e )
    {
      // this can be thrown by createNewFile
      retVal = false;
      mLogger.warn("touch caught IOException");
    }
    
    mLogger.info("touch returned " + retVal);
    return retVal;
  }
  
  /**returns true if free disk space > 10G
   * this may become configurable if the need arises
   * refer to the sequence diagram check environment
   */
  private boolean hasSufficientDiskSpace()
  {
    mLogger.debug("hasSufficientDiskSpace");
    boolean retVal = true;
    long freeSpace = 0;
    
    try
    {
      File cwd = new File( "." );

      // 5G = 5368709120 bytes
      // 1G = 1073741824 bytes - useful for testing
      if ( mUnitTest.compareTo("unit test check environment") == 0 )
      {
        if ( ReleaseManager.mPersistedRunLevelCollection.size() == 0 )
        {
          retVal = false;
        }
        else
        {
          retVal = true;
        }
      }
      else
      {
        freeSpace = cwd.getUsableSpace();
        
        if ( freeSpace < 5368709120L )
        {
          mLogger.warn("hasSufficientDiskSpace on " + cwd.getAbsolutePath() + " freeSpace " + freeSpace);
          retVal = false;
        }
      }
    }
    catch( SecurityException e )
    {
      // this can be thrown by getFreeSpace
       mLogger.warn("hasSufficientDiskSpace caught SecurityException");
    }
    
    mLogger.info("hasSufficientDiskSpace returned " + retVal + " " + freeSpace);
    return retVal;
  }

  /**abstract method
   */
  public abstract void run();

  /**deletes directory and all its files
   */
  protected void deleteDirectory(File directory)
  {
    mLogger.debug("deleteDirectory " + directory.getName());
    try
    {
      if ( directory.exists() )
      {
        FilenameFilter filter = new FilenameFilter()
        {
          public boolean accept(File file, String name)
          {
            mLogger.debug("accept " + name);
            boolean retVal = false;
            
            if ( name.compareTo(".") != 0 && ( name.compareTo("..") != 0 ) )
            {
              retVal = true;
            }
            
            mLogger.info("accept returned " + retVal);
            return retVal;
          }
        };
        
        File[] children = directory.listFiles( filter );
        
        if ( children != null )
        {
          for ( int child=0; child < children.length; child++ )
          {
            if ( children[ child ].isDirectory() )
            {
              deleteDirectory( children[ child ] );
            }
            else
            {
              children[ child ].delete();
            }
          }
        }
        directory.delete();
      }
    }
    catch( SecurityException e )
    {
      // this can be thrown by exists and delete
       mLogger.warn("deleteDirectory caught SecurityException");
    }
  }

  /**abstract method
   */
  protected abstract char getMode();

  /**
   * builds a buildFile from the buildFileContent
   * triggers ant to operate on the buildFile
   */
  protected void deliverChange(String buildFileContent, String target, boolean master)
  {
    mLogger.debug("deliverChange");
    
    // always perform a AbtSetUp and AbtTearDown
    if ( ( target == null && mErrorReported ) || ( target == "AbtPublish" && mErrorReported ) )
    {
      // AbtSetUp or the build failed
      // the default target will inevitably fail and will generate further email if allowed to proceed
      // do not mask the root cause
      return;
    }
    
    File buildFile = new File(mRtagId + "build.xml");
    boolean logError = true;
    Project p = new Project();
    
    try
    {
      //    AbtSetUp
      //    Create the build's xml file
      //
      if ( buildFileContent != null && target != null && target.compareTo("AbtSetUp") == 0 )
      {
        FileOutputStream buildFileOutputStream = new FileOutputStream(buildFile, false);
        buildFileOutputStream.close();
      
        StringReader buildFileContentStringReader = new StringReader(buildFileContent);
        BufferedReader buildFileBufferedReader = new BufferedReader(buildFileContentStringReader);
        
        // sanitise the buildFileContent
        //      it may contain line.separators of "\n", "\r", or "\r\n" variety, 
        //      depending on the location of the ripple engine
        String sanitisedBFC = new String();
        String lf = new String( System.getProperty("line.separator") );
        String line = new String();
        
        while( ( line = buildFileBufferedReader.readLine() ) != null)
        {
          sanitisedBFC += line + lf;
        }
        buildFileBufferedReader.close();
        FileWriter buildFileWriter = new FileWriter(buildFile);
        buildFileWriter.write(sanitisedBFC);
        buildFileWriter.close();
      }
      
      mReportingPackageName = null;
      mReportingPackageVersion = null;
      mReportingPackageExtension = null;
      mReportingPackageLocation = null;
      mReportingPackageDepends = null;
      mReportingIsRipple = null;
      mReportingPackageVersionId = null;
      mReportingDoesNotRequireSourceControlInteraction = null;
      mReportingTestBuild = null;
      mReportingFullyPublished = null;
      mReportingNewVcsTag = null;

      if ( buildFile.exists() )
      {
        p.setProperty("ant.file", buildFile.getAbsolutePath());

        // Add listener for logging the complete build process
        // If the daemon has been restarted, then the listener will not have been
        // set up - do don't add it. Perhaps we need a way to open an existing
        // logfile to append.
        if ( mBuildLogger != null )
        {
          p.addBuildListener(mBuildLogger);
        }
        
        p.init();
        ProjectHelper pH = ProjectHelper.getProjectHelper();
        p.addReference("ant.projectHelper", pH);
        
        // parse can throw BuildException, this is serious
        pH.parse(p, buildFile);
        mLogger.warn("deliverChange ant launched on " + buildFile.getAbsolutePath());
        
        if ( target == null )
        {
          target = p.getDefaultTarget();
        }
        mLogger.warn("deliverChange ant launched against target " + target);
        
        // executeTarget can throw BuildException, this is not serious
        logError = false;
        // set up project properties for reporting purposes
        // this first group are hard coded in the build file
        mReportingPackageName = p.getProperty("abt_package_name");
        mReportingPackageVersion = p.getProperty("abt_package_version");
        mReportingPackageExtension = p.getProperty("abt_package_extension");
        mReportingPackageLocation = p.getProperty("basedir") + p.getProperty("abt_package_location");
        mReportingPackageDepends = p.getProperty("abt_package_depends");
        mReportingIsRipple = p.getProperty("abt_is_ripple");
        mReportingPackageVersionId = p.getProperty("abt_package_version_id");
        mReportingDoesNotRequireSourceControlInteraction = p.getProperty("abt_does_not_require_source_control_interaction");
        mReportingTestBuild = p.getProperty("abt_test_build_instruction");
        
        p.executeTarget(target);
        mLogger.warn("deliverChange ant returned");
        
      }
    }
    //
    //  Catch exceptions
    //  Do not catch ALL exceptions. The MasterThread::run and SlaveThread::run
    //  relies on exceptions propergating upwards for the indefinite pause feature
    //
    catch( BuildException e )
    {
      if ( logError )
      {
        mLogger.error("deliverChange caught BuildException, the build failed " + e.getMessage());
      }
      else
      {
        if ( mReportingBuildFailureLogFile == null )
        {
          mReportingBuildFailureLogFile = e.getMessage();
        }
        mLogger.debug("deliverChange caught BuildException, big deal, the build failed " + mReportingBuildFailureLogFile);
      }
      
      mErrorReported = true;
    }
    catch( FileNotFoundException e )
    {
      mLogger.error("deliverChange caught FileNotFoundException");
    }
    catch( IOException e )
    {
      mLogger.error("deliverChange caught IOException");
    }
    
    // this group are set at run time (by the AbtPublish target only)
    // they will be null for every other target,
    // and null if an error occurs in the AbtPublish target
    mReportingFullyPublished = p.getProperty("abt_fully_published");
    mReportingNewVcsTag = p.getProperty("abt_new_vcstag");
    
  }
  
  /**Extract source from Version Control
   */
  protected void setViewUp(String content, boolean master) throws SQLException, Exception
  {
    mLogger.debug("setViewUp");
    mReportingBuildFailureLogFile = null;
    mErrorReported = false;
    
    if ( !master && mGbeGatherMetricsOnly != null )
    {
      // do not run AbtSetUp on slave in metrics gathering mode
      return;
    }
    
    // run ant on the AbtSetUp target
    deliverChange(content, "AbtSetUp", master);
  }
  
  /**
   * indefinite pause notification
  */
   protected void indefinitePause(RippleEngine rippleEngine, String cause)
   {
     mLogger.debug("indefinitePause");
       
     String body =
     "Hostname: " + BuildDaemon.mHostname + "<p>" +
     "Release: " + rippleEngine.mBaselineName + "<p>" +
     "Cause: " + cause + "<p><hr>";
     
     try
     {
       Smtpsend.send(
       rippleEngine.mMailServer,        // mailServer
       rippleEngine.mMailSender,        // source
       rippleEngine.mGlobalTarget,      // target (list)
       null,                            // cc
       null,                            // bcc
       "BUILD DAEMON INDEFINITE PAUSE", // subject
       body,                            // body
       null                             // attachment
       );
     }
     catch( Exception e )
     {
     }
   }

    /**
     *  Nagios interface
     *      Returns true if the thread looks OK
    */
    boolean checkThread()
    {
      boolean retVal = true;
      if ( mRunLevel == RunLevel.CANNOT_CONTINUE )
      {
        retVal = false;
      }
      else
      {
        retVal = checkThreadExtended();
      }

      mLogger.warn("checkThread returned " + retVal);
      return retVal;
    }

    /**
     * Nagios interface extension
     * This method should be overriden by classes that extend this class
     * If not overriden then the test indicates OK
     *
     *      Returns true if the thread looks OK
    */
    boolean checkThreadExtended()
    {
      return true;
    }

}