Subversion Repositories DevTools

Rev

Go to most recent revision | Details | Last modification | View Log | RSS feed

Rev Author Line No. Line
6914 dpurdie 1
package com.erggroup.buildtool.daemon;
2
 
3
import com.erggroup.buildtool.abt.BuildToolException;
4
import com.erggroup.buildtool.abt.RemoteExecution;
5
import com.erggroup.buildtool.abt.RemoteExecution.LogOutput;
6
import com.erggroup.buildtool.daemon.BuildDaemon.NagiosInfo;
7
import com.erggroup.buildtool.daemon.ResumeTimerTask;
8
import com.erggroup.buildtool.ripple.Package;
9
import com.erggroup.buildtool.ripple.Phase;
10
import com.erggroup.buildtool.ripple.ReleaseManager;
11
import com.erggroup.buildtool.ripple.ReportingData;
12
import com.erggroup.buildtool.ripple.RunLevel;
13
import com.erggroup.buildtool.smtp.CreateUrls;
14
import com.erggroup.buildtool.smtp.Smtpsend;
15
import com.erggroup.buildtool.utilities.MutableDate;
16
import com.erggroup.buildtool.utilities.utilities;
17
import com.erggroup.buildtool.ripple.RippleEngine;
18
import com.erggroup.buildtool.ripple.RunLevel.BuildState;
19
import com.erggroup.buildtool.daemon.BuildDaemon;
20
 
21
import java.io.BufferedReader;
22
import java.io.File;
23
import java.io.FileNotFoundException;
24
import java.io.FileWriter;
25
import java.io.FilenameFilter;
26
import java.io.IOException;
27
import java.io.PrintStream;
28
import java.io.StringReader;
29
import java.sql.SQLException;
30
import java.text.SimpleDateFormat;
31
import java.util.Date;
32
import java.util.LinkedHashMap;
33
import java.util.Map;
34
import java.util.Timer;
35
 
36
import org.apache.log4j.Logger;
37
import org.apache.tools.ant.BuildException;
38
import org.apache.tools.ant.DefaultLogger;
39
import org.apache.tools.ant.Project;
40
import org.apache.tools.ant.ProjectHelper;
41
 
42
/**Build Thread sub component
43
 */
44
public abstract class BuildThread
45
  extends Thread
46
{
47
  /**baseline identifier (which release manager release this BuildThread is dealing with)
48
   * @attribute
49
   */
50
  protected int mRtagId = 0;
51
 
52
  /**unique identifier of this BuildThread
53
   * @attribute
54
   */
55
  protected int mRconId = 0;
56
 
57
  /**
58
   * Used as a monitor for:
59
   * Slave: Indication that a command has been issued by the master
60
   * Master: Not yet used
61
   */
62
  protected Object mActiveBuildMonitor = new Object();
63
 
64
 
65
  /**
66
   * @aggregation composite
67
   */
68
  protected RunLevel mRunLevel = new RunLevel(BuildState.DB_IDLE);
69
 
70
  /**
71
   * @aggregation composite
72
   */
73
  protected ReleaseManager mReleaseManager;
74
 
75
  /**
76
   * 
77
   */
78
  protected RippleEngine mRippleEngine;
79
 
80
  /** Parsed ANT project
81
   * 
82
   */
83
  Project mProject = null;
84
 
85
  /**
86
   * Indicate the start time of the thread
87
   * Not nanosecond accurate, just an indication of when it was started
88
   */
89
  protected final long mStartTime = System.currentTimeMillis();
90
 
91
  /**unit test support
92
   * @attribute
93
   */
94
  protected String mUnitTest = "";
95
 
96
  /**
97
   * @aggregation composite
98
   */
99
  protected ResumeTimerTask mResumeTimerTask;
100
 
101
  /**
102
   * @aggregation composite
103
   */
104
  private Timer mTimer;
105
 
106
  /**Synchroniser object
107
   * Use to Synchronize on by multiple threads
108
   * @attribute
109
   */
110
  static final Object mSynchroniser = new Object();
111
 
112
  /**BuildThread group
113
   * @attribute
114
   */
115
  public static final ThreadGroup mThreadGroup = new ThreadGroup("BuildThread");
116
 
117
  /**the advertised build file content when either
118
   * a) no package in the baseline has a build requirement
119
   * b) the next package in the baseline with a build requirement is generic
120
   * @attribute
121
   */
122
  protected static final String mDummyBuildFileContent = "<dummy/>";
123
 
124
  /**Set true when last build cycle was benign.
125
   * @attribute
126
   */
127
  protected boolean mLastBuildWasBenign;
128
 
129
  /** Last retrieved release sequence number
130
   *  Used to determine if the Release Content has been changed by a user
131
   *  The sequence number does not take into account changes by the build system
132
   */
133
  protected int mReleaseSeqNum = -1;
134
 
135
  /** Holds the current buildfile 
136
   * 
137
   */
138
  protected File mBuildFile = new File("");
139
 
140
  /**Set true when last build cycle caught SQLException or Exception.
141
   * @attribute
142
   */
143
  protected boolean mException;
144
 
145
  /**Set true when ant error reported on any target.
146
   * @attribute
147
   */
148
  protected boolean mErrorReported;
149
 
150
  /**Logger
151
   * @attribute
152
   */
153
  private static final Logger mLogger = Logger.getLogger(BuildThread.class);
154
 
155
  /** Data about the package that is being built
156
   * @attribute
157
   */
158
  protected ReportingData mReporting = new ReportingData();
159
 
160
  /**Indicate the type of indefinite pause being processed
161
   * If true, then it is one that the daemon threads can recover from
162
   * If false, then wait for RM Database to indicate that its OK to proceed
163
   * @attribute
164
   */
165
  protected boolean mRecoverable;
166
 
167
  /**Logger for the entire build process
168
  * @attribute
169
  */
170
  DefaultLogger mBuildLogger ;
171
 
172
  /**Nagios Monitoring
173
   * @attribute
174
   */
175
  Phase mPhase = new Phase();
176
 
177
 
178
  /**constructor
179
   * @param rtagId          - Identifes the Release
180
   * @param rconId          - Identifies the Controlling entry
181
   * @param releaseManager  - A ReleaseManager entry to use
182
   * @param unitTest        - Unit Test data 
183
   */
184
  BuildThread(int rtagId, int rconId, ReleaseManager releaseManager, String unitTest)
185
  {
186
    super(mThreadGroup, "");
187
    mLogger.debug("BuildThread");
188
    mLastBuildWasBenign = true;
189
    mException = false;
190
    mErrorReported = false;
191
    mRecoverable = false;
192
 
193
    mRtagId = rtagId;
194
    mRconId = rconId;
195
    mReleaseManager = releaseManager;
196
    mBuildFile = new File(mRtagId + "build.xml");
197
    mUnitTest = ( unitTest == null ) ? "" : unitTest;
198
 
199
    // no need to be synchronized - BuildThreads are instantiated in a single thread
200
    if ( mResumeTimerTask == null )
201
    {
202
      mResumeTimerTask = new ResumeTimerTask();
203
      mTimer = new Timer();
204
      mResumeTimerTask.setTimer(mTimer);
205
    }
206
  }
207
 
208
  /**Flags that a new build cycle is about to start
209
  *  Create a new logger to capture all the complete build log
210
  *  even though its done in stages we want a complete log.
211
  */
212
  void flagStartBuildCycle()
213
  {
214
    try
215
    {
216
      mBuildLogger = new DefaultLogger();
217
      PrintStream ps = new PrintStream(mRtagId + ".log");
218
      mBuildLogger.setOutputPrintStream(ps);
219
      mBuildLogger.setMessageOutputLevel(Project.MSG_INFO);
220
    }
221
    catch( FileNotFoundException e )
222
    {
223
      mLogger.error("BuildThread caught FileNotFoundException");
224
    }
225
  }
226
 
227
  /**
228
   *    Sleeps when mException is set
229
   *    Called at the top of the run() loop, just after exception processing
230
   */
231
  protected void sleepCheck()
232
  {
233
    if (mException && ! BuildDaemon.mShutDown)
234
    {
235
        Integer sleepTime = 5 * 60;
236
        mLogger.warn("sleepCheck sleep " + sleepTime.toString() + " secs");
237
        BuildDaemon.daemonSleepSecs(sleepTime);
238
        mLogger.info("sleepCheck sleep returned");
239
 
240
        // Force planning cycle when recovering from an exception
241
        mReleaseSeqNum = -1;
242
    }
243
    mException = false;
244
  }
245
 
246
  /**initially changes the run level to IDLE
247
   * <br>Determines if the BuildThread is still configured
248
   * <br>Detects shutdown requests
249
   * <br>a) Determines if the BuildThread is running in scheduled down time
250
   * <br>b) Determines if the BuildThread is directed to pause
251
   * <br>Changes the run level to PAUSED if a) or b) are true
252
   * <br>throws ExitException when not configured or shutdown request
253
   * 
254
   * @param     master  True: Master Daemon
255
   * @exception ExitException
256
   * @exception SQLException
257
   * @exception Exception
258
   */
259
  protected void allowedToProceed(boolean master) throws ExitException, SQLException, Exception
260
  {
261
    mLogger.debug("allowedToProceed");
262
 
263
    //  Release Manager Database operations
264
    //      Change the run level to IDLE
265
    //      Discard any reserved version numbers
266
    //      Reset the current package being built
267
    try
268
    {
269
      mLogger.fatal("allowedToProceed changing run level to IDLE for rcon_id " + mRconId);
270
      mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_IDLE);
271
    }
272
    catch(SQLException e)
273
    {
274
      mLogger.warn("allowedToProceed caught SQLException");
275
    }
276
 
277
    //  Sleep for the required time - if the last build cycle was benign
278
    //      Slaves sleep for 3 seconds
279
    //      Masters will sleep for up to 60 minutes
280
    if (mLastBuildWasBenign && !BuildDaemon.mShutDown)
281
    {
282
        if (master)
283
        {
284
            long startMasterWait = System.currentTimeMillis();
285
            while(!BuildDaemon.mShutDown)
286
            {
287
                int masterCount = mReleaseManager.queryMasterCount( mRtagId );
288
                if ( masterCount != 1)
289
                {
290
                    mLogger.warn("allowedToProceed Invalid MasterCount: " + masterCount);
291
                    throw new ExitException();   
292
                }
293
 
294
                int seqNum = mReleaseManager.queryReleaseSeqNum(mRtagId, mRconId, BuildDaemon.mHostname);
295
                if ( seqNum < 0 )
296
                {
297
                    mLogger.warn("allowedToProceed Not part of buildset");
298
                    throw new ExitException();
299
                }
300
                if ( seqNum != mReleaseSeqNum)
301
                {
302
                    //  Release Content has been modified
303
                    mReleaseSeqNum = seqNum;
304
                    mLogger.fatal("allowedToProceed. Release Sequence changed");
305
                    break;
306
                }
307
 
308
                //
309
                //    Backup plan
310
                //    Force a planning session every 60 minutes
311
                //
312
                if ( System.currentTimeMillis() - startMasterWait > 60 * 60 * 1000)
313
                {
314
                    mLogger.fatal("allowedToProceed. Force periodic plan");
315
                    break;
316
                }
317
 
318
                //  Have no reason to suspect that we need to plan a release
319
                //  Persist idle state to indicate the master is polling
320
                //  Wait 60 seconds and test again
321
                //
322
                mLogger.warn("allowedToProceed sleep 60 secs no build requirement");
323
                mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_IDLE);
324
                BuildDaemon.daemonSleepSecs(60);
325
                mLogger.info("allowedToProceed sleep returned");
326
            }
327
        }
328
        else
329
        {
330
            //  Slave - Wait 3 seconds
331
            mLogger.warn("allowedToProceed sleep 3 secs no build requirement");
332
            BuildDaemon.daemonSleepSecs(3);
333
            mLogger.info("allowedToProceed sleep returned");
334
        }
335
    }
336
 
337
    if (BuildDaemon.mShutDown)
338
    {
339
        mLogger.warn("allowedToProceed ShutDown requested");
340
        throw new ExitException();
341
    }
342
 
343
    //  Loop until this daemon can proceed - or no longer configured as a part of the build set
344
    //  Things that will hold it up:
345
    //      Indefinite pause
346
    //      Scheduled down time
347
    //      Waiting for recoverable error (dpkg_archive access)
348
 
349
    boolean proceed = false;
350
    try
351
    {
352
      while ( !proceed &&  !BuildDaemon.mShutDown )
353
      {
354
        mLogger.fatal("allowedToProceed calling mReleaseManager.connect");                      
355
        mReleaseManager.connect();
356
 
357
        //  Ensure that the machine is a part of the current build set
358
        mLogger.fatal("allowedToProceed calling mReleaseManager.queryReleaseConfig");                      
359
        if ( !mReleaseManager.queryReleaseConfig(mRtagId, mRconId, BuildDaemon.mHostname, getMode(), mStartTime) )
360
        {
361
            mLogger.warn("allowedToProceed queryReleaseConfig failed");
362
            mReleaseManager.disconnect();
363
            throw new ExitException();
364
        }
365
 
366
        //  If dpkg_archive has been flagged as unavailable, then check for its recovery
367
        //
368
        if ( mRecoverable )
369
        {
370
            if ( Package.recover() )
371
            {
372
                // Package archive(s) exist
373
                // clear the indefinite pause condition
374
                mReleaseManager.resumeIndefinitePause();
375
                notifyIndefiniteRecovery();
376
                mRecoverable = false;
377
            }
378
        }
379
 
380
        if ( mUnitTest.compareTo("unit test not allowed to proceed") == 0 )
381
        {
382
          mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_PAUSED);
383
          throw new ExitException();
384
        }
385
 
386
        //  Check for scheduled downtime or indefinite pause
387
        MutableDate resumeTime = new MutableDate();
388
        mLogger.fatal("allowedToProceed calling mReleaseManager.queryRunLevelSchedule");                      
389
        if ( !mReleaseManager.queryRunLevelSchedule(resumeTime) )
390
        {
391
          mLogger.info("allowedToProceed scheduled downtime");
392
          mReleaseManager.disconnect();
393
 
394
          mLogger.fatal("allowedToProceed changing run level to PAUSED for rcon_id " + mRconId);
395
          mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_PAUSED);
396
 
397
          //
398
          //    Limit the pause to 10 minutes at a time
399
          //    Allows early get out for the daemons
400
          long  tenMinutesFromNow = System.currentTimeMillis() + 10 * 60 * 1000;
401
          if (resumeTime.value.getTime() > tenMinutesFromNow)
402
          {
403
              resumeTime.value.setTime(tenMinutesFromNow);
404
              mLogger.warn("allowedToProceed delay limited to 10 minutes");
405
          }
406
 
407
          synchronized(mSynchroniser)
408
          {
409
            // contain the schedule and wait in the same synchronized block to prevent a deadlock
410
            // eg this thread calls schedule, timer thread calls notifyall, this thread calls wait (forever)
411
            try
412
            {
413
              if (mResumeTimerTask.isCancelled())
414
              {
415
                mResumeTimerTask = new ResumeTimerTask();
416
                mTimer = new Timer();
417
                mResumeTimerTask.setTimer(mTimer);
418
              }
419
              mLogger.warn("allowedToProceed schedule passed " + resumeTime.value.getTime());
420
              mTimer.schedule(mResumeTimerTask, resumeTime.value);
421
            }
422
            catch( IllegalStateException e )
423
            {
424
              // this may be thrown by schedule if already scheduled
425
              // it signifies another BuildThread has already scheduled the ResumeTimerTask
426
               mLogger.warn("allowedToProceed already scheduled");
427
            }
428
 
429
            try
430
            {
431
              mLogger.warn("allowedToProceed wait");
432
              mSynchroniser.wait();
433
              mLogger.warn("allowedToProceed wait returned");
434
            }
435
            catch( InterruptedException e )
436
            {
437
              mLogger.warn("allowedToProceed caught InterruptedException");
438
              Thread.currentThread().interrupt();
439
            }
440
          }
441
 
442
        }
443
        else
444
        {
445
            //
446
            //  If commanded to pause, then wait around until the command has been removed.
447
            //
448
            mLogger.fatal("allowedToProceed calling mReleaseManager.queryDirectedRunLevel");                      
449
            if ( !mReleaseManager.queryDirectedRunLevel(mRconId) && !BuildDaemon.mShutDown )
450
            {
451
                mLogger.fatal("allowedToProceed changing run level to PAUSED for rcon_id " + mRconId);
452
                mReleaseManager.disconnect();
453
                mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_PAUSED);
454
 
455
                //  Sleep for periodicMs
456
                mLogger.warn("allowedToProceed sleep 1 mins directed downtime");
457
                BuildDaemon.daemonSleepSecs(60);
458
                mLogger.info("allowedToProceed sleep returned");
459
            }
460
            else
461
            {
462
                mReleaseManager.disconnect();
463
                proceed = true;
464
            }
465
        }
466
      }
467
    }
468
    finally
469
    {
470
      // this block is executed regardless of what happens in the try block
471
      // even if an exception is thrown
472
      // ensure disconnect
473
      mLogger.fatal("allowedToProceed calling mReleaseManager.disconnect");                      
474
      mReleaseManager.disconnect();
475
    }
476
 
477
    if (BuildDaemon.mShutDown)
478
    {
479
        mLogger.warn("allowedToProceed ShutDown requested");
480
        throw new ExitException();
481
    }
482
 
483
  }
484
 
485
/**periodically 
486
   * <br>a) performs disk housekeeping
487
   * <br>b) determines if a minimum threshold of disk space is available
488
   * <br>c) determines if a file can be touched
489
   * <br>d) Master: Checks that a ssh connection can be established to the package server
490
   * <br>changes the run level to CANNOT_CONTINUE if insufficient disk space
491
   * <br>otherwise changes the run level to ACTIVE and returns
492
   * <p>This function does not return until all check have passed
493
   * 
494
   * @return	true - The task was delayed until all checks passed. The caller may 
495
   * 				   wish to reevaluate the ability to continue the current task.
496
   * 
497
   */
498
  protected boolean checkEnvironment() throws Exception
499
  {
500
    mLogger.debug("checkEnvironment");
501
    boolean exit = false;
502
    boolean waitPerformed = false;
503
    String  reason;
504
 
505
    while( !exit )
506
    {
507
      mPhase.setPhase("checkEnvironment");
508
      exit = false;
509
      reason = "No Reason";
510
 
511
      //  Ugly Unit Testing Hook
512
      if ( mUnitTest.startsWith("unit test") )
513
      {
514
          if ( mUnitTest.startsWith("unit test check environment") )
515
          {
516
              if (mUnitTest.endsWith("Pass2"))
517
              {
518
                  exit = true;
519
              }
520
              else
521
              {
522
                  mUnitTest += ".Pass2";
523
                  reason = "Unit Test Check";
524
              }
525
          }
526
          else
527
          {
528
              exit= true;
529
          }
530
      }
531
      else
532
      {
533
          housekeep();
534
 
535
          //	Perform checks in order
536
          //	Failure of the first one terminates the check
537
          //
538
 
539
          if ( ! hasSufficientDiskSpace() )
540
          {
541
              mLogger.warn("checkEnvironment below disk free threshold");
542
              mPhase.setPhase("checkEnvironment:Wait for diskspace");
543
              reason = "Insufficient disk space";
544
          }
545
          else if (!touch() ) 
546
          {
547
              mLogger.warn("checkEnvironment read only file system detected");
548
              mPhase.setPhase("checkEnvironment:Wait for writable files system");
549
              reason = "Readonly file system";
550
          }
551
          else if ( !checkRemoteExecution() ) 
552
          {
553
              mLogger.warn("checkEnvironment Remote connection not established");
554
              mPhase.setPhase("checkEnvironment:Wait for remote execution");
555
              reason = "No remote connection";
556
          }
557
          else
558
          {
559
              exit = true;
560
              mPhase.setPhase("checkEnvironment");
561
              reason = "No Reason";
562
          }
563
      }
564
      if ( !exit )
565
      {
566
    	waitPerformed = true;
567
        mLogger.fatal("checkEnvironment changing run level to CANNOT_CONTINUE for rcon_id " + mRconId);
568
        mRunLevel.persistFault(mReleaseManager, mRconId, reason);
569
 
570
        if ( !mUnitTest.startsWith("unit test check environment") )
571
        {
572
          mLogger.warn("checkEnvironment sleep 5 mins");
573
          exit = BuildDaemon.daemonSleepSecs(5 * 60);
574
          mLogger.info("checkEnvironment sleep returned");
575
        }
576
      }
577
    }
578
 
579
    mLogger.fatal("checkEnvironment changing run level to ACTIVE for rcon_id " + mRconId);
580
    mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_ACTIVE);
581
 
582
    return waitPerformed;
583
  }
584
 
585
  /**	Perform check of the remote execution target machine
586
   * 	Used so that we know at the start of a build that the target machine can be contacted
587
   * 	Check is only done on the master
588
   * 
589
   * @return	true - the check has passed
590
   * 
591
   */
592
  private boolean checkRemoteExecution()
593
  {
594
	  if ( getMode() != 'M' || mUnitTest.startsWith("unit test") )
595
	  {
596
		  // Don't check if
597
		  //	This thread is not a master
598
		  //	This is a unit test
599
		  return true;
600
	  }
601
 
602
	  RemoteExecution mRemoteExecution = new RemoteExecution(new LogOutput(){
603
 
604
			@Override
605
			public void data(String message) {
606
				mLogger.info(message);
607
			}
608
 
609
			@Override
610
			public void fatal(String message) {
611
				mLogger.fatal(message);
612
			}
613
 
614
			@Override
615
			public void info(String message) {
616
				mLogger.info(message);
617
			}});
618
 
619
	  return mRemoteExecution.testConnection();
620
 
621
  }
622
 
623
 
624
  /**performs disk housekeeping which involves deleting build directories > 5 days old
625
   * refer to the sequence diagram check environment
626
   */
627
  private void housekeep()
628
  {
629
    mLogger.debug("housekeep");
630
    FilenameFilter filter = new FilenameFilter()
631
    {
632
 
633
      // Filter function to process the files processed within this routine
634
      public boolean accept(File file, String name)
635
      {
636
        mLogger.debug("accept " + name);
637
        boolean retVal = false;
638
 
639
        if ( file.isDirectory() && !name.startsWith( "." ) )
640
        {
641
          retVal = true;
642
        }
643
 
644
        mLogger.info("accept returned " + retVal);
645
        return retVal;
646
      }
647
    };
648
 
649
    try
650
    {
651
      // DEVI 46729, 46730, solaris 10 core dumps implicate deleteDirectory
652
      // let each BuildThread look after its own housekeeping
653
      String cwdPath = utilities.catDir(BuildDaemon.mGbeLog ,BuildDaemon.mHostname ,String.valueOf( mRtagId )); 
654
      File cwd = utilities.freshFile(cwdPath);
655
 
656
      File[] children = cwd.listFiles( filter );
657
 
658
      if ( children != null )
659
      {
660
        for ( int child=0; child < children.length; child++ )
661
        {
662
          // child is named uniquely to encapsulate a build
663
          // 5 days = 432,000,000 milliseconds
664
          if ( ( System.currentTimeMillis() - children[ child ].lastModified() ) > 432000000 )
665
          {
666
            // the directory is over 5 days old
667
            mLogger.warn("housekeep deleting directory " + children[ child ].getName());
668
            deleteDirectory( children[ child ] );
669
          }
670
        }
671
      }
672
    }
673
    catch( SecurityException e )
674
    {
675
      // this can be thrown by lastModified
676
      mLogger.warn("housekeep caught SecurityException");
677
    }
678
 
679
  }
680
 
681
  /**returns true if a file exists and can be deleted,
682
   * created and exists in the file system
683
   * this is to guard against read-only file systems
684
   */
685
  private boolean touch()
686
  {
687
    mLogger.debug("touch");
688
    boolean retVal = true;
689
 
690
    try
691
    {
692
      File touch = new File( String.valueOf( mRtagId ) + "touch" );
693
 
694
      if ( touch.exists() )
695
      {
696
        // delete it
697
        retVal = touch.delete();
698
      }
699
 
700
      if ( retVal )
701
      {
702
        // file does not exist
703
        retVal = touch.createNewFile();
704
      }
705
    }
706
    catch( SecurityException e )
707
    {
708
      // this can be thrown by exists, delete, createNewFile
709
      retVal = false;
710
      mLogger.warn("touch caught SecurityException");
711
    }
712
    catch( IOException e )
713
    {
714
      // this can be thrown by createNewFile
715
      retVal = false;
716
      mLogger.warn("touch caught IOException");
717
    }
718
 
719
    mLogger.info("touch returned " + retVal);
720
    return retVal;
721
  }
722
 
723
  /**returns true if free disk space > 5G
724
   * this may become configurable if the need arises
725
   * refer to the sequence diagram check environment
726
   */
727
  private boolean hasSufficientDiskSpace()
728
  {
729
      mLogger.debug("hasSufficientDiskSpace");
730
      boolean retVal = true;
731
      long freeSpace = 0;
732
 
733
      try
734
      {
735
          File cwd = new File( "." );
736
 
737
          // 5G = 5368709120 bytes
738
          // 1G = 1073741824 bytes - useful for testing
739
          freeSpace = cwd.getUsableSpace();
740
 
741
          if ( freeSpace < 5368709120L )
742
          {
743
              mLogger.warn("hasSufficientDiskSpace failed: " + cwd.getAbsolutePath() + " freeSpace " + freeSpace);
744
              retVal = false;
745
          }
746
      }
747
      catch( SecurityException e )
748
      {
749
          // this can be thrown by getFreeSpace
750
          mLogger.warn("hasSufficientDiskSpace caught SecurityException");
751
      }
752
 
753
      mLogger.info("hasSufficientDiskSpace returned " + retVal + " " + freeSpace);
754
      return retVal;
755
  }
756
 
757
  /**abstract method
758
   */
759
  @Override
760
  public abstract void run();
761
 
762
  /**   deletes directory and all its files
763
   *    Will set files and directories to be writable in case the user has messed with
764
   *    file permissions
765
   */
766
  FilenameFilter deleteDirectoryFilter = new FilenameFilter()
767
  {
768
    public boolean accept(File file, String name)
769
    {
770
      mLogger.debug("accept " + name);
771
      boolean retVal = false;
772
 
773
      if ( name.compareTo(".") != 0 && ( name.compareTo("..") != 0 ) )
774
      {
775
        retVal = true;
776
      }
777
 
778
      mLogger.info("accept returned " + retVal);
779
      return retVal;
780
    }
781
  };
782
 
783
 
784
  @SuppressWarnings("squid:S899")  
785
  protected void deleteDirectory(File directory)
786
  {
787
    mLogger.debug("deleteDirectory " + directory.getName());
788
    try
789
    {
790
      if ( directory.exists() )
791
      {
792
 
793
        //
794
        directory.setWritable(true);
795
        File[] children = directory.listFiles( deleteDirectoryFilter );
796
 
797
        if ( children != null )
798
        {
799
          for ( int child=0; child < children.length; child++ )
800
          {
801
            if ( children[ child ].isDirectory() )
802
            {
803
              deleteDirectory( children[ child ] );
804
            }
805
            else
806
            {
807
                children[ child ].setWritable(true);
808
                children[ child ].delete();
809
            }
810
          }
811
        }
812
        directory.delete();
813
      }
814
    }
815
    catch( SecurityException e )
816
    {
817
      // this can be thrown by exists and delete
818
       mLogger.warn("deleteDirectory caught SecurityException");
819
    }
820
  }
821
 
822
  /**
823
   * abstract method. Re-implemented by each class
824
   * Returns the Mode in which the thread is operating.
825
   * 
826
   * @return M if Master, S if Slave
827
   */
828
  protected abstract char getMode();
829
 
830
  /** Save the buildfile to disk
831
   *  @param buildFileContent The generated build file that will control this run
832
 * @throws Exception 
833
   * 
834
   */
835
  protected void saveBuildFile(String buildFileContent) throws BuildSystemException
836
  {
837
      mReporting.buildFailureLogFile = null;
838
      mErrorReported = false;
839
 
840
      mBuildFile.delete();
841
      FileWriter buildFileWriter = null;
842
      BufferedReader buildFileBufferedReader = null;
843
      boolean errorSeen = false;
844
 
845
      if ( buildFileContent != null ) {
846
 
847
          try {
848
 
849
              StringReader buildFileContentStringReader = new StringReader(buildFileContent);
850
              buildFileBufferedReader = new BufferedReader(buildFileContentStringReader);
851
 
852
              // Sanitize the buildFileContent
853
              //      it may contain line.separators of "\n", "\r", or "\r\n" variety, 
854
              //      depending on the location of the ripple engine
855
              StringBuilder  sanitisedBFC = new StringBuilder ();
856
              String lf = System.getProperty("line.separator") ;
857
              String line;
858
 
859
              while( ( line = buildFileBufferedReader.readLine() ) != null)
860
              {
861
                  sanitisedBFC.append (line).append(lf);
862
              }
863
              buildFileBufferedReader.close();
864
              buildFileBufferedReader = null;
865
 
866
              buildFileWriter = new FileWriter(mBuildFile);
867
              buildFileWriter.write(sanitisedBFC.toString());
868
              buildFileWriter.close();
869
              buildFileWriter = null;
870
 
871
          }
872
          catch( IOException e )
873
          {
874
              mLogger.error("SaveBuildFile caught IOException");
875
              errorSeen = true;
876
 
877
          }
878
          finally
879
          {
880
              if ( buildFileWriter!= null ) {
881
                  try {
882
                    buildFileWriter.close();
883
                } catch (IOException e) {
884
                    errorSeen = true;
885
 
886
                }
887
              }
888
              if ( buildFileBufferedReader != null) {
889
                  try {
890
                    buildFileBufferedReader.close();
891
                } catch (IOException e) {
892
                    errorSeen = true;
893
                }
894
              }
895
          }
896
      }
897
 
898
      if (errorSeen) {
899
          mBuildFile.delete();
900
          throw new BuildSystemException("SaveBuildFile caught FileNotFoundException");    
901
      }
902
  }
903
 
904
  /** Delete the build file
905
   * 
906
   */
907
  protected void deleteBuildFile()
908
  {
909
      if ( mBuildFile.exists() )
910
      {
911
          mBuildFile.delete(); //NOSONAR
912
      }    
913
  }
914
 
915
  /** Parse the ANT project
916
   *  Done once per build - useful information is extracted for use within the build
917
 * @param updateReporting 
918
   */
919
  protected void parseBuildFile(boolean updateReporting)
920
  {
921
      mLogger.debug("parseBuildFile");
922
 
923
      //try
924
      {
925
          mProject = null;
926
          if (updateReporting) {
927
              mReporting.resetData();
928
          }
929
 
930
          if ( mBuildFile.exists() )
931
          {
932
              mProject = new Project();
933
              mProject.setProperty("ant.file", mBuildFile.getAbsolutePath());
934
 
935
              // Add listener for logging the complete build process
936
              // If the daemon has been restarted, then the listener will not have been
937
              // set up - so don't add it. Perhaps we need a way to open an existing
938
              // logfile to append.
939
              if ( mBuildLogger != null )               {
940
                  mProject.addBuildListener(mBuildLogger);
941
              }
942
 
943
              // parse can throw BuildException, this is serious
944
              //  Generate a general exception to force an indefinite pause
945
              //  Delete build file to prevent attempts to use it 
946
              try {
947
                  mProject.init();
948
                  ProjectHelper pH = ProjectHelper.getProjectHelper();
949
                  mProject.addReference("ant.projectHelper", pH);
950
                  pH.parse(mProject, mBuildFile);
951
              }
952
              catch (BuildException be)
953
              {
954
                  mLogger.fatal("Parse Error:" + be.toString() );
955
 
956
                  mBuildFile.delete(); //NOSONAR
957
                  throw new BuildToolException("Error parsing build file:"+ be.getCause());
958
              }
959
 
960
              if (updateReporting) {
961
                  // set up project properties for reporting purposes
962
                  // this first group are hard coded in the build file
963
 
964
                  mReporting.rtagId = mRtagId;
965
                  mReporting.buildRef = mProject.getProperty("abt_daemon");
966
                  mReporting.packageName = mProject.getProperty("abt_package_name");
967
                  mReporting.packageVersion = mProject.getProperty("abt_package_version");
968
                  mReporting.packageExtension = mProject.getProperty("abt_package_extension");
969
                  mReporting.packageLocation = mProject.getProperty("basedir") + mProject.getProperty("abt_package_location");
970
                  mReporting.packageDepends = mProject.getProperty("abt_package_depends");
971
                  mReporting.packageOwners = mProject.getProperty("abt_package_ownerlist");
972
                  mReporting.packageBuildInfo = mProject.getProperty("abt_package_build_info");
973
                  mReporting.isRipple = ReportingData.toBool(mProject.getProperty("abt_is_ripple"));
974
                  mReporting.buildReason = ReportingData.toBuildReason(mProject.getProperty("abt_build_reason"));
975
                  mReporting.packageVersionId = ReportingData.toInt(mProject.getProperty("abt_package_version_id"), -1);
976
                  mReporting.doesNotRequireSourceControlInteraction = ReportingData.toBool(mProject.getProperty("abt_does_not_require_source_control_interaction"));
977
                  mReporting.isaTestBuild = ReportingData.toBool (mProject.getProperty("abt_test_build_instruction"));
978
              }
979
              //
980
              //  Log the abt_daemon value so that we can trace the build log
981
              //  Build logs from all machines are stored with this number
982
              //  Only do it at the start of the build
983
 
984
              mLogger.fatal("BuildRef: " + mReporting.buildRef );    
985
 
986
          }
987
      }
988
  }
989
 
990
 
991
  /**
992
   * builds a buildFile from the buildFileContent
993
   * triggers ant to operate on the buildFile
994
   * 
995
   * @param target           The command to be run. This is one of the commands known to the ABT class
996
   * @param updateReporting  True to update the mReportingData from the buildfiles properties
997
   * @param forceOpr         True - force operation on errors
998
   * @throws Exception
999
   * @throws BuildException 
1000
   * @return Success - False on error             
1001
   */
1002
  protected boolean deliverChange(String target, boolean updateReporting, boolean forceOpr) throws Exception
1003
  {
1004
      mLogger.debug("deliverChange");
1005
 
1006
      //  Bypass the operation if an error is being propagated
1007
      //      Always perform a AbtSetUp and AbtTearDown
1008
      //
1009
      if (!forceOpr) {
1010
          if ( mErrorReported ) {
1011
              // SaveBuildFile, AbtTestPath, AbtSetUp or the build failed
1012
              // the default target will inevitably fail and will generate further email if allowed to proceed
1013
              // do not mask the root cause
1014
              mLogger.debug("deliverChange - Error already detected");
1015
              return false;    
1016
          }
1017
      }
1018
 
1019
      //  Parse the build file
1020
      parseBuildFile(updateReporting);
1021
      if (mProject != null)
1022
      {
1023
 
1024
          if ( target == null )     {
1025
              target = mProject.getDefaultTarget();
1026
          }
1027
          mLogger.warn("deliverChange ant launched: " + mBuildFile.getAbsolutePath() + " Target:" + target);
1028
 
1029
          try {
1030
              mProject.executeTarget(target);
1031
              mLogger.warn("deliverChange ant returned");
1032
          }
1033
 
1034
          //   
1035
          //  Do not catch ALL exceptions. The MasterThread::run and SlaveThread::run
1036
          //  relies on exceptions propagating upwards for the indefinite pause feature
1037
          //
1038
          catch( BuildException e )
1039
          {
1040
 
1041
              if ( mReporting.buildFailureLogFile == null )
1042
              {
1043
                  mReporting.buildFailureLogFile = e.getMessage();
1044
              }
1045
              mLogger.debug("deliverChange caught BuildException, big deal, the build failed " + mReporting.buildFailureLogFile);
1046
              mErrorReported = true;
1047
          }
1048
 
1049
 
1050
          if (updateReporting )
1051
          {
1052
              // this group is set at run time (by the AbtPublish target only)
1053
              // they will be null for every other target,
1054
              // and null if an error occurs in the AbtPublish target
1055
              mReporting.isFullyPublished = ReportingData.toBool(mProject.getProperty("abt_fully_published"));
1056
              mReporting.newVcsTag = mProject.getProperty("abt_new_vcstag");
1057
          }
1058
      }
1059
 
1060
      return !mErrorReported;
1061
  }
1062
 
1063
   /**
1064
    * eMail an indefinite pause notification
1065
    * 
1066
    * @param cause  Reason for the pause
1067
    */
1068
   protected void notifyIndefinitePause(String cause)
1069
   {
1070
     mLogger.debug("indefinitePause");
1071
 
1072
     String body =
1073
     "Hostname: " + BuildDaemon.mHostname + "<p>" +
1074
     "Release: " + mRippleEngine.mBaselineName + "<p>" +
1075
     "Cause: " + cause + "<p>" +
1076
     "RmRef: "   + CreateUrls.generateReleaseUrl(mRippleEngine.getRtagId()) +"<p>";
1077
 
1078
     try
1079
     {
1080
       Smtpsend.send(
1081
       mRippleEngine.getMailServer(),       // mailServer
1082
       mRippleEngine.getMailSender(),       // source
1083
       mRippleEngine.getMailGlobalTarget(), // target (list)
1084
       null,                            // cc
1085
       null,                            // bcc
1086
       "BUILD DAEMON Indefinite Pause", // subject
1087
       body,                            // body
1088
       null                             // attachment
1089
       );
1090
     }
1091
     catch( Exception e )
1092
     {
1093
     }
1094
   }
1095
 
1096
 
1097
   /**
1098
    * eMail an indefinite pause recovery notification
1099
    */
1100
   private void notifyIndefiniteRecovery()
1101
   {
1102
       mLogger.debug("indefiniteRecovery");
1103
 
1104
       String body =
1105
       "Hostname: " + BuildDaemon.mHostname + "<p>" +
1106
       "Release: "  + mRippleEngine.mBaselineName + "<p>" +
1107
       "Cause: "    + "Recovery of previous condition" + "<p>" +
1108
       "RmRef: "    + CreateUrls.generateReleaseUrl(mRippleEngine.getRtagId()) +"<p>";
1109
 
1110
       try
1111
       {
1112
         Smtpsend.send(
1113
         mRippleEngine.getMailServer(),                 // mailServer
1114
         mRippleEngine.getMailSender(),                 // source
1115
         mRippleEngine.getMailGlobalTarget(),           // target (list)
1116
         null,                                      // cc
1117
         null,                                      // bcc
1118
         "BUILD DAEMON Indefinite Pause Recovery",  // subject
1119
         body,                                      // body
1120
         null                                       // attachment
1121
         );
1122
       }
1123
       catch( Exception e )
1124
       {
1125
       }
1126
 
1127
   }
1128
 
1129
 
1130
    /**
1131
     *  Nagios interface
1132
     *      Returns true if the thread looks OK
1133
     * @param nagInfo 
1134
    */
1135
    boolean checkThread(NagiosInfo nagInfo)
1136
    {
1137
      boolean retVal = true;
1138
      if ( mRunLevel.mBuildState == BuildState.DB_CANNOT_CONTINUE ) {
1139
        retVal = false;
1140
        nagInfo.extendedReason.add("[" + mRtagId + "] " + mRunLevel.toString());
1141
      } else  {
1142
        retVal = checkThreadExtended(nagInfo);
1143
      }
1144
 
1145
      if (this.getMode() == 'M') {
1146
          nagInfo.masterCount++;
1147
      } else {
1148
          nagInfo.slaveCount++;
1149
      }
1150
 
1151
      mLogger.warn("checkThread[" + mRtagId + "] returned " + retVal);
1152
      return retVal;
1153
    }
1154
 
1155
    /**
1156
     * Nagios interface extension
1157
     * This method should be overridden by classes that extend this class
1158
     * If not overridden then the test indicates OK
1159
     *
1160
     *      Returns true if the thread looks OK
1161
    */
1162
    boolean checkThreadExtended(NagiosInfo nagInfo)
1163
    {
1164
        boolean retVal = mPhase.isHappy();
1165
        if ( ! retVal) {
1166
            nagInfo.extendedReason.add("[" + mRtagId + "] " + mPhase.happyText());
1167
        }
1168
        mLogger.info("checkThreadExtended: " + retVal + ":" + mPhase.toStringSecs() );
1169
        return retVal;
1170
    }
1171
 
1172
 
1173
    /**
1174
     * Nagios interface extension
1175
     * This method should be overridden by classes that extend this class
1176
     * If not overridden then the test indicates OK
1177
     *
1178
     *      Returns true if the thread looks OK
1179
     * @param wr Output buffer
1180
     * @param nagInfo 
1181
     * @param estatus 
1182
     * @throws IOException 
1183
    */
1184
    public void extendedStatus(NagiosInfo nagInfo, Map<String, Object> estatus)
1185
    {
1186
        LinkedHashMap<String, Object> data = new LinkedHashMap<String, Object>();
1187
        estatus.put(String.valueOf(mRtagId), data);
1188
 
1189
        data.put("rtagId", Integer.valueOf(mRtagId));
1190
        data.put("rconId", Integer.valueOf(mRconId));
1191
        data.put("Thread Start Text", new SimpleDateFormat("dd-MM-yyyy HH:mm:ss").format(new Date(mStartTime)));
1192
        data.put("Thread Start", Long.valueOf(mStartTime));
1193
        data.put("Thread Alive:", Boolean.valueOf(this.isAlive() ));
1194
        data.put("Thread Check:", Boolean.valueOf(checkThread(nagInfo) ));
1195
        data.put("Mode" , String.valueOf(this.getMode() ));
1196
        data.put("mRunLevel" , mRunLevel.toString() );
1197
        data.put("mLastBuildWasBenign" , Boolean.valueOf(mLastBuildWasBenign));
1198
        data.put("mException" , Boolean.valueOf(mException));
1199
        data.put("mErrorReported" , Boolean.valueOf(mErrorReported));
1200
        data.put("mRecoverable", Boolean.valueOf(mRecoverable));
1201
        data.put("mPhase" ,  mPhase.sText);
1202
        data.put("mPhaseDelta" ,  mPhase.getDeltaSecs());
1203
        if ( mReleaseManager != null)
1204
            data.put("Rm Mutex" , mReleaseManager.mMutexState );
1205
        else
1206
            data.put("Rm Mutex" , null );
1207
    }
1208
 
1209
}