| 6914 |
dpurdie |
1 |
package com.erggroup.buildtool.daemon;
|
|
|
2 |
|
|
|
3 |
import com.erggroup.buildtool.abt.BuildToolException;
|
|
|
4 |
import com.erggroup.buildtool.abt.RemoteExecution;
|
|
|
5 |
import com.erggroup.buildtool.abt.RemoteExecution.LogOutput;
|
|
|
6 |
import com.erggroup.buildtool.daemon.BuildThread;
|
|
|
7 |
import com.erggroup.buildtool.ripple.*;
|
|
|
8 |
import com.erggroup.buildtool.ripple.BuildFile.BuildFileState;
|
|
|
9 |
import com.erggroup.buildtool.ripple.Package;
|
|
|
10 |
import com.erggroup.buildtool.ripple.ReleaseManager.BuildResult;
|
|
|
11 |
import com.erggroup.buildtool.ripple.RunLevel.BuildState;
|
|
|
12 |
import com.erggroup.buildtool.smtp.CreateUrls;
|
|
|
13 |
import com.erggroup.buildtool.smtp.Smtpsend;
|
|
|
14 |
import com.erggroup.buildtool.utilities.*;
|
|
|
15 |
|
|
|
16 |
import java.io.BufferedReader;
|
|
|
17 |
import java.io.DataInputStream;
|
|
|
18 |
import java.io.File;
|
|
|
19 |
import java.io.FileInputStream;
|
|
|
20 |
import java.io.InputStreamReader;
|
|
|
21 |
import java.sql.SQLException;
|
|
|
22 |
import java.util.Iterator;
|
|
|
23 |
|
|
|
24 |
import org.apache.log4j.Logger;
|
|
|
25 |
|
|
|
26 |
/**Master Thread sub component
|
|
|
27 |
*/
|
|
|
28 |
public class MasterThread extends BuildThread
|
|
|
29 |
{
|
|
|
30 |
|
|
|
31 |
/**Logger
|
|
|
32 |
* @attribute
|
|
|
33 |
*/
|
|
|
34 |
private static final Logger mLogger = Logger.getLogger(MasterThread.class);
|
|
|
35 |
|
|
|
36 |
/** Current BuildFile
|
|
|
37 |
*
|
|
|
38 |
*/
|
|
|
39 |
private BuildFile mMasterBuildFile = new BuildFile();
|
|
|
40 |
|
|
|
41 |
/** Measure the duration of the current build
|
|
|
42 |
*
|
|
|
43 |
*/
|
|
|
44 |
private ElapseTime mBuildDuration = new ElapseTime();
|
|
|
45 |
|
|
|
46 |
/**constructor
|
|
|
47 |
* @param rtagId - Identifies the Release
|
|
|
48 |
* @param rconId - Identifies the Controlling entry
|
|
|
49 |
* @param releaseManager - A ReleaseManager entry to use
|
|
|
50 |
* @param unitTest - Unit Test data
|
|
|
51 |
*/
|
|
|
52 |
public MasterThread(int rtagId, int rconId, ReleaseManager releaseManager, String unitTest)
|
|
|
53 |
{
|
|
|
54 |
super(rtagId, rconId, releaseManager, unitTest);
|
|
|
55 |
mLogger.warn("MasterThread rtag_id " + rtagId + " rcon_id " + rconId);
|
|
|
56 |
}
|
|
|
57 |
|
|
|
58 |
/**implements the sequence diagrams coordinate slave threads, generate build files, allowed to proceed, check environment
|
|
|
59 |
*/
|
|
|
60 |
public void run()
|
|
|
61 |
{
|
|
|
62 |
Integer id = Integer.valueOf(mRtagId);
|
|
|
63 |
setName(id.toString());
|
|
|
64 |
mLogger.warn("Master Thread run");
|
|
|
65 |
boolean exit = false;
|
|
|
66 |
mRippleEngine = new RippleEngine(mReleaseManager, mRtagId, true);
|
|
|
67 |
|
|
|
68 |
while(!exit)
|
|
|
69 |
{
|
|
|
70 |
try
|
|
|
71 |
{
|
|
|
72 |
mLogger.fatal("run calling sleepCheck");
|
|
|
73 |
mPhase.setPhase("sleepCheck");
|
|
|
74 |
sleepCheck();
|
|
|
75 |
|
|
|
76 |
mLogger.fatal("run calling rippleEngine.collectMetaData");
|
|
|
77 |
mPhase.setPhase("collectMetaData", 600);
|
|
|
78 |
mRippleEngine.collectMetaData();
|
|
|
79 |
|
|
|
80 |
if ( Thread.currentThread().isInterrupted() )
|
|
|
81 |
{
|
|
|
82 |
mLogger.warn("run is interrupted");
|
|
|
83 |
// unit test technique
|
|
|
84 |
throw new ExitException();
|
|
|
85 |
}
|
|
|
86 |
|
|
|
87 |
if ( mUnitTest.compareTo("unit test spawn thread") == 0)
|
|
|
88 |
{
|
|
|
89 |
throw new Exception();
|
|
|
90 |
}
|
|
|
91 |
|
|
|
92 |
if ( !mUnitTest.startsWith("unit test check environment"))
|
|
|
93 |
{
|
|
|
94 |
if ( mUnitTest.compareTo("unit test generate build files") != 0)
|
|
|
95 |
{
|
|
|
96 |
if ((mUnitTest.compareTo("unit test allowed to proceed") != 0) &&
|
|
|
97 |
(mUnitTest.compareTo("unit test not allowed to proceed") != 0) &&
|
|
|
98 |
(mUnitTest.compareTo("unit test exit") != 0))
|
|
|
99 |
{
|
|
|
100 |
//---------------------------------------------------------------
|
|
|
101 |
// Wait for all Slaves to enter waiting or paused state
|
|
|
102 |
// They may be building
|
|
|
103 |
// They may be starting up
|
|
|
104 |
//
|
|
|
105 |
mLogger.fatal("run changing run level to WAITING (New) for rcon_id " + mRconId);
|
|
|
106 |
mPhase.setPhase("Wait for Slave Threads");
|
|
|
107 |
mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_WAITING);
|
|
|
108 |
|
|
|
109 |
while (true)
|
|
|
110 |
{
|
|
|
111 |
StringBuilder waitList = new StringBuilder();
|
|
|
112 |
mLogger.info("run calling mReleaseManager.queryRunLevel");
|
|
|
113 |
mReleaseManager.queryRunLevel(mRtagId);
|
|
|
114 |
|
|
|
115 |
// Process all run entries
|
|
|
116 |
|
|
|
117 |
for (Iterator<RunLevelData> it = mReleaseManager.mRunLevelCollection.iterator(); it.hasNext(); )
|
|
|
118 |
{
|
|
|
119 |
RunLevelData rl = it.next();
|
|
|
120 |
|
|
|
121 |
//
|
|
|
122 |
// Only examine Slaves.
|
|
|
123 |
// This is being run on a Master and at this point the Master is not waiting
|
|
|
124 |
// If there are multiple (misconfigured) Masters, then we can get confused
|
|
|
125 |
//
|
|
|
126 |
// If I am no longer a Master we will process till the end of the build cycle
|
|
|
127 |
// at which point it will be detected.
|
|
|
128 |
//
|
|
|
129 |
// If there are other Masters, then we assume that they are not waiting
|
|
|
130 |
//
|
|
|
131 |
|
|
|
132 |
// Ignore myself. I am not waiting. I am asking the questions
|
|
|
133 |
if ( rl.get_rcon_id() == mRconId) {
|
|
|
134 |
continue;
|
|
|
135 |
}
|
|
|
136 |
|
|
|
137 |
// Ignore other Master. They should not be waiting.
|
|
|
138 |
if ( rl.isMaster()) {
|
|
|
139 |
continue;
|
|
|
140 |
}
|
|
|
141 |
|
|
|
142 |
//
|
|
|
143 |
// A slave that is 'Paused' is not processing a package
|
|
|
144 |
// It will have been included in the build-set, so we must wait for
|
|
|
145 |
// the pause to be removed.
|
|
|
146 |
//
|
|
|
147 |
// A slave that is 'Waiting' must also have no BuildFile present
|
|
|
148 |
// This signals that the buildfile has been taken by the slave
|
|
|
149 |
//
|
|
|
150 |
if (! (rl.isAt(BuildState.DB_WAITING) && !rl.hasBuildFile() ) )
|
|
|
151 |
{
|
|
|
152 |
// A non-waiting slave has been found
|
|
|
153 |
waitList.append(" ");
|
|
|
154 |
waitList.append(rl.get_rcon_id());
|
|
|
155 |
}
|
|
|
156 |
}
|
|
|
157 |
|
|
|
158 |
|
|
|
159 |
if ( waitList.length() > 0 )
|
|
|
160 |
{
|
|
|
161 |
// One or more slaves still processing ...
|
|
|
162 |
if ( mUnitTest.compareTo("unit test coordinate slave threads") == 0 )
|
|
|
163 |
{
|
|
|
164 |
Thread.currentThread().interrupt();
|
|
|
165 |
}
|
|
|
166 |
else
|
|
|
167 |
{
|
|
|
168 |
|
|
|
169 |
mLogger.warn("Waiting 3 seconds for slaves:" + waitList.toString() );
|
|
|
170 |
Thread.sleep(3000);
|
|
|
171 |
mLogger.info("run sleep returned");
|
|
|
172 |
}
|
|
|
173 |
continue;
|
|
|
174 |
}
|
|
|
175 |
else
|
|
|
176 |
{
|
|
|
177 |
// All slaves have completed there tasks
|
|
|
178 |
break;
|
|
|
179 |
}
|
|
|
180 |
}
|
|
|
181 |
|
|
|
182 |
if ( mUnitTest.compareTo("unit test coordinate slave threads") == 0 )
|
|
|
183 |
{
|
|
|
184 |
throw new ExitException();
|
|
|
185 |
}
|
|
|
186 |
|
|
|
187 |
//---------------------------------------------------------------
|
|
|
188 |
// Publish Package Build Results
|
|
|
189 |
// Indicate to RM that we are publishing
|
|
|
190 |
// Ensure that all build machines have placed a marker in dpkg_archive
|
|
|
191 |
// Save updated build files into VCS
|
|
|
192 |
//
|
|
|
193 |
mLogger.fatal("run changing run level to PUBLISHING for rcon_id " + mRconId);
|
|
|
194 |
mPhase.setPhase("Publishing");
|
|
|
195 |
mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_PUBLISHING);
|
|
|
196 |
|
|
|
197 |
mLogger.info("run calling deliverChange on AbtPublish");
|
|
|
198 |
deliverChange("AbtPublish", true, false);
|
|
|
199 |
|
|
|
200 |
//---------------------------------------------------------------
|
|
|
201 |
// TearDown the build workspace
|
|
|
202 |
// Remove the build workspace
|
|
|
203 |
// Save build log files
|
|
|
204 |
// Don't update the Reporting Information as it will null out
|
|
|
205 |
// some essential information that has been collected.
|
|
|
206 |
//
|
|
|
207 |
mLogger.info("run calling deliverChange on AbtTearDown");
|
|
|
208 |
mPhase.setPhase("AbtTearDown");
|
|
|
209 |
deliverChange("AbtTearDown", false, true);
|
|
|
210 |
|
|
|
211 |
// Delete the build file now
|
|
|
212 |
// This will prevent any chance of multiply publishing a package version
|
|
|
213 |
deleteBuildFile();
|
|
|
214 |
|
|
|
215 |
|
|
|
216 |
//---------------------------------------------------------------
|
|
|
217 |
// Update Release Manager (publish to Release Manager)
|
|
|
218 |
//
|
|
|
219 |
mPhase.setPhase("Update Release Manager");
|
|
|
220 |
if ( mReporting.packageName != null )
|
|
|
221 |
{
|
|
|
222 |
// A dummy build file did not apply
|
|
|
223 |
// Publishing is done outside ant by design
|
|
|
224 |
// Do ALL Release Manager work outside ant (and JATS)
|
|
|
225 |
|
|
|
226 |
boolean buildErrorOccurred = true;
|
|
|
227 |
boolean publishError = true;
|
|
|
228 |
|
|
|
229 |
try
|
|
|
230 |
{
|
|
|
231 |
// Create a build instance entry in the Release Manager database
|
|
|
232 |
// Create one per build
|
|
|
233 |
// mReporting.packageVersionId will have been updated if this is a new package-version
|
|
|
234 |
// Save the build id
|
|
|
235 |
mPhase.setPhase("Create Build Instance");
|
|
|
236 |
mReporting.buildId = mReleaseManager.createBuildInstance(
|
|
|
237 |
mRtagId,
|
|
|
238 |
mReporting.packageVersionId,
|
|
|
239 |
mReporting.buildReason );
|
|
|
240 |
//
|
|
|
241 |
// Locate and collect the unit test results
|
|
|
242 |
//
|
|
|
243 |
mPhase.setPhase("Parse Unit Test Results");
|
|
|
244 |
BuildTestResults btr = new BuildTestResults(System.getenv("GBE_DPKG"), mReporting, mLogger);
|
|
|
245 |
mReleaseManager.insertTestResults(mReporting.buildId, btr);
|
|
|
246 |
|
|
|
247 |
if ( mReporting.isaTestBuild )
|
|
|
248 |
{
|
|
|
249 |
|
|
|
250 |
// Test Build
|
|
|
251 |
// Notify the user requesting the build
|
|
|
252 |
// Build Failure emails have already been sent
|
|
|
253 |
//
|
|
|
254 |
mLogger.info("run completing test build on " + mReporting.packageName + mReporting.packageExtension);
|
|
|
255 |
mPhase.setPhase("Report Test Build Complete");
|
|
|
256 |
Package p = mRippleEngine.findPackage(mReporting.packageName + mReporting.packageExtension);
|
|
|
257 |
if (p != ReleaseManager.NULL_PACKAGE)
|
|
|
258 |
{
|
|
|
259 |
mLogger.fatal("run calling p.completeTestBuild");
|
|
|
260 |
p.completeTestBuild(mRippleEngine, mReporting.isFullyPublished);
|
|
|
261 |
}
|
|
|
262 |
|
|
|
263 |
//
|
|
|
264 |
// Determine the total time taken by the build
|
|
|
265 |
// Done as late a possible in the process
|
|
|
266 |
// Does not include planning time
|
|
|
267 |
// Does not handle case where build cycle is broken
|
|
|
268 |
// Only record for successful tests. Don't want to record short build times on failure
|
|
|
269 |
// in particular if the package has been released. Would be OK for a WIP, but how to identify these
|
|
|
270 |
if ( mReporting.isFullyPublished)
|
|
|
271 |
{
|
|
|
272 |
int buildDuration = mBuildDuration.toIntSecs();
|
|
|
273 |
mLogger.fatal("Build Duration:" + buildDuration);
|
|
|
274 |
mReleaseManager.updateBuildDuration(mReporting.packageVersionId, buildDuration);
|
|
|
275 |
}
|
|
|
276 |
|
|
|
277 |
mPhase.setPhase("Update Build Instance for Test");
|
|
|
278 |
mReleaseManager.updateBuildInstance(mReporting.buildId, 0, mReporting.isFullyPublished ? BuildResult.Complete :BuildResult.BuildError );
|
|
|
279 |
|
|
|
280 |
// ... and clean up the mess in the archive
|
|
|
281 |
publishError = false;
|
|
|
282 |
throw new Exception();
|
|
|
283 |
}
|
|
|
284 |
|
|
|
285 |
// Package not fully published
|
|
|
286 |
// An error has occurred in the building of the package
|
|
|
287 |
if ( !mReporting.isFullyPublished )
|
|
|
288 |
{
|
|
|
289 |
mLogger.info("run build error occurred on " + mReporting.packageName + mReporting.packageVersion);
|
|
|
290 |
|
|
|
291 |
// ... and clean up the mess in the archive
|
|
|
292 |
publishError = false;
|
|
|
293 |
throw new Exception();
|
|
|
294 |
}
|
|
|
295 |
|
|
|
296 |
//---------------------------------------------------------------
|
|
|
297 |
// Package was successfully built and published into dpkg_archive by all daemons
|
|
|
298 |
// Update Release Manager - formally release the package
|
|
|
299 |
// Insert Package Metrics
|
|
|
300 |
// Insert Unit Test Results
|
|
|
301 |
buildErrorOccurred = false;
|
|
|
302 |
|
|
|
303 |
//
|
|
|
304 |
// Detect Badly formed VCS label returned by the builder
|
|
|
305 |
// Have had zero length values returned
|
|
|
306 |
// Treat as a publishing error
|
|
|
307 |
//
|
|
|
308 |
if ( mReporting.newVcsTag == null || mReporting.newVcsTag.isEmpty() )
|
|
|
309 |
{
|
|
|
310 |
mLogger.fatal("run package not labelled in Version Control on " + mReporting.packageName + mReporting.packageVersion);
|
|
|
311 |
mReporting.errMsg = "Error publishing to Version Control System"; // Max 50 Characters
|
|
|
312 |
throw new Exception();
|
|
|
313 |
}
|
|
|
314 |
|
|
|
315 |
|
|
|
316 |
// Publish to release manager
|
|
|
317 |
// On error mReporting.errMsg will contain error string
|
|
|
318 |
// Publishing errors only affect the current package
|
|
|
319 |
//
|
|
|
320 |
mLogger.fatal("run calling mReleaseManager.autoMakeRelease");
|
|
|
321 |
mPhase.setPhase("autoMakeRelease");
|
|
|
322 |
publishError = mReleaseManager.autoMakeRelease(mReporting);
|
|
|
323 |
if ( publishError )
|
|
|
324 |
{
|
|
|
325 |
mLogger.fatal("autoMakeRelease publishError: " + mReporting.errMsg);
|
|
|
326 |
throw new Exception();
|
|
|
327 |
}
|
|
|
328 |
|
|
|
329 |
//
|
|
|
330 |
// Determine the total time taken by the build
|
|
|
331 |
// Done as late a possible in the process
|
|
|
332 |
// Does not include planning time
|
|
|
333 |
// Does not handle case where build cycle is broken
|
|
|
334 |
//
|
|
|
335 |
int buildDuration = mBuildDuration.toIntSecs();
|
|
|
336 |
mLogger.fatal("Build Duration:" + buildDuration);
|
|
|
337 |
mReleaseManager.updateBuildDuration(mReporting.packageVersionId, buildDuration);
|
|
|
338 |
|
|
|
339 |
//
|
|
|
340 |
// Update the Build Instance information
|
|
|
341 |
// If this has been a ripple then the pv_id of the build needs to be associated with
|
|
|
342 |
// the newly created version
|
|
|
343 |
|
|
|
344 |
mPhase.setPhase("Update Build Instance");
|
|
|
345 |
mReleaseManager.updateBuildInstance(mReporting.buildId, mReporting.packageVersionId, BuildResult.Complete );
|
|
|
346 |
|
|
|
347 |
//
|
|
|
348 |
// Insert package metrics into Release Manager
|
|
|
349 |
// Metrics have been placed in a file in a format suitable for RM
|
|
|
350 |
//
|
|
|
351 |
mReporting.errMsg = "Error publishing Package Metrics"; // Max 50 Characters
|
|
|
352 |
mPhase.setPhase("Insert Metrics");
|
|
|
353 |
FileInputStream abtmetrics = new FileInputStream( String.valueOf(mRtagId) + "abtmetrics.txt" );
|
|
|
354 |
DataInputStream din = new DataInputStream( abtmetrics );
|
|
|
355 |
InputStreamReader isr = new InputStreamReader( din );
|
|
|
356 |
BufferedReader br = new BufferedReader( isr );
|
|
|
357 |
String metrics = br.readLine();
|
|
|
358 |
mLogger.warn( "execute read metrics string " + metrics );
|
|
|
359 |
br.close();
|
|
|
360 |
isr.close();
|
|
|
361 |
din.close();
|
|
|
362 |
|
|
|
363 |
mLogger.fatal("run calling mReleaseManager.insertPackageMetrics");
|
|
|
364 |
mReleaseManager.insertPackageMetrics(mRtagId, mReporting.packageName, mReporting.packageExtension, metrics );
|
|
|
365 |
|
|
|
366 |
//
|
|
|
367 |
// Info reporting
|
|
|
368 |
// Send an email to track build system usage - may be noise
|
|
|
369 |
//
|
|
|
370 |
mReporting.errMsg = null;
|
|
|
371 |
mLogger.fatal("run calling emailBuildComplete");
|
|
|
372 |
mPhase.setPhase("Post Build Email");
|
|
|
373 |
emailPostBuild(mRippleEngine, false);
|
|
|
374 |
|
|
|
375 |
//---------------------------------------------------------------
|
|
|
376 |
// All done
|
|
|
377 |
// The package has been built and released
|
|
|
378 |
mPhase.setPhase("Build Complete");
|
|
|
379 |
}
|
|
|
380 |
catch( Exception e)
|
|
|
381 |
{
|
|
|
382 |
// Some form of error has occurred. Types include
|
|
|
383 |
// Test build (Treated as an error so that dpkg_archive gets removed)
|
|
|
384 |
// buildErrorOccurred - True
|
|
|
385 |
// publishError - False
|
|
|
386 |
// Build Error - Package was not fully published to dpkg_archive
|
|
|
387 |
// buildErrorOccurred - True
|
|
|
388 |
// publishError - False
|
|
|
389 |
//
|
|
|
390 |
// Badly formed VCS tag (Error Publishing to Version Control)
|
|
|
391 |
// buildErrorOccurred - False
|
|
|
392 |
// publishError - True
|
|
|
393 |
// Error Publishing to Release Manager
|
|
|
394 |
// buildErrorOccurred - False
|
|
|
395 |
// publishError - True
|
|
|
396 |
// Error inserting package metrics
|
|
|
397 |
// buildErrorOccurred - False
|
|
|
398 |
// publishError - True
|
|
|
399 |
// Error inserting unit test results
|
|
|
400 |
// buildErrorOccurred - False
|
|
|
401 |
// publishError - True
|
|
|
402 |
//
|
|
|
403 |
// Uses
|
|
|
404 |
// buildErrorOccurred - Package not in dpkg_archive as expected (or was a test build)
|
|
|
405 |
// Email should have been sent to the user
|
|
|
406 |
// publishError - Could not publish to Release Manager
|
|
|
407 |
// Post build error. Email has not been sent to user
|
|
|
408 |
// Takes precedence over buildErrorOccurred
|
|
|
409 |
|
|
|
410 |
// Insert generic message if we don't have one
|
|
|
411 |
// Should only occur under unexpected conditions
|
|
|
412 |
if (mReporting.errMsg == null || mReporting.errMsg.isEmpty())
|
|
|
413 |
{
|
|
|
414 |
mReporting.errMsg = "Error publishing to Release Manager";
|
|
|
415 |
mReporting.errMsgDetail = mPhase.sText;
|
|
|
416 |
}
|
|
|
417 |
|
|
|
418 |
// Indicate some form of build error
|
|
|
419 |
if ( ! mReporting.isaTestBuild )
|
|
|
420 |
{
|
|
|
421 |
mPhase.setPhase("Update Build Instance");
|
|
|
422 |
mReleaseManager.updateBuildInstance(mReporting.buildId, 0, publishError ? BuildResult.SystemError :BuildResult.BuildError );
|
|
|
423 |
}
|
|
|
424 |
|
|
|
425 |
//
|
|
|
426 |
// Delete the dpkg_archive entry - the package is a dud
|
|
|
427 |
//
|
|
|
428 |
mPhase.setPhase("Delete package from archive");
|
|
|
429 |
mLogger.fatal("Delete package from archive");
|
|
|
430 |
deletePackageVersion();
|
|
|
431 |
|
|
|
432 |
// A publishing error. The error occurred, after the build, in the RM publishing
|
|
|
433 |
// We won't have a log file, but we will have a rootCause
|
|
|
434 |
// Exclude package from the build
|
|
|
435 |
// Unless its a test build (handled within mReleaseManager.excludeFromBuild)
|
|
|
436 |
//
|
|
|
437 |
if ( publishError )
|
|
|
438 |
{
|
|
|
439 |
mLogger.fatal("an error occurred publishing to Version Control or Release Manager");
|
|
|
440 |
|
|
|
441 |
// Post Build Error
|
|
|
442 |
// eMails have not been sent by the ABT class as the error has occurred outside of that process
|
|
|
443 |
mPhase.setPhase("Post Build Failure Email");
|
|
|
444 |
emailPostBuild(mRippleEngine, true);
|
|
|
445 |
|
|
|
446 |
mLogger.fatal("run calling excludeFromBuild");
|
|
|
447 |
mPhase.setPhase("Excluded from build");
|
|
|
448 |
mReleaseManager.excludeFromBuild(false,
|
|
|
449 |
mReporting.packageVersionId,
|
|
|
450 |
null,
|
|
|
451 |
mReporting.rtagId,
|
|
|
452 |
null,
|
|
|
453 |
mReporting.errMsg,
|
|
|
454 |
null,
|
|
|
455 |
false, mReporting.isaTestBuild);
|
|
|
456 |
|
|
|
457 |
}
|
|
|
458 |
else if ( buildErrorOccurred )
|
|
|
459 |
{
|
|
|
460 |
// Build Error Occurred
|
|
|
461 |
// Package has not been built on all configured platforms
|
|
|
462 |
// Under normal circumstances, this root cause will be masked by a previously reported error
|
|
|
463 |
// Under abnormal circumstances amidst build activity such as:
|
|
|
464 |
// - slave build machine reboots
|
|
|
465 |
// - slave build machines with a read only file system
|
|
|
466 |
// The rootCause is required to keep the end user informed
|
|
|
467 |
//
|
|
|
468 |
// Use excludeFromBuild such that this call will not supersede existing entries
|
|
|
469 |
// In this manner the log file/cause provided here will only be used if we haven't
|
|
|
470 |
// already excluded this package from the build
|
|
|
471 |
|
|
|
472 |
// If we have a log file, then provide it to the user, else insert a generic message
|
|
|
473 |
|
|
|
474 |
mReporting.errMsg = (mReporting.buildFailureLogFile == null) ? "Buildtool env error occurred. Attempt build again" : null;
|
|
|
475 |
|
|
|
476 |
mLogger.fatal("run calling excludeFromBuild");
|
|
|
477 |
mPhase.setPhase("Excluded from build");
|
|
|
478 |
mReleaseManager.excludeFromBuild(false,
|
|
|
479 |
mReporting.packageVersionId,
|
|
|
480 |
mReporting.packageVersion,
|
|
|
481 |
mReporting.rtagId,
|
|
|
482 |
null,
|
|
|
483 |
mReporting.errMsg,
|
|
|
484 |
mReporting.buildFailureLogFile,
|
|
|
485 |
false, mReporting.isaTestBuild);
|
|
|
486 |
}
|
|
|
487 |
else
|
|
|
488 |
{
|
|
|
489 |
// Not a buildError and not a publishing Error
|
|
|
490 |
// Don't know how we got here - I don't think we can
|
|
|
491 |
// Will cause an indefinite pause !
|
|
|
492 |
throw new Exception("an error occurred publishing to Version Control or Release Manager");
|
|
|
493 |
|
|
|
494 |
}
|
|
|
495 |
} // End Catch
|
|
|
496 |
}
|
|
|
497 |
|
|
|
498 |
if ( mUnitTest.compareTo("unit test coordinate slave threads") == 0 )
|
|
|
499 |
{
|
|
|
500 |
throw new ExitException();
|
|
|
501 |
}
|
|
|
502 |
mLogger.info("run coordinate slave threads returned");
|
|
|
503 |
}
|
|
|
504 |
|
|
|
505 |
|
|
|
506 |
//---------------------------------------------------------------
|
|
|
507 |
// Start of a new build cycle
|
|
|
508 |
// Determine if the daemon has been paused.
|
|
|
509 |
// Set RunLevel.IDLE (within allowedToProceed)
|
|
|
510 |
//
|
|
|
511 |
mLogger.fatal("run calling allowedToProceed");
|
|
|
512 |
mPhase.setPhase("allowedToProceed");
|
|
|
513 |
mReleaseManager.discardVersion();
|
|
|
514 |
mReleaseManager.clearCurrentPackageBeingBuilt(mRconId);
|
|
|
515 |
allowedToProceed(true);
|
|
|
516 |
mLogger.info("run allowedToProceed returned");
|
|
|
517 |
|
|
|
518 |
if ( mUnitTest.compareTo("unit test allowed to proceed") == 0 )
|
|
|
519 |
{
|
|
|
520 |
throw new ExitException();
|
|
|
521 |
}
|
|
|
522 |
|
|
|
523 |
// Set up mReleaseConfigCollection
|
|
|
524 |
// This will be used when creating a build file
|
|
|
525 |
mLogger.fatal("Collect fresh MetaData");
|
|
|
526 |
mPhase.setPhase("collectMetaData");
|
|
|
527 |
mRippleEngine.collectMetaData();
|
|
|
528 |
|
|
|
529 |
//---------------------------------------------------------------
|
|
|
530 |
// Plan the next build
|
|
|
531 |
// Snap current Release Sequence Number
|
|
|
532 |
// Determine what is to be built
|
|
|
533 |
// Generate build files
|
|
|
534 |
mLogger.fatal("run calling planRelease");
|
|
|
535 |
mPhase.setPhase("Plan Release", 600);
|
|
|
536 |
mReleaseSeqNum = mReleaseManager.queryReleaseSeqNum(mRtagId, mRconId, BuildDaemon.mHostname);
|
|
|
537 |
mRippleEngine.planRelease(!mLastBuildWasBenign);
|
|
|
538 |
|
|
|
539 |
// get the build file from the ripple engine
|
|
|
540 |
mPhase.setPhase("Get Build File", 10);
|
|
|
541 |
mMasterBuildFile = mRippleEngine.getFirstBuildFileContent();
|
|
|
542 |
|
|
|
543 |
}
|
|
|
544 |
}
|
|
|
545 |
|
|
|
546 |
//---------------------------------------------------------------
|
|
|
547 |
// Check that we are good to proceed with the build
|
|
|
548 |
// Check environment
|
|
|
549 |
// Ensure we have enough disk space
|
|
|
550 |
//
|
|
|
551 |
mLogger.fatal("run calling checkEnvironment");
|
|
|
552 |
mPhase.setPhase("checkEnvironment");
|
|
|
553 |
checkEnvironment();
|
|
|
554 |
mLogger.info("run checkEnvironment returned");
|
|
|
555 |
|
|
|
556 |
if ( mUnitTest.startsWith("unit test check environment") )
|
|
|
557 |
{
|
|
|
558 |
throw new ExitException();
|
|
|
559 |
}
|
|
|
560 |
|
|
|
561 |
mLogger.info("run generated build files");
|
|
|
562 |
// Start of a build cycle. Set new log file to capture entire build log
|
|
|
563 |
mPhase.setPhase("Start Build Cycle");
|
|
|
564 |
flagStartBuildCycle();
|
|
|
565 |
mBuildDuration.reset();
|
|
|
566 |
mReporting.resetData();
|
|
|
567 |
mRunLevel.persist(mReleaseManager, mRconId, BuildState.DB_ACTIVE);
|
|
|
568 |
mReleaseManager.setCurrentPackageBeingBuilt(mRconId, mMasterBuildFile.mPkgId, mMasterBuildFile.mPvId);
|
|
|
569 |
|
|
|
570 |
// Save the buildfile
|
|
|
571 |
mLogger.fatal("Save buildfile");
|
|
|
572 |
mPhase.setPhase("Save buildfile");
|
|
|
573 |
saveBuildFile(mMasterBuildFile.content);
|
|
|
574 |
|
|
|
575 |
// Quick Test to validate the provided Version Control tag/label
|
|
|
576 |
mPhase.setPhase("Validate Version Tag");
|
|
|
577 |
if ( deliverChange("AbtTestPath", true, false) )
|
|
|
578 |
{
|
|
|
579 |
|
|
|
580 |
//---------------------------------------------------------------
|
|
|
581 |
// Notify all slaves that there is work to be done
|
|
|
582 |
// This is done by publishing a buildfile for them to consume
|
|
|
583 |
// A NonGeneric build will be done with a real build file on all machines
|
|
|
584 |
// Otherwise do a Dummy build on all Machines
|
|
|
585 |
//
|
|
|
586 |
mPhase.setPhase("Notify slaves of work", 10);
|
|
|
587 |
if ( mMasterBuildFile.state != BuildFileState.NonGeneric)
|
|
|
588 |
{
|
|
|
589 |
// publish a dummy build file for either generic or dummy cases
|
|
|
590 |
// this results in the slave not running ant
|
|
|
591 |
mLogger.fatal("run calling publishBuildFile on dummy");
|
|
|
592 |
mMasterBuildFile.content = mDummyBuildFileContent;
|
|
|
593 |
}
|
|
|
594 |
else
|
|
|
595 |
{
|
|
|
596 |
// publish the build file
|
|
|
597 |
mLogger.fatal("run calling publishBuildFile");
|
|
|
598 |
}
|
|
|
599 |
mReleaseManager.publishBuildFile(mRtagId, mMasterBuildFile);
|
|
|
600 |
|
|
|
601 |
|
|
|
602 |
if ( mUnitTest.compareTo("unit test generate build files") == 0 )
|
|
|
603 |
{
|
|
|
604 |
throw new ExitException();
|
|
|
605 |
}
|
|
|
606 |
|
|
|
607 |
mLogger.info("run changed run levels");
|
|
|
608 |
|
|
|
609 |
//---------------------------------------------------------------
|
|
|
610 |
// Build the package
|
|
|
611 |
// Setup the workspace
|
|
|
612 |
// Build the package - deliver change to product baseline
|
|
|
613 |
//
|
|
|
614 |
mLogger.fatal("Build Package " + mReporting.packageName + "_" + mReporting.packageVersion);
|
|
|
615 |
|
|
|
616 |
mLogger.info("run calling setViewUp");
|
|
|
617 |
mPhase.setPhase("Setup View for build");
|
|
|
618 |
deliverChange("AbtSetUp", true, false);
|
|
|
619 |
|
|
|
620 |
mLogger.info("run calling deliverChange - the actual build");
|
|
|
621 |
mPhase.setPhase("Build Package");
|
|
|
622 |
deliverChange(null, true, false);
|
|
|
623 |
mLogger.info("run deliverChange returned");
|
|
|
624 |
}
|
|
|
625 |
|
|
|
626 |
mLastBuildWasBenign = false;
|
|
|
627 |
if ( mMasterBuildFile.state == BuildFileState.Dummy)
|
|
|
628 |
{
|
|
|
629 |
// no build requirement, so let things settle
|
|
|
630 |
mLogger.warn("run no build requirement, so let things settle");
|
|
|
631 |
mLastBuildWasBenign = true;
|
|
|
632 |
}
|
|
|
633 |
}
|
|
|
634 |
//---------------------------------------------------------------
|
|
|
635 |
// Build loop exception handling (only)
|
|
|
636 |
//
|
|
|
637 |
catch( SQLException e )
|
|
|
638 |
{
|
|
|
639 |
// Oracle connection issues
|
|
|
640 |
// Request a prolonged sleep at the start of the build loop
|
|
|
641 |
mLogger.fatal("run oracle connection issues");
|
|
|
642 |
mException = true;
|
|
|
643 |
}
|
|
|
644 |
catch( ExitException e )
|
|
|
645 |
{
|
|
|
646 |
mLogger.warn("run ExitException");
|
|
|
647 |
exit = true;
|
|
|
648 |
}
|
|
|
649 |
catch( InterruptedException e )
|
|
|
650 |
{
|
|
|
651 |
mLogger.warn("run InterruptedException");
|
|
|
652 |
Thread.currentThread().interrupt();
|
|
|
653 |
}
|
|
|
654 |
catch (BuildSystemException e) {
|
|
|
655 |
// BuildSystem exception - a (possibly) correctable error
|
|
|
656 |
// Flag as recoverable
|
|
|
657 |
// - Can't create XML build file
|
|
|
658 |
// -
|
|
|
659 |
//
|
|
|
660 |
mLogger.fatal("run BuildSystemException");
|
|
|
661 |
mPhase.setPhase("BuildSystemException indefinitePause");
|
|
|
662 |
notifyIndefinitePause(e.getMessage());
|
|
|
663 |
mReleaseManager.indefinitePause(false);
|
|
|
664 |
mException = true;
|
|
|
665 |
mRecoverable = true;
|
|
|
666 |
|
|
|
667 |
}
|
|
|
668 |
catch( BuildToolException e)
|
|
|
669 |
{
|
|
|
670 |
// Buildtool exception - an uncorrectable error
|
|
|
671 |
// - Can't parse XML build file
|
|
|
672 |
//
|
|
|
673 |
mLogger.fatal("run BuildToolException");
|
|
|
674 |
mPhase.setPhase("BuildToolException indefinitePause");
|
|
|
675 |
notifyIndefinitePause(e.getMessage());
|
|
|
676 |
mReleaseManager.indefinitePause(false);
|
|
|
677 |
mException = true;
|
|
|
678 |
}
|
|
|
679 |
catch( Exception e )
|
|
|
680 |
{
|
|
|
681 |
// Uncaptured exception
|
|
|
682 |
// These are not good. Current mechanism to handle this condition is:
|
|
|
683 |
// 1) Email build system administrator
|
|
|
684 |
// 2) Pause the build system
|
|
|
685 |
//
|
|
|
686 |
mLogger.error("run indefinitePause " + e.toString());
|
|
|
687 |
|
|
|
688 |
// DEVI 51366 force sleep at beginning of while loop
|
|
|
689 |
mException = true;
|
|
|
690 |
|
|
|
691 |
String cause = e.getMessage();
|
|
|
692 |
if ( cause == null || cause.compareTo("null") == 0 )
|
|
|
693 |
{
|
|
|
694 |
cause = "Internal Error: " + e.toString();
|
|
|
695 |
}
|
|
|
696 |
|
|
|
697 |
try
|
|
|
698 |
{
|
|
|
699 |
// notify first
|
|
|
700 |
// many reasons for indefinite pause, including database related, so do database last
|
|
|
701 |
mRecoverable = false;
|
|
|
702 |
|
|
|
703 |
if ( cause.compareTo(Package.mRecoverable) == 0 )
|
|
|
704 |
{
|
|
|
705 |
mRecoverable = true;
|
|
|
706 |
}
|
|
|
707 |
|
|
|
708 |
mPhase.setPhase("Notify indefinitePause");
|
|
|
709 |
notifyIndefinitePause(cause);
|
|
|
710 |
mReleaseManager.indefinitePause(mRecoverable);
|
|
|
711 |
mReleaseManager.updateBuildInstance(mReporting.buildId, 0, BuildResult.SystemError );
|
|
|
712 |
}
|
|
|
713 |
catch( Exception f )
|
|
|
714 |
{
|
|
|
715 |
mLogger.error("run indefinitePause failed");
|
|
|
716 |
}
|
|
|
717 |
}
|
|
|
718 |
}
|
|
|
719 |
try {
|
|
|
720 |
mReleaseManager.clearCurrentPackageBeingBuilt(mRconId);
|
|
|
721 |
} catch (Exception e) {
|
|
|
722 |
mLogger.error("Exception in clearCurrentPackageBeingBuilt:" + e.getMessage());
|
|
|
723 |
}
|
|
|
724 |
mPhase.setPhase("Exit Thread");
|
|
|
725 |
}
|
|
|
726 |
|
|
|
727 |
/** Delete a specified version of a package from dpkg_archive
|
|
|
728 |
* Used during error processing to cleanup packages that did not build correctly
|
|
|
729 |
*
|
|
|
730 |
* The package-version is in dpkg_archive. This may be on a remote server
|
|
|
731 |
* Execute the deletion command on the remote server as it will be a lot faster
|
|
|
732 |
* for large packages.
|
|
|
733 |
*
|
|
|
734 |
* Ensure that the file system cache is cleared before we attempt to look
|
|
|
735 |
* for the directory
|
|
|
736 |
*/
|
|
|
737 |
private void deletePackageVersion()
|
|
|
738 |
{
|
|
|
739 |
|
|
|
740 |
// Setup a Remote Execution environment
|
|
|
741 |
// The logging interface is is hard part
|
|
|
742 |
RemoteExecution mRemoteExecution = new RemoteExecution(new LogOutput(){
|
|
|
743 |
|
|
|
744 |
@Override
|
|
|
745 |
public void data(String message) {
|
|
|
746 |
mLogger.info(message);
|
|
|
747 |
}
|
|
|
748 |
|
|
|
749 |
@Override
|
|
|
750 |
public void fatal(String message) {
|
|
|
751 |
mLogger.fatal(message);
|
|
|
752 |
}
|
|
|
753 |
|
|
|
754 |
@Override
|
|
|
755 |
public void info(String message) {
|
|
|
756 |
mLogger.info(message);
|
|
|
757 |
}});
|
|
|
758 |
|
|
|
759 |
|
|
|
760 |
// Execute the remote command
|
|
|
761 |
// This is a best effort deletion - can't really handle conditions
|
|
|
762 |
// where the command fails.
|
|
|
763 |
CommandBuilder cmd = new CommandBuilder();
|
|
|
764 |
|
|
|
765 |
cmd.add("sudo","-n", "-u", "buildadm", "/home/releasem/sbin/jatsTool", "assemble_dpkg");
|
|
|
766 |
cmd.add("-DeleteVersion");
|
|
|
767 |
cmd.add(!ReleaseManager.getUseMutex(), "-testArchive");
|
|
|
768 |
cmd.add("'-pname=" + mReporting.packageName + "'");
|
|
|
769 |
cmd.add("'-pversion=" + mReporting.packageVersion + "'");
|
|
|
770 |
|
|
|
771 |
mLogger.fatal("run remote command to delete " + mReporting.packageName +", "+ mReporting.packageVersion);
|
|
|
772 |
int rv = mRemoteExecution.execute(cmd.toString());
|
|
|
773 |
mLogger.fatal("run remote command to delete. Returned " + rv);
|
|
|
774 |
|
|
|
775 |
// Belt and Braces
|
|
|
776 |
// Ensure that the package-version has been deleted,
|
|
|
777 |
// and if not then try to delete it the slow way.
|
|
|
778 |
//
|
|
|
779 |
// Note: Need to flush NFS cache before we check existence
|
|
|
780 |
|
|
|
781 |
String dpkgArchiveEntry = utilities.catDir(Package.mGbeDpkg, mReporting.packageName, mReporting.packageVersion);
|
|
|
782 |
File dpkgArchiveEntryFile = utilities.freshFile(dpkgArchiveEntry);
|
|
|
783 |
mLogger.info("run checking existence of " + dpkgArchiveEntry);
|
|
|
784 |
|
|
|
785 |
if ( dpkgArchiveEntryFile.exists() )
|
|
|
786 |
{
|
|
|
787 |
if ( dpkgArchiveEntryFile.isDirectory() )
|
|
|
788 |
{
|
|
|
789 |
mLogger.fatal("run calling deleteDirectory on " + dpkgArchiveEntryFile);
|
|
|
790 |
deleteDirectory(dpkgArchiveEntryFile);
|
|
|
791 |
}
|
|
|
792 |
}
|
|
|
793 |
}
|
|
|
794 |
|
|
|
795 |
/** Sends email notification and mark the successful completion of a build
|
|
|
796 |
* Used simply to keep an email trail of build operations
|
|
|
797 |
*/
|
|
|
798 |
public void emailPostBuild( RippleEngine rippleEngine, boolean isaError )
|
|
|
799 |
{
|
|
|
800 |
mLogger.debug("emailPostBuild");
|
|
|
801 |
String subject;
|
|
|
802 |
String emailTo;
|
|
|
803 |
String mailBody = "";
|
|
|
804 |
|
|
|
805 |
if (isaError)
|
|
|
806 |
{
|
|
|
807 |
subject = "BUILD FAILURE on package " + mReporting.packageName + " " + mReporting.packageVersion;
|
|
|
808 |
emailTo = mReporting.packageOwners;
|
|
|
809 |
mailBody="An error occured within the build system after the package had been built<p><hr>";
|
|
|
810 |
}
|
|
|
811 |
else
|
|
|
812 |
{
|
|
|
813 |
subject = "BUILD SUCCESS on package " + mReporting.packageName + " " + mReporting.packageVersion;
|
|
|
814 |
emailTo = rippleEngine.getMailSender();
|
|
|
815 |
}
|
|
|
816 |
|
|
|
817 |
// Cleanup the dependency string
|
|
|
818 |
// Expecting: 'junit','3.7.0.cots';'ant-ejbdoclet','1.0000.cr'
|
|
|
819 |
String indent = "<br> ";
|
|
|
820 |
String depends = mReporting.packageDepends.replace("','", " ");
|
|
|
821 |
depends = depends.replace("';'",indent);
|
|
|
822 |
depends = depends.replace("'","");
|
|
|
823 |
|
|
|
824 |
// Cleanup the build standard string
|
|
|
825 |
// Expecting win32: jats all;linux_i386: jats prod
|
|
|
826 |
String buildStandards = mReporting.packageBuildInfo;
|
|
|
827 |
buildStandards = buildStandards.replace(";",indent);
|
|
|
828 |
|
|
|
829 |
mailBody += "Release : " + rippleEngine.mBaselineName + "<br>" +
|
|
|
830 |
"Package : " + mReporting.packageName + "<br>" +
|
|
|
831 |
"Version : " + mReporting.packageVersion + "<br>";
|
|
|
832 |
if (isaError)
|
|
|
833 |
{
|
|
|
834 |
mailBody += "Cause : " + mReporting.errMsg + "<br>";
|
|
|
835 |
if (mReporting.errMsgDetail != null && mReporting.errMsgDetail.length() > 0)
|
|
|
836 |
{
|
|
|
837 |
mailBody += "Detail : " + mReporting.errMsgDetail + "<br>";
|
|
|
838 |
}
|
|
|
839 |
}
|
|
|
840 |
|
|
|
841 |
mailBody += "RM Link : " + CreateUrls.generateRmUrl(mRtagId,mReporting.packageVersionId) + "<br>" +
|
|
|
842 |
"PVID : " + mReporting.packageVersionId + "<br>" +
|
|
|
843 |
"VcsTag : " + mReporting.newVcsTag + "<br>" +
|
|
|
844 |
"Ripple : " + (mReporting.isRipple ? "Yes" : "No") + "<br>" +
|
|
|
845 |
"Dependencies : " + indent + depends + "<br>" +
|
|
|
846 |
"Build Standards : " + indent + buildStandards;
|
|
|
847 |
|
|
|
848 |
mailBody += "<p><hr>";
|
|
|
849 |
try
|
|
|
850 |
{
|
|
|
851 |
Smtpsend.send( rippleEngine.getMailServer(), // mailServer
|
|
|
852 |
rippleEngine.getMailSender(), // source
|
|
|
853 |
emailTo, // target - send to package owners, or myself
|
|
|
854 |
rippleEngine.getMailSender(), // cc
|
|
|
855 |
null, // bcc
|
|
|
856 |
subject, // subject
|
|
|
857 |
mailBody, // body
|
|
|
858 |
null // attachment
|
|
|
859 |
);
|
|
|
860 |
}
|
|
|
861 |
catch( Exception e )
|
|
|
862 |
{
|
|
|
863 |
mLogger.warn("Email Failure: emailPostBuild:" + e.getMessage());
|
|
|
864 |
}
|
|
|
865 |
mLogger.debug("emailPostBuild. Returning");
|
|
|
866 |
}
|
|
|
867 |
|
|
|
868 |
/**returns 'M'
|
|
|
869 |
*/
|
|
|
870 |
protected char getMode()
|
|
|
871 |
{
|
|
|
872 |
mLogger.debug("getMode");
|
|
|
873 |
return 'M';
|
|
|
874 |
}
|
|
|
875 |
|
|
|
876 |
|
|
|
877 |
}
|