Subversion Repositories DevTools

Rev

Rev 5486 | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 5486 Rev 5498
Line 62... Line 62...
62
#
62
#
63
my $tmpDirInfo;
63
my $tmpDirInfo;
64
my $workDir;
64
my $workDir;
65
my $startDir;
65
my $startDir;
66
my $maxHostNameLength = 8;
66
my $maxHostNameLength = 8;
-
 
67
my $maxTypeLength = 8;
67
my $pkgTargetDir;
68
my $pkgTargetDir;
68
my $deleteTargetDir;
69
my $deleteTargetDir;
69
 
70
 
70
#
71
#
71
#   Option variables
72
#   Option variables
Line 205... Line 206...
205
        Error("Tar extraction error: $srcfile") if ($rv);
206
        Error("Tar extraction error: $srcfile") if ($rv);
206
    }
207
    }
207
 
208
 
208
    #
209
    #
209
    #   Read in the XML from each of the files
210
    #   Read in the XML from each of the files
-
 
211
    #   Process the XML
-
 
212
    #       Detect merge clashes
-
 
213
    #       Create new XML - assuming the extraction will NOT overwrite existing files
210
    #
214
    #
211
    my %fileData;
215
    my %fileData;
-
 
216
    my @newXml;
212
    foreach my $srcfile ( keys %pkgData )
217
    foreach my $srcfile ( keys %pkgData )
213
    {
218
    {
214
        my @extracted = glob(catfile($pkgData{$srcfile}{basename}, 'built.files.*.xml'));
219
        my @extracted = glob(catfile($pkgData{$srcfile}{basename}, 'built.files.*.xml'));
215
        foreach my $srcfile ( @extracted)
220
        foreach my $srcfile ( @extracted)
216
        {
221
        {
217
            my $ref = XML::Simple::XMLin($srcfile, ForceArray => 1, KeyAttr => []);
222
            my $ref = XML::Simple::XMLin($srcfile, ForceArray => 1, KeyAttr => []);
218
            #DebugDumpData("REF - $srcfile, " .ref($ref), $ref);
223
            #DebugDumpData("REF - $srcfile, " .ref($ref), $ref);
219
 
224
 
-
 
225
            my $entryExists;
-
 
226
            my $keepEntry;
220
            foreach my $entry (@{$ref->{file}})
227
            foreach my $entry (@{$ref->{file}})
221
            {
228
            {
-
 
229
                #
-
 
230
                #   Calculate some common data items
-
 
231
                #       Calc max host name length for pretty printing
-
 
232
                my $hostnameLen = length ($entry->{host} || '');
-
 
233
                $maxHostNameLength = $hostnameLen if ($hostnameLen > $maxHostNameLength);
-
 
234
 
-
 
235
                my $typeLen = length ($entry->{type} || '');
-
 
236
                $maxTypeLength = $typeLen if ($typeLen > $maxTypeLength);
-
 
237
 
-
 
238
                my $hostEntry = {host => $entry->{host}, md5sum => $entry->{md5sum}, type => $entry->{type}};
-
 
239
                push @{$fileData{$entry->{fullname}}{hosts}}, $hostEntry;
-
 
240
                my $store = $fileData{$entry->{fullname}};
-
 
241
 
-
 
242
                #
-
 
243
                #   Determine if we have seen this file before
-
 
244
                #   If so then we need to:
-
 
245
                #       Perform a merge clash
-
 
246
                #       Ensure that its of the same type
-
 
247
                #       Mark the new XML as 'merge'
-
 
248
                #
-
 
249
                $entryExists = 0;
-
 
250
                $keepEntry = 1;
-
 
251
                if (exists $store->{type})
-
 
252
                {
-
 
253
                    $entryExists = 1;
-
 
254
                    if ($store->{type} ne $entry->{type})
-
 
255
                    {
-
 
256
                        $store->{bad} = 1;
-
 
257
                        $store->{badType} = 1;
-
 
258
                    }
-
 
259
                }
-
 
260
                else
-
 
261
                {
-
 
262
                    $store->{type} = $entry->{type};
-
 
263
                }
-
 
264
 
222
                #   directory - no processing required
265
                #   directory - no processing required
223
                next if $entry->{type} eq 'dir';
266
                if ($entry->{type} eq 'dir')
-
 
267
                {
-
 
268
                    $keepEntry = 0 if $entryExists;
-
 
269
                    next;
-
 
270
                }
224
 
271
 
225
                #   link - no processing reqiuired
272
                #   link - no processing reqiuired
226
                next if $entry->{type} eq 'link';
273
                if ($entry->{type} eq 'link')
-
 
274
                {
-
 
275
                    $keepEntry = 0 if $entryExists;
-
 
276
                    next;
-
 
277
                }
227
 
278
 
228
                #   file - ensure there is no clash
279
                #   file - ensure there is no clash
229
                if ($entry->{type} eq 'file')
280
                if ($entry->{type} eq 'file')
230
                {
281
                {
231
                    # Calc max host name length for pretty printing
-
 
232
                    my $hostnameLen = length $entry->{host};
-
 
233
                    $maxHostNameLength = $hostnameLen if ($hostnameLen > $maxHostNameLength);
-
 
234
 
-
 
235
                    my $hostEntry = {host => $entry->{host}, md5sum => $entry->{md5sum}}; 
-
 
236
                    push @{$fileData{$entry->{fullname}}{hosts}}, $hostEntry;
-
 
237
                    my $store = $fileData{$entry->{fullname}}; 
-
 
238
 
-
 
239
                    if (exists $store->{md5sum})
282
                    if (exists $store->{md5sum})
240
                    {
283
                    {
241
                        # Compare existing entry and add new info
-
 
242
                        unless ($store->{md5sum} eq $entry->{md5sum})
284
                        $store->{bad} = 1 unless ($store->{md5sum} eq $entry->{md5sum});
243
                        {
-
 
244
                            $store->{bad} = 1;
-
 
245
                        }
-
 
246
                    }
285
                    }
247
                    else
286
                    else
248
                    {
287
                    {
249
                        # Create new entry
-
 
250
                        $store->{md5sum} = $entry->{md5sum};
288
                        $store->{md5sum} = $entry->{md5sum};
251
                    }
289
                    }
252
                next;
290
                next;
253
                }
291
                }
254
                #   Unknown - just a warning for now
292
                #   Unknown - just a warning for now
255
                Warning( "Unknown type:" . $entry->{fullname} . ':' . $entry->{type} );
293
                Warning( "Unknown type: " . $entry->{type} , "    Path: ". $entry->{fullname} );
-
 
294
            }
-
 
295
            continue
-
 
296
            {
-
 
297
                #
-
 
298
                #   This block is always executed
-
 
299
                #   It is used to maintain the entry and the rewrite the XML file list
-
 
300
                #   Do not include the build.files.xxx.xml
-
 
301
                #       They are about to be deleted
-
 
302
                #       Not detailed in the non-tar package merge process
-
 
303
                #
-
 
304
                if ($keepEntry)
-
 
305
                {
-
 
306
                    unless ($entry->{fullname} =~ m~^built\.files\..*\.xml$~ )
-
 
307
                    {
-
 
308
                        if ($entryExists)
-
 
309
                        {
-
 
310
                            delete $entry->{md5sum};
-
 
311
                            delete $entry->{size};
-
 
312
                            $entry->{type} = 'merge';
-
 
313
                        }
-
 
314
                        push @newXml, $entry;
-
 
315
                    }
-
 
316
                }
256
            }
317
            }
257
        }
318
        }
258
    }
319
    }
-
 
320
    #DebugDumpData("newXml",\@newXml);
259
 
321
 
260
    #
322
    #
261
    #   Cleanout the non-bad entries
323
    #   Cleanout the non-bad entries
262
    #   Report on merge errors
324
    #   Report on merge errors
263
    #
325
    #
Line 289... Line 351...
289
        {
351
        {
290
            delete $fileData{$entry};
352
            delete $fileData{$entry};
291
            next;
353
            next;
292
        }
354
        }
293
 
355
 
294
        #
-
 
295
        #   Have a merge error
-
 
296
        #       Detail what has happened
-
 
297
        #       Generate pretty output showning on which machines that are command.
-
 
298
        #
-
 
299
        unless ($headerReported)
356
        unless ($headerReported)
300
        {
357
        {
301
            $headerReported = 1;
358
            $headerReported = 1;
302
            reportMergeError('Package Merge Error. File provided by different builds are not identical');
359
            reportMergeError('Package Merge Error. File provided by different builds are not identical');
303
            reportMergeError('This prevents the build from being reproducible.');
360
            reportMergeError('This prevents the build from being reproducible.');
304
        }
361
        }
305
        reportMergeError('File Name:  ' . $entry);
-
 
306
 
-
 
307
#       foreach my $e ( @{$fileData{$entry}{hosts}} )
-
 
308
#       {
-
 
309
#           reportMergeError('    Provided by:  ' . sprintf('%-*s',$maxHostNameLength,$e->{host}) . '  Signature: ' . $e->{md5sum});
-
 
310
#       }
-
 
311
 
-
 
312
        my %md5List;
-
 
313
        foreach my $e ( @{$fileData{$entry}{hosts}} ) {
-
 
314
            UniquePush (\@{$md5List{$e->{md5sum}}}, $e->{host});
-
 
315
        }
-
 
316
 
362
 
317
        foreach my $e ( @{$fileData{$entry}{hosts}} )
363
        if ($fileData{$entry}{badType})
318
        {
364
        {
-
 
365
            #
-
 
366
            #   Have a TYPE merge error
-
 
367
            #       Detail what has happened
-
 
368
            #       Generate pretty output showning on which machines that are command.
-
 
369
            #
-
 
370
            my %typeList;
-
 
371
            foreach my $e ( @{$fileData{$entry}{hosts}} ) {
-
 
372
                UniquePush (\@{$typeList{$e->{type}}}, $e->{host});
-
 
373
            }
-
 
374
 
-
 
375
            reportMergeError('Entry Path: ' . $entry);
-
 
376
            foreach my $e ( @{$fileData{$entry}{hosts}} )
-
 
377
            {
319
            my $hostList;
378
                my $hostList;
320
            my @sameHosts = @{$md5List{$e->{md5sum}}};
379
                my @sameHosts = @{$typeList{$e->{type}}};
321
            ArrayDelete (\@sameHosts, $e->{host});
380
                ArrayDelete (\@sameHosts, $e->{host});
322
            if (@sameHosts) {
381
                if (@sameHosts) {
323
                $hostList = ' Same as: ' . join(', ', @sameHosts);
382
                    $hostList = ' Same as: ' . join(', ', @sameHosts);
324
            } else {
383
                } else {
325
                $hostList = ' Unique to: '. $e->{host};
384
                    $hostList = ' Unique to: '. $e->{host};
-
 
385
                }
-
 
386
 
-
 
387
                reportMergeError('    Provided by: ' . sprintf('%-*s',$maxHostNameLength,$e->{host}) . ' Type: ' . sprintf('%-*s',$maxTypeLength,$e->{type}) . $hostList );
326
            }
388
            }
327
 
389
 
328
            reportMergeError('    Provided by:  ' . sprintf('%-*s',$maxHostNameLength,$e->{host}) . $hostList );
-
 
329
        }
390
        }
-
 
391
        else
-
 
392
        {
-
 
393
            #
-
 
394
            #   Have a FILE merge error
-
 
395
            #       Detail what has happened
-
 
396
            #       Generate pretty output showning on which machines that are common.
-
 
397
            #
-
 
398
            my %md5List;
-
 
399
            foreach my $e ( @{$fileData{$entry}{hosts}} ) {
-
 
400
                UniquePush (\@{$md5List{$e->{md5sum}}}, $e->{host});
-
 
401
            }
330
 
402
 
-
 
403
            reportMergeError('File Name: ' . $entry);
-
 
404
            foreach my $e ( @{$fileData{$entry}{hosts}} )
-
 
405
            {
-
 
406
                my $hostList;
-
 
407
                my @sameHosts = @{$md5List{$e->{md5sum}}};
-
 
408
                ArrayDelete (\@sameHosts, $e->{host});
-
 
409
                if (@sameHosts) {
-
 
410
                    $hostList = ' Same as: ' . join(', ', @sameHosts);
-
 
411
                } else {
-
 
412
                    $hostList = ' Unique to: '. $e->{host};
-
 
413
                }
331
 
414
 
-
 
415
                reportMergeError('    Provided by: ' . sprintf('%-*s',$maxHostNameLength,$e->{host}) . $hostList );
-
 
416
            }
-
 
417
        }
332
    }
418
    }
333
    ErrorDoExit();
419
    ErrorDoExit();
334
 
420
 
335
    #
421
    #
336
    #   Calculate target package location
422
    #   Calculate target package location
Line 353... Line 439...
353
        my $rv = System ('tar', '-xzmf', $srcfile, IsVerbose(1) ? '-v' : undef, '-C', $pkgTargetDir );
439
        my $rv = System ('tar', '-xzmf', $srcfile, IsVerbose(1) ? '-v' : undef, '-C', $pkgTargetDir );
354
        Error("Tar extraction error: $srcfile") if ($rv);
440
        Error("Tar extraction error: $srcfile") if ($rv);
355
    }
441
    }
356
 
442
 
357
    #
443
    #
-
 
444
    #   Replace the built.files.xxx.xml files that came with each package fragment
-
 
445
    #   with a new one caclulated as we merged the fragemnts. The new one will not
-
 
446
    #   have duplicate files - they will be merked as merged.
-
 
447
    #   
-
 
448
    #   Delete existing built.files.xxx.xml
-
 
449
    #   Write out file meta data for the assembled package
-
 
450
    #
-
 
451
    foreach my $item (glob(catdir($pkgTargetDir, 'built.files.*.xml')))
-
 
452
    {
-
 
453
        Verbose("Delete metadata file: $item");
-
 
454
        unlink $item;
-
 
455
    }
-
 
456
 
-
 
457
    Message("Write new archive metadata");
-
 
458
    writeFileInfo(catfile($pkgTargetDir, 'built.files.packageAssembly.xml'),\@newXml);
-
 
459
 
-
 
460
    #
358
    #   Fix file permissions
461
    #   Fix file permissions
359
    #   We know we are running under unix so we will use a unix command
462
    #   We know we are running under unix so we will use a unix command
360
    #
463
    #
361
    Message('Setting file permissions');
464
    Message('Setting file permissions');
362
    System('chmod', '-R', 'a+rx', $pkgTargetDir);
465
    System('chmod', '-R', 'a+rx', $pkgTargetDir);
Line 429... Line 532...
429
        rmdir($pkgDir) && Message("Remove package dir: $pkgDir");
532
        rmdir($pkgDir) && Message("Remove package dir: $pkgDir");
430
    }
533
    }
431
}
534
}
432
 
535
 
433
#-------------------------------------------------------------------------------
536
#-------------------------------------------------------------------------------
-
 
537
# Function        : writeFileInfo 
-
 
538
#
-
 
539
# Description     : Write out an XML file that contains this processes
-
 
540
#                   contribution to the output package 
-
 
541
#
-
 
542
# Inputs          : $targetFile             - File to write XML into
-
 
543
#                   $fileList               - Ref to an array of file data 
-
 
544
#
-
 
545
# Returns         : 
-
 
546
#
-
 
547
sub writeFileInfo
-
 
548
{
-
 
549
    my ($targetFile, $fileList) = @_;
-
 
550
 
-
 
551
    my $data;
-
 
552
    $data->{file} = $fileList;
-
 
553
 
-
 
554
    #
-
 
555
    #   Write out sections of XML
-
 
556
    #       Want control over the output order
-
 
557
    #       Use lots of attributes and only elements for arrays
-
 
558
    #       Save as one attribute per line - for readability
-
 
559
    #
-
 
560
    my $xs = XML::Simple->new( NoAttr =>0, AttrIndent => 1 );
-
 
561
 
-
 
562
    open (my $XML, '>', $targetFile) || Error ("Cannot create output file: $targetFile", $!);
-
 
563
    $xs->XMLout($data, 
-
 
564
                'RootName' => 'files', 
-
 
565
                'XMLDecl'  => '<?xml version="1.0" encoding="UTF-8"?>',
-
 
566
                'OutputFile' => $XML);
-
 
567
    close $XML;
-
 
568
 
-
 
569
}
-
 
570
 
-
 
571
 
-
 
572
#-------------------------------------------------------------------------------
434
# Function        : reportMergeError 
573
# Function        : reportMergeError 
435
#
574
#
436
# Description     : Report an error or a warning
575
# Description     : Report an error or a warning
437
#
576
#
438
# Inputs          : All arguments passed to ReportError or Warning
577
# Inputs          : All arguments passed to ReportError or Warning