Subversion Repositories DevTools

Rev

Rev 7300 | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
227 dpurdie 1
########################################################################
7300 dpurdie 2
# COPYRIGHT - VIX IP PTY LTD ("VIX"). ALL RIGHTS RESERVED.
227 dpurdie 3
#
4
# Module name   : cache_dpkg
5744 dpurdie 5
# Module type   : JATS Utility
6
# Compiler(s)   : Perl
227 dpurdie 7
# Environment(s): jats
8
#
9
# Description:
5743 dpurdie 10
#       Maintain a local cache of dpkg_archive
227 dpurdie 11
#
12
# Notes:
335 dpurdie 13
#       Stopped using the JATS "cp.exe" utility as there was a weird problem under
227 dpurdie 14
#       windows. The package orahops-ssw-install/1.0.1000.ssw could not be installed
335 dpurdie 15
#       correctly. It appears that the 'cp' could not process the subdir pair
227 dpurdie 16
#       "DBMGR/DBMGR". Change the name it was OK.
17
#
18
#       Solution: Avoid system functions
19
#
20
#......................................................................#
21
 
22
use strict;
23
use warnings;
24
use JatsError;
25
use FileUtils;
26
use File::Find;
27
use File::Path;
28
use File::Copy;
29
 
30
use Getopt::Long;
31
use Pod::Usage;                             # required for help support
32
 
5743 dpurdie 33
my $VERSION = "1.5.0";
227 dpurdie 34
 
35
 
36
#
37
#   Options
38
#
39
my $opt_debug   = $ENV{'GBE_DEBUG'};        # Allow global debug
40
my $opt_verbose = $ENV{'GBE_VERBOSE'};      # Allow global verbose
41
my $opt_help = 0;
42
my $opt_clear;
43
my $opt_flush;
44
my $opt_refresh;
45
my $opt_refresh_all;
46
my $opt_list;
47
my $opt_list_short;
48
my $opt_export;
49
my $opt_quiet;
50
my $opt_update_all;
51
my $opt_age;
361 dpurdie 52
my $opt_test;
5744 dpurdie 53
my $opt_wait = 0;
5783 dpurdie 54
my $opt_cache = 1;
227 dpurdie 55
 
56
#
57
#   Global Variables
58
#
4688 dpurdie 59
my $GBE_DPKG            = $ENV{'GBE_DPKG'};                     # The Master repository
60
my $GBE_DPKG_CACHE      = $ENV{'GBE_DPKG_CACHE'} || "";         # Caches
61
my $GBE_DPKG_LOCAL      = $ENV{'GBE_DPKG_LOCAL'} || "";         # Local scratch
62
my $GBE_DPKG_STORE      = $ENV{'GBE_DPKG_STORE'} || "";         # Global Store
63
my $GBE_DPKG_REPLICA    = $ENV{'GBE_DPKG_REPLICA'} || "";       # Site Local Replica
7319 dpurdie 64
my $GBE_DPKG_ESCROW     = $ENV{'GBE_DPKG_ESCROW'} || "";        # Escrow Store
4688 dpurdie 65
my $GBE_BIN             = $ENV{'GBE_BIN'};
227 dpurdie 66
 
4688 dpurdie 67
my $istore;
7319 dpurdie 68
my $estore;
227 dpurdie 69
my $gstore;
70
my $archive;
71
my $cache;
72
my $parchive;
73
my $local = '';
74
my $local_pkg;
75
my @package_list;
76
my @refresh_list;
5818 dpurdie 77
 
227 dpurdie 78
#
79
#   Globals for recursive copy
80
#
81
my $copyFind_dst;
82
my $copyFind_src;
83
my $copyFind_src_len;
5818 dpurdie 84
my $copyFind_touch;
5862 dpurdie 85
my $copyFind_time;
86
my @copyFindDups;
227 dpurdie 87
 
88
#
89
#   Globals for error recovery
90
#
91
my  $remove_on_error;
5818 dpurdie 92
my  $cacheMarker;
227 dpurdie 93
 
94
#-------------------------------------------------------------------------------
95
# Function        : Mainline Entry Point
96
#
97
# Description     :
98
#
99
# Inputs          :
100
#
101
my $result = GetOptions (
337 dpurdie 102
                "help|h:+"                  => \$opt_help,
103
                "manual:3"                  => \$opt_help,
104
                "verbose:+"                 => \$opt_verbose,           # flag, multiple use allowed
105
                "debug:+"                   => \$opt_debug,             # flag, multiple use allowed
227 dpurdie 106
                "flush"                     => \$opt_flush,             # flag
107
                "clear"                     => \$opt_clear,             # flag
5744 dpurdie 108
                "wait!"                     => \$opt_wait,              # [no]flag
5783 dpurdie 109
                "cache!"                    => \$opt_cache,             # [no]flag
5744 dpurdie 110
                "refresh!"                  => \$opt_refresh,           # [no]flag
227 dpurdie 111
                "refresh_all|refresh-all"   => \$opt_refresh_all,       # flag
112
                "update_all|update-all"     => \$opt_update_all,        # flag
113
                "list"                      => \$opt_list,              # flag
114
                "dir"                       => \$opt_list_short,        # flag
115
                "export"                    => \$opt_export,            # flag
116
                "quiet"                     => \$opt_quiet,             # flag
117
                "age=i"                     => \$opt_age,               # integer
361 dpurdie 118
                "test"                      => \$opt_test,              # flag
227 dpurdie 119
                );
120
 
121
                #
122
                #   UPDATE THE DOCUMENTATION AT THE END OF THIS FILE !!!
123
                #
124
 
125
#
126
#   Process help and manual options
127
#
128
pod2usage(-verbose => 0, -message => "Version: $VERSION")  if ($opt_help == 1  || ! $result);
129
pod2usage(-verbose => 1)  if ($opt_help == 2 );
337 dpurdie 130
pod2usage(-verbose => 2)  if ($opt_help > 2);
227 dpurdie 131
 
132
#
133
#   Configure the error reporting process now that we have the user options
134
#
135
ErrorConfig( 'name'    =>'CACHE',
136
             'verbose' => $opt_verbose,
137
             'debug'   => $opt_debug );
138
 
139
#
140
#   Validate user options
141
#
142
Error( "GBE_BIN not defined in the environment" ) unless ( defined($GBE_BIN) );
143
Error( "GBE_DPKG not defined in the environment" ) unless ( defined($GBE_DPKG) );
144
 
5744 dpurdie 145
$opt_refresh = 1
146
    if ( $opt_refresh_all );
147
 
227 dpurdie 148
#
5744 dpurdie 149
#   Locate the various package stores
150
#   Search order for package should be:
151
#       DPKG_SANDBOX (Not cached)
152
#       DPKG_LOCAL   (To be deprecated)
153
#       DPKG_CACHE
7319 dpurdie 154
#       DPKG_ESCROW
5744 dpurdie 155
#       DPKG_REPLICA
156
#       DPKG         (Writable to build system)
157
#       DPKG_STORE
227 dpurdie 158
#
5744 dpurdie 159
$local = $GBE_DPKG_LOCAL;
160
$cache = $GBE_DPKG_CACHE;
7319 dpurdie 161
$estore = $GBE_DPKG_ESCROW;
5744 dpurdie 162
$istore = $GBE_DPKG_REPLICA;
163
$archive = $GBE_DPKG;
164
$gstore = $GBE_DPKG_STORE;
165
 
166
Error ("dpkg_archive cache is the main archive")    if ( $cache eq $archive );
167
Error ("dpkg_archive replica is the main archive")  if ( $istore && $istore eq $archive );
168
Error ("main archive is local archive" )            if ( $local && $local eq $archive );
169
Warning  ("dpkg_archive_cache is local_archive" )   if ( $cache && $local && $local eq $cache );
170
 
171
#
172
#   Perform package/version replication wait
173
#   
174
#   Do not need a cache for this to be performed as the operation is used on build servers
175
#   that do not have a cache (most Unix build machines)
176
#   
177
#   Cannot wait if we don't have a REPLICA
178
#   
179
if ($opt_wait && $istore)
227 dpurdie 180
{
5744 dpurdie 181
    Verbose("Wait for package replication");
182
    #
183
    #   Scan the argument list
184
    #   Only wait for package/version. Cannot wait on an entire package
185
    #
186
    foreach (@ARGV)
187
    {
188
        unless (m~.+/.+~)
189
        {
190
            Verbose("Wait Skip $_");
191
            next;
192
        }
227 dpurdie 193
 
5744 dpurdie 194
        if (-d "$istore/$_")
195
        {
196
            Verbose("Wait not required. Version in replica: $_");
197
            next;
198
        }
227 dpurdie 199
 
5744 dpurdie 200
        unless (-d "$archive/$_")
201
        {
202
            Verbose("Wait not required. Version not in store: $_");
203
            next;
204
        }
227 dpurdie 205
 
5744 dpurdie 206
        #
207
        #   Wait for the package to be replicated
208
        #       Wait up to 10 minutes
209
        #       Or the package to appear in the replica
210
        #       Or the package to disappear from the main store
211
        #
212
        Verbose("Waiting for replication of: $_");
213
        my $waitStart = time();
214
        while (1)
215
        {
216
            sleep(30);
217
            my $delta = time() - $waitStart;
218
            if (-d "$istore/$_")
219
            {
220
                Message("PkgReplication. Package replicated ($delta): $_");
221
                last;
222
            }
223
            unless (-d "$archive/$_")
224
            {
225
                Message("PkgReplication. Package deleted from main archive: $_");
226
                last;
227
            }
228
            #
229
            #   Done wait forever
230
            #   If the package has not been replicate we will just have to copy it the slow way
231
            #   
232
            if ( $delta >= 600)
233
            {
234
                Message("PkgReplication. Package not replicated: $_");
235
                last;
236
            }
237
        }
238
    }
239
}
240
 
227 dpurdie 241
#
5783 dpurdie 242
#   Allow replication update without also caching the package
243
#       This is used to prevent ALL replicated packages beinng cached.
244
#       Packages do not need to be cached if they are not used by a build on the current machine
245
#       This will prevent the cache from filling up with unused packages.
246
#
247
unless ($opt_cache)
248
{
249
    Message("Packages not cached");
250
    exit 0;
251
}
252
 
253
#
5744 dpurdie 254
#   No cache - nothing to do
255
#   Generate a status message and exit
227 dpurdie 256
#
5744 dpurdie 257
unless ( $cache )
227 dpurdie 258
{
5744 dpurdie 259
    Warning ( "GBE_DPKG_CACHE not defined in the environment" );
260
    exit 0;
227 dpurdie 261
}
262
 
263
#
264
#   Export the list of cache entries
265
#
266
if ( $opt_export or $opt_refresh_all or $opt_list_short or $opt_update_all )
267
{
268
    opendir (DIR, $cache) || die "Cannot open $cache\n";
269
    my @dir_list = readdir(DIR);
270
    closedir DIR;
271
 
272
    for my $fn ( @dir_list)
273
    {
274
        my $count = 0;
275
        next if ( $fn =~ m/^\./ );
276
        next unless ( -d "$cache/$fn" );
277
 
278
        opendir( DIR, "$cache/$fn" ) || die "Cannot open $cache/$fn\n";
279
        while ( my $fn1 = readdir(DIR) )
280
        {
281
            next if ( $fn1 =~ m/^\./ );
282
            push @package_list, "$fn/$fn1";
283
        }
284
        closedir DIR;
285
    }
286
 
287
    if ( $opt_refresh_all or $opt_update_all )
288
    {
289
        @refresh_list = @package_list
290
    }
291
 
292
    #
293
    #   Simply export a list of packages in the cache
294
    #   Used by other programs
295
    #
296
    if ( $opt_export )
297
    {
298
        print join ' ', @package_list;
299
        exit 0;
300
    }
301
}
302
 
303
#
304
#   Display cache information
305
#   This is done AFTER the export - since the export command does not
306
#   need this header.
307
if ( $opt_verbose || $#ARGV < 0 )
308
{
7319 dpurdie 309
    print  "dpkg_store          : $gstore\n" if ($gstore);
310
    print  "dpkg_escrow         : $estore\n" if ($estore);
311
    print  "dpkg_archive        : $archive\n";
312
    print  "dpkg_archive replica: $istore\n";
313
    print  "dpkg_archive cache  : $cache\n";
314
    print  "dpkg_archive local  : $local\n";
227 dpurdie 315
    Verbose ("args:               @ARGV");
316
}
317
 
318
#
319
#   List the contents of the cache
320
#
321
if ( $opt_list_short )
322
{
323
    print join "\n", @package_list;
324
}
325
 
326
if ( $opt_list )
327
{
328
    #
329
    #   Process user commands
330
    #   List: A nice pretty display of the cache
331
    #
332
    opendir (DIR, $cache) || die "Cannot open $cache\n";
333
    my @dir_list = readdir(DIR);
334
    closedir DIR;
335
 
336
    #
337
    #   Determine max length of a name so that the display is nicely aligned
338
    #
339
    my $nl = 10;
340
    for my $fn ( @dir_list)
341
    {
342
        my $ns = length($fn);
343
        $nl = $ns if ( $ns > $nl )
344
    }
345
 
346
    #
347
    #   Display information by package
348
    #   A nicely formatted list with line wrapping
349
    #
350
    for my $fn ( @dir_list)
351
    {
352
        my $count = 0;
353
        next if ( $fn =~ m/^\./ );
354
        next unless ( -d "$cache/$fn" );
355
 
356
        opendir( DIR, "$cache/$fn" ) || die "Cannot open $cache/$fn\n";
357
        printf "%-*s :", $nl, $fn;
358
        while ( my $fn1 = readdir(DIR) )
359
        {
360
            next if ( $fn1 =~ m/^\./ );
361
            if ( $count++ >= 4 )
362
            {
363
                $count = 1;
364
                printf "\n%*s  ", $nl, "";
365
            }
366
            printf " %14.14s", $fn1;
367
        }
368
        closedir DIR;
369
        print "\n";
370
    }
371
}
372
 
373
#
374
#   Clear the cache
375
#
376
if ( $opt_clear )
377
{
378
    Warning   ( "Deleting the ENTIRE cache");
379
    rmtree( $cache, $opt_debug );
380
    mkpath( $cache, $opt_debug, 0777);
381
}
382
 
383
#
384
#   Perform cache aging
385
#
386
if ( $opt_age )
387
{
388
    Error ("Age parameter cannot be negative") if ( $opt_age < 0 );
389
    age_the_cache();
390
}
391
 
392
#
393
#   Process the command line arguments
394
#   These MUST be of the form
395
#       packagename
396
#       packagename/version
397
#
398
for (@ARGV, @refresh_list)
399
{
400
    $remove_on_error = undef;
5818 dpurdie 401
    $cacheMarker = undef;
402
 
227 dpurdie 403
    if ( $opt_flush )
404
    {
405
        unless ( -d "$cache/$_" )
406
        {
407
            Warning ("Package not in cache: $_")
408
                unless ( $opt_quiet );
409
        }
410
        else
411
        {
412
            rmtree( "$cache/$_", $opt_debug );
413
            print "Package removed: $_\n"
414
                unless ( $opt_quiet );
415
        }
416
    }
417
    else
418
    {
419
        #
420
        #   Speed up - for remote access
421
        #   If the package is found in the local archive and we aren't doing
422
        #   anything fancy then don't access the remote archive
423
        #
424
        my $local_pkg = ( -d "$local/$_" || -f "$local/$_.lnk" );
425
        if ( $local_pkg )
426
        {
427
            if ( !$opt_refresh )
428
            {
429
                print "Cache SkipLocal: $_ Local copy found\n";
430
 
431
                #
432
                # Delete the cache copy
433
                # The user is playing with local copy
434
                #
435
                if ( -d "$cache/$_" )
436
                {
437
                    print  "Cache SkipLocalDelete: $_ -> $cache\n";
438
                    Verbose ( "Remove: $_" );
439
                    rmtree( "$cache/$_", $opt_debug );
440
                    Verbose ( "Remove Complete: $_" );
441
                }
442
                next;
443
            }
444
        }
445
 
446
        #
447
        #   Locate package.
4688 dpurdie 448
        #   It may be in GBE_DPKG_REPLICA, GBE_DPKG or GBE_DPKG_STORE
7319 dpurdie 449
        #   Or it may only be in GBE_DPKG_ESCROW
6133 dpurdie 450
        #       Package may (jats2_current) be a symlink
227 dpurdie 451
        #
4688 dpurdie 452
        my $pkg_found;
7319 dpurdie 453
        my @storeList;
454
        if ($estore) {
455
            push @storeList, $estore;
456
        } else {
457
            push @storeList,  $istore, $archive, $gstore ;
458
        }
459
        foreach my $store (@storeList)
227 dpurdie 460
        {
4688 dpurdie 461
            $parchive = "$store/$_";
6133 dpurdie 462
 
463
            if (-l $parchive)
464
            {
465
                my $linkDir = readlink($parchive);
466
                $parchive =~ s~/[^/]+$~/$linkDir~;
467
            }
468
 
4688 dpurdie 469
            if ( -d $parchive )
227 dpurdie 470
            {
4688 dpurdie 471
                $pkg_found = 1;
472
                last;
227 dpurdie 473
            }
474
        }
4688 dpurdie 475
 
476
        unless ($pkg_found )
477
        {
478
            Warning ("Package not in archive: $_")
479
                unless ( $opt_quiet );
480
            next;
481
        }
227 dpurdie 482
 
483
        ########################################################################
484
        #   We have a package to process
485
        #
5818 dpurdie 486
        my $dir_found = 0;
227 dpurdie 487
        my $force_update = 0;
488
        my $opr = "Update";
489
 
5818 dpurdie 490
        #
491
        #   Generate the 'name' of a marker file used to indicate that the version is being updated
492
        #   This is outside the target directory
493
        #   
494
        $cacheMarker = "$_/built.cache";
495
        $cacheMarker =~ tr~/~_~s;
496
        $cacheMarker =~ tr~_~_~s;
497
        $cacheMarker =  $cache . '/' . $cacheMarker;
498
        Verbose2("cacheMarker: $cacheMarker");
227 dpurdie 499
 
500
        #
501
        #   Setup error recovery
335 dpurdie 502
        #       1) Tag the directory to be deleted on error
227 dpurdie 503
        #
504
        $remove_on_error = "$cache/$_";
505
        ErrorConfig( 'on_exit' => \&error_recovery );
506
 
507
        #
335 dpurdie 508
        #   Not a forced refresh. Ensure that the cached copy is
227 dpurdie 509
        #   up to date. Examine descpkg
510
        #
5818 dpurdie 511
        $dir_found = waitForComplete($_,$cache);
512
        if ( $dir_found && !$opt_refresh )
227 dpurdie 513
        {
5818 dpurdie 514
            if ( FileIsNewer( "$parchive/descpkg", "$cache/$_/descpkg" ) )
227 dpurdie 515
            {
516
                $force_update = 1;
517
                $opr = "OutOfDate";
5818 dpurdie 518
                TouchFile ( $cacheMarker, "Marks the cache copy as incomplete");
227 dpurdie 519
                Verbose ("Cache out-of-date: $_");
520
            }
521
        }
522
 
523
        #
524
        #   If we need to refresh the cache copy - delete it first
525
        #
526
        if ( ($opt_refresh || $force_update) && $dir_found )
527
        {
528
            print  "Cache $opr: $_ -> $cache\n";
529
            Verbose ( "Remove: $_" );
530
            rmtree( "$cache/$_", $opt_debug );
531
            Verbose ( "Remove Complete: $_" );
229 dpurdie 532
 
533
            #
534
            #   Force transfer, but without the status message
535
            #
227 dpurdie 536
            $dir_found = 0;
229 dpurdie 537
            $opr = '';
227 dpurdie 538
        }
539
 
540
        #
5818 dpurdie 541
        #   If the directory exists, then we need to avoid a race condition
542
        #   where multiple instances are updating the same directory
543
        #       Need to aviod hanging forever if the primary updator does not
544
        #       complete the task
227 dpurdie 545
        #   If its not in the cache then copy it in
546
        #
5818 dpurdie 547
        $dir_found = waitForComplete($_,$cache);
227 dpurdie 548
        unless ( $dir_found )
549
        {
229 dpurdie 550
            print "Cache $opr: $_ -> $cache\n" if $opr;
227 dpurdie 551
            mkpath( "$cache/$_", $opt_debug, 0777);
5818 dpurdie 552
            TouchFile ( $cacheMarker, "Marks the cache copy as incomplete");
227 dpurdie 553
            Verbose ( "Copy in: $_" );
554
            $copyFind_dst = "$cache/$_";
555
            $copyFind_src = $parchive;
556
            $copyFind_src_len = length( $copyFind_src );
5818 dpurdie 557
            $copyFind_touch = $cacheMarker;
558
            $copyFind_time = 0;
5862 dpurdie 559
            undef @copyFindDups;
227 dpurdie 560
            File::Find::find( \&copyFind, $parchive );
5862 dpurdie 561
            if (@copyFindDups)
562
            {
563
                Warning("The following items where not transferred as they already existed",
564
                        "This may be due to symlinks or (Windows) case insensitive filename", @copyFindDups);
565
            }
5818 dpurdie 566
            rmtree( $cacheMarker, $opt_debug );  # Works on files too !!
227 dpurdie 567
        }
568
        else
569
        {
570
            $opr = "Skip";
571
            print "Cache $opr: $_ -> $cache\n";
572
        }
573
    }
574
}
575
 
576
#-------------------------------------------------------------------------------
5818 dpurdie 577
# Function        : waitForComplete 
578
#
579
# Description     : Wait for a package-version to complete
580
#                   At the end of this operation the directory will be 
581
#                   completly transferred, or it will be deleted
582
#                   
583
#                   If the directory exists, then we need to avoid a race condition
584
#                   where multiple instances are updating the same directory
585
#                   Need to avoid hanging forever if the primary updator does not
586
#                   complete the task
587
#
588
# Inputs          : $dir    - Subdir directory to monitor
589
#                   $cache  - Target cache
590
#
591
# Returns         : 0   - $dir does not exist
592
#                   1   - $dir does exist 
593
#
594
sub waitForComplete
595
{
596
    my ($dir, $cache) = @_;
597
    my $opr;
598
    my $tgtDir = "$cache/$dir";
599
    #
600
    #   Directory not preset
601
    #       All done - dir does not exist 
602
    #
603
    if (! -d $tgtDir)
604
    {
605
        return 0;
606
    }
607
 
608
    # If the package is not being updated, then we are done if the 
609
    # package being updated marker is NOT present
610
    #     
611
    if (  -d $tgtDir && ! -f $cacheMarker)
612
    {
613
        return 98;
614
    }
615
 
616
    #
617
    #   The package-version is being updated by another instance
618
    #   Hang around waiting for the update to complete or terminate
619
    #       Wait 10 minutes max
620
    #
621
    my  $waitStart = time();
622
    $opr = "Wait";
623
    print "Cache $opr: $dir -> $cache\n";
624
    while (1)
625
    {
626
        if ( ! -d $tgtDir )
627
        {
628
            # Directory has gone away
629
            #   Must have been a bad transfer by another instance
630
            #   Start the processing again
631
            $opr = "Retry";
632
            print "Cache $opr: $dir -> $cache\n";
633
            return 0;
634
        }
635
 
636
        if ( ! -f $cacheMarker )
637
        {
638
            #
639
            #   Update must have completed - marker file has been removed
640
            #
641
            $opr = "Found";
642
            print "Cache $opr: $_ -> $cache\n";
643
            return 97;
644
        }
645
 
646
        #
647
        #   Determine the 'age' of the marker file
648
        #       It will be updated after each file in the package has been copied
649
        #       If it is too old, then we must assume that the other instance died
650
        #   Complication. Windows does not correctly report the 'modified' time of a file
651
        #                 It gets messed up by the file system. Can't use the modified info
652
        #                 from Perls stat function to determine the absolute modified time of the
653
        #                 file.
654
        #                 
655
        #       Solution: Create a temp file and determine the age relative to that file
656
        #       
657
 
658
        my $cacheMarkerTmp = $cacheMarker . ".tmp";
659
        TouchFile($cacheMarkerTmp, "Relative time stamp");
660
        my $now   = (stat ($cacheMarkerTmp))[9];
661
        my $mtime = (stat ($cacheMarker))[9];
662
        rmtree( $cacheMarkerTmp, $opt_debug );
663
 
664
        #
665
        #   If the marker file is older than 5 minutes then we can consider the package as
666
        #   a dud. The marker file will be updated be every file write into the cache. Its
667
        #   an indication of activity.
668
        #   
669
        #   Assume that no file is going to take more then 5 minutes to copy
670
        #   Lets see how this goes.
671
        #   
672
        my $age = $now - $mtime;
673
        if ( $age > 5 * 60)
674
        {
675
            #
676
            #   Update taking too long
677
            #       Delete the package-version
678
            #       Try again
679
            #
680
            $opr = "Remove Bad";
681
            print "Cache $opr: $_ -> $cache\n";
682
            rmtree( "$cache/$_", $opt_debug );
683
 
684
            return 0;
685
        }
686
 
687
        #
688
        #   Wait a short while
689
        #
690
        Verbose("Waiting: " . (time() - $waitStart));
691
        sleep(5);
692
    }
693
}
694
 
695
 
696
#-------------------------------------------------------------------------------
227 dpurdie 697
# Function        : copyFind
698
#
699
# Description     : File:Find:find callback function to transfer files
700
#
701
# Inputs          : None
5862 dpurdie 702
#                   Global: $copyFind_dst       - Target directory
703
#                   Global: $copyFind_src       - Source directory
704
#                   Global: $copyFind_src_len   - Length of Source dir
705
#                   Global: $copyFind_touch     - File to touch after each operation
706
#                   Global: $copyFind_time      - Time of last touch
227 dpurdie 707
#
5862 dpurdie 708
# Returns         : Global: @copyFindDups       - Array of files not copied due to previous existence
227 dpurdie 709
#
710
 
711
sub copyFind
712
{
713
    my $item = $File::Find::name;
5818 dpurdie 714
#Debug0("Remove sleep in copy");
715
#sleep(1);
227 dpurdie 716
 
717
    #
718
    #   Calculate the target directory name
719
    #
5743 dpurdie 720
    my $tgt_path = substr($item, $copyFind_src_len );
721
    my $target = $copyFind_dst . $tgt_path;
722
 
723
    # Do not cache some parts of a package when being used by the build system
724
    #   /lcov ...
725
    #
726
    if ( $ENV{GBE_ABT})
727
    {
728
        if ($tgt_path =~ m~^/lcov(/|$)~)
729
        {
730
            Verbose("Prune directory: $tgt_path");
731
            $File::Find::prune = 1;
732
            return;
733
        }
734
    }
735
 
227 dpurdie 736
    if ( -d $item )
737
    {
738
        #
739
        #   Directories are handled differently
740
        #       - Directories are created with nice permissions
741
        #
742
        if ( ! -d $target )
743
        {
744
            mkpath( $target, $opt_debug, 0755);
745
        }
746
    }
747
    else
748
    {
749
        #
750
        #   File copy
751
        #
752
        #
753
        #   Copy file to destination
754
        #   If the file is a link, then duplicate the link contents
755
        #   Use: Unix libraries are created as two files:
756
        #        lib.xxxx.so -> libxxxx.so.vv.vv.vv
757
        #
758
        if ( -l $item )
759
        {
760
            Debug("Clone Link: $target");
761
            my $link = readlink $item;
762
            Debug( "Link: $item, $link");
763
            symlink ($link, $target );
764
            unless ( $link && -l $target )
765
            {
766
                Error("Failed to copy link [$item] to [$target]: $!");
767
            }
768
        }
5862 dpurdie 769
        elsif (-f $target)
770
        {
771
            # File already exists
772
            #   Most likely Windows filename clash. Windows files are case-insensitive
773
            push @copyFindDups, $tgt_path
774
        }
227 dpurdie 775
        elsif (File::Copy::copy($item, $target))
776
        {
777
            Debug("Copying File: $target");
778
 
779
            #   Make the file ReadOnly
780
            my $perm = (stat $item)[2] & 07777;
781
            CORE::chmod $perm & 0555, $target;
782
        }
783
        else
784
        {
785
            Error("Failed to copy file [$item] to [$target]: $!");
786
        }
787
    }
5818 dpurdie 788
 
789
    #
790
    #   Touch this file to indicate that the copy operation is still in progress
791
    #   Assists in early detection of partial copies
792
    #   
793
    #   Only touch once a minute to prevent hammering the file system
794
    #
795
    my $age = time() - $copyFind_time;
796
    if ($age > 60)
797
    {
798
        $copyFind_time = time();
799
        TouchFile ( $copyFind_touch, "Marks the cache copy as incomplete");
800
    }
227 dpurdie 801
}
802
 
803
#-------------------------------------------------------------------------------
804
# Function        : error_recovery
805
#
806
# Description     : Error recovery routine
807
#                   Delete the cached entry
808
#
809
# Inputs          :  Globals
810
#
811
# Returns         : None
812
#
813
sub error_recovery
814
{
815
    if ( $remove_on_error )
816
    {
345 dpurdie 817
        ReportError ("Error cleanup. Delete cache entry: $remove_on_error");
227 dpurdie 818
        rmtree( $remove_on_error, $opt_debug );
819
        $remove_on_error = undef;
820
    }
5818 dpurdie 821
 
822
    if ($cacheMarker)
823
    {
824
        ReportError ("Error cleanup. Delete cache marker: $cacheMarker");
825
        rmtree( $cacheMarker, $opt_debug );
826
        $cacheMarker = undef;
827
    }
227 dpurdie 828
}
829
 
830
 
831
#-------------------------------------------------------------------------------
832
# Function        : age_the_cache
833
#
834
# Description     : Age cache entries
835
#                   Determine the age by:
836
#                       Use used.cache file if present
837
#                       Use descpkg file
838
#                       Not a proper entry - delete
839
#
840
# Inputs          : opt_age         - Delete packages older than XX days
841
#
842
# Returns         : Nothing
843
#
844
sub age_the_cache
845
{
846
    my $now = time;
847
 
848
    opendir (DIR, $cache) || die "Cannot open $cache\n";
849
    my @dir_list = readdir(DIR);
850
    closedir DIR;
851
 
852
    for my $fn ( @dir_list)
853
    {
854
        my $keep_dir = 0;
855
        next if ( $fn =~ m/^\./ );
856
        next unless ( -d "$cache/$fn" );
361 dpurdie 857
        next if ( $fn eq "core_devl" );
227 dpurdie 858
 
859
        opendir( DIR, "$cache/$fn" ) || die "Cannot open $cache/$fn\n";
860
        while ( my $fn1 = readdir(DIR) )
861
        {
862
            next if ( $fn1 =~ m/^\./ );
863
            my $dir = "$cache/$fn/$fn1";
864
            my $file = "$dir/used.cache" ;
865
            $file = "$dir/descpkg" unless ( -f $file );
866
 
867
            if ( ! -f $file )
868
            {
869
                #
870
                #   No descpkg file
871
                #   This is a badly formed entry - so delete it
872
                #
361 dpurdie 873
                if ( $opt_test )
874
                {
875
                    Message("Would Purge: $fn/$fn1" );
876
                    $keep_dir = 1;
877
                }
878
                else
879
                {
880
                    Message("Purging: $fn/$fn1" );
881
                    rmtree( $dir, $opt_debug );
882
                }
227 dpurdie 883
            }
884
            else
885
            {
886
                #
887
                #   used.cache or descpkg file found
888
                #   How old is it
889
                #
890
                my $timestamp = (stat($file))[9] || 0;
891
                my $age = int( ($now - $timestamp) / (60 * 60 * 24));
892
 
893
                if ( $age > $opt_age )
894
                {
361 dpurdie 895
                    if ( $opt_test )
896
                    {
897
                        Message("Could Age: $fn/$fn1, $age" );
898
                        $keep_dir = 1;
899
                    } else {
900
                        Message("Aging: $fn/$fn1, $age" );
901
                        rmtree( $dir, $opt_debug );
902
                    }
227 dpurdie 903
                }
904
                else
905
                {
906
                    Verbose("Age of: $fn/$fn1, $age" );
907
                    $keep_dir = 1;
908
                }
909
            }
910
        }
911
        closedir DIR;
912
 
913
        #
914
        #   Delete the entire directory if is is empty
915
        #
916
        unless ( $keep_dir )
917
        {
918
            Message("Remove Empty Dir: $cache/$fn"  );
919
            rmtree( "$cache/$fn", $opt_debug );
920
        }
921
    }
922
}
923
 
924
#-------------------------------------------------------------------------------
925
#   Documentation
926
#
927
 
928
=pod
929
 
361 dpurdie 930
=for htmltoc    SYSUTIL::
931
 
227 dpurdie 932
=head1 NAME
933
 
934
cache_dpkg - Maintain a local cache of packages
935
 
936
=head1 SYNOPSIS
937
 
938
 jats cache_dpkg.pl [options] package/version ...
939
 
940
 Options:
941
    -help              - brief help message
942
    -help -help        - Detailed help message
943
    -man               - Full documentation
944
    -clear             - Delete the entire cache
945
    -flush             - Flush all named packages
946
    -[no]refresh       - Refresh package in cache
947
    -list              - List cache contents with nice format
948
    -dir               - List cache contents
949
    -export            - Generate a list of cached packages
950
    -refresh_all       - Refresh all packages within the cache
951
    -update_all        - Update all packages within the cache as required
5744 dpurdie 952
    -[no]wait          - Wait for package replication
5783 dpurdie 953
    -[no]cache         - Cache packages [default]
227 dpurdie 954
    -quiet             - Suppress warnings
361 dpurdie 955
    -age=nn            - Remove all packages older than nn days
956
    -test              - Use with -age to report ages
227 dpurdie 957
 
958
=head1 OPTIONS
959
 
960
=over 8
961
 
962
=item B<-help>
963
 
964
Print a brief help message and exits.
965
 
966
=item B<-help -help>
967
 
968
Print a detailed help message with an explanation for each option.
969
 
970
=item B<-man>
971
 
972
Prints the manual page and exits.
973
 
974
=item B<-clear>
975
 
976
Delete the B<entire> contents of the dpkg_archive cache. This will occur before
977
any new packages are copied into the cache.
978
 
979
=item B<-flush>
980
 
981
If set then the utility will delete the named packages from the cache. All named
982
packaged will be deleted. This option affects all named packages. It is not
983
possible to flush and copy packages with the same command.
984
 
985
=item B<-[no]refresh>
986
 
987
If the B<refresh> option has been specified then packages will be deleted, from
988
the cache and then a new copy will be copied in. If not specified then no copy
989
will occur if the package is present in the cache.
990
 
991
=item B<-list>
992
 
993
Display a list of all packages in the cache. A formatted display is generated.
994
 
995
This will be done before any packages are transferred.
996
 
997
=item B<-dir>
998
 
999
Display a list of all packages in the cache. This is a raw directory like
1000
listing of the cache.
1001
 
1002
This will be done before any packages are transferred.
1003
 
1004
=item B<-export>
1005
 
1006
Generate a space separated list of cached packages as a single line. This is
1007
intended to allow a list be exported for later import.
1008
 
1009
=item B<-refresh_all>
1010
 
1011
This option will force the program to refresh all the packages in the cache.
1012
This forces a B<-refresh> and may be combined with other packages specified
1013
on the command line.
1014
 
1015
=item B<-update_all>
1016
 
1017
This option will force the program to examine all packages within the cache and
1018
refresh packages that are out of date. This option may be combined with other
1019
packages specified on the command line.
1020
 
1021
A package is deemed to be out-of-date if the modification time of the package's
335 dpurdie 1022
descpkg file in the cache is older than the one in the archive.
227 dpurdie 1023
 
5744 dpurdie 1024
=item B<-[no]wait>
1025
 
1026
This option will cause the utility to wait for specified package versions to be replicated
1027
into a package replica. The dafult mode is to not-wait, unless the operation is invoked 
1028
from within the build phase.
1029
 
1030
The utility will wait upto 10 minutes (600 seconds) for a named version to be replicated
1031
from the main archive to a replica.
1032
 
1033
A dpkg_cache need not be present for the replication-wait to be performed.
1034
 
1035
The wait-for replication step is designed to address an issue where the build system
1036
is remotely located from the main archive. If required package versions are not in the
1037
replica then this utility would copy them from the main archive. This can be very very 
1038
slow - much slower than waiting for the replication to complete.
1039
 
5783 dpurdie 1040
=item B<-[no]cache>
1041
 
1042
This option can be used to prevent the named packages from being cached. The default mode
1043
is to cache the named packages.
1044
 
1045
It is intended to be used with the '-wait' option so that replication is complete
1046
but the local caching is not perform. To be used for non-ant builds to prevent local caching
1047
of packages that are not used in a build.
1048
 
227 dpurdie 1049
=item B<-quiet>
1050
 
1051
This option will suppress almost all of the progress messages, except for a single
1052
copy message. It is intended to be used when the program is called from another
1053
script.
1054
 
1055
=item B<-age=nn>
1056
 
1057
This option will delete all package versions that are older than the nn days.
1058
The age of a package is calculated from the timestamp of the descpkg file.
1059
 
361 dpurdie 1060
=item B<-test>
1061
 
1062
This option modifies the operation of the B<-age=nn> option such that it will not
1063
delete old package-versions. It will simply report what would be deleted.
1064
 
227 dpurdie 1065
=back
1066
 
1067
=head1 DESCRIPTION
1068
 
1069
This program simplifies the operation of maintaining a local copy of
1070
used packages from the maintaining dpkg_archive store. The cache should be
1071
stored on your local disk for speed.
1072
 
1073
=head2 Location of the cache
1074
 
4688 dpurdie 1075
The local cache is specified with the EnvVar GBE_DPKG_CACHE
227 dpurdie 1076
 
4688 dpurdie 1077
=head2 Location of the maintaining archive
227 dpurdie 1078
 
4688 dpurdie 1079
The required pacjage version can be found in three archives. These are:
227 dpurdie 1080
 
4688 dpurdie 1081
=over 4
227 dpurdie 1082
 
4688 dpurdie 1083
=item *
227 dpurdie 1084
 
4688 dpurdie 1085
GBE_DPKG_REPLICA. A local image of the main dpkg_archive. This is used when dpkg_archive is synced locally and is expected to be faster than GBE_DPKG
1086
 
1087
=item *
1088
 
1089
GBE_DPKG. The main dpkg_archive
1090
 
1091
=item *
1092
 
1093
GBE_DPKG_STORE. A global package archive. The search repository of last choice.
1094
 
1095
=back
1096
 
227 dpurdie 1097
=head2 Interaction with local_dpkg_archive
1098
 
1099
If a package is located in the users local_dpkg_archive and we are doing a
1100
simple cache update then the package will be deleted from the cache. This is
1101
done to speed use on slow remote links and ensure cache consistency.
1102
 
5743 dpurdie 1103
=head2 Interaction with build system
1104
 
1105
If the cache operating is done withinthe context of the Build System (GBE_ABT is not zero), 
1106
then some parts of the package will not be transferred. This is done to speed up the caching.
1107
 
1108
Directories that will not be cached:
1109
 
1110
=over 4
1111
 
1112
=item * 
1113
 
1114
lcov - A directory that contains code coverage information. This directory normally 
1115
contains a large number of files, none of which are used by the build system. Over long 
1116
links the trasnfer time of the 'lcov' directory can take hours.
1117
 
1118
=back
1119
 
227 dpurdie 1120
=head1 EXAMPLE
1121
 
1122
=head2 jats dpkg_cache -list
1123
 
1124
This will list the current contents of the cache.
1125
 
1126
=head2 jats dpkg_cache -refresh crc/1.0.4.cr
1127
 
1128
This will delete any cached copy of the package crc/1.0.4.cr, if one exists,
1129
and then copy in a new version.
1130
 
1131
=head2 jats dpkg_cache crc
1132
 
1133
This will copy in all versions of the crc package. This may not be desirable.
1134
 
1135
=head2 jats dpkg_cache -update_all
1136
 
1137
This will examine all packages in the cache and refresh those packages that are
1138
out of date.
1139
 
1140
=cut
1141