Subversion Repositories DevTools

Rev

Rev 5818 | Rev 6133 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
227 dpurdie 1
########################################################################
5709 dpurdie 2
# Copyright (c) VIX TECHNOLOGY (AUST) LTD
227 dpurdie 3
#
4
# Module name   : cache_dpkg
5744 dpurdie 5
# Module type   : JATS Utility
6
# Compiler(s)   : Perl
227 dpurdie 7
# Environment(s): jats
8
#
9
# Description:
5743 dpurdie 10
#       Maintain a local cache of dpkg_archive
227 dpurdie 11
#
12
# Notes:
335 dpurdie 13
#       Stopped using the JATS "cp.exe" utility as there was a weird problem under
227 dpurdie 14
#       windows. The package orahops-ssw-install/1.0.1000.ssw could not be installed
335 dpurdie 15
#       correctly. It appears that the 'cp' could not process the subdir pair
227 dpurdie 16
#       "DBMGR/DBMGR". Change the name it was OK.
17
#
18
#       Solution: Avoid system functions
19
#
20
#......................................................................#
21
 
22
use strict;
23
use warnings;
24
use JatsError;
25
use FileUtils;
26
use File::Find;
27
use File::Path;
28
use File::Copy;
29
 
30
use Getopt::Long;
31
use Pod::Usage;                             # required for help support
32
 
5743 dpurdie 33
my $VERSION = "1.5.0";
227 dpurdie 34
 
35
 
36
#
37
#   Options
38
#
39
my $opt_debug   = $ENV{'GBE_DEBUG'};        # Allow global debug
40
my $opt_verbose = $ENV{'GBE_VERBOSE'};      # Allow global verbose
41
my $opt_help = 0;
42
my $opt_clear;
43
my $opt_flush;
44
my $opt_refresh;
45
my $opt_refresh_all;
46
my $opt_list;
47
my $opt_list_short;
48
my $opt_export;
49
my $opt_quiet;
50
my $opt_update_all;
51
my $opt_age;
361 dpurdie 52
my $opt_test;
5744 dpurdie 53
my $opt_wait = 0;
5783 dpurdie 54
my $opt_cache = 1;
227 dpurdie 55
 
56
#
57
#   Global Variables
58
#
4688 dpurdie 59
my $GBE_DPKG            = $ENV{'GBE_DPKG'};                     # The Master repository
60
my $GBE_DPKG_CACHE      = $ENV{'GBE_DPKG_CACHE'} || "";         # Caches
61
my $GBE_DPKG_LOCAL      = $ENV{'GBE_DPKG_LOCAL'} || "";         # Local scratch
62
my $GBE_DPKG_STORE      = $ENV{'GBE_DPKG_STORE'} || "";         # Global Store
63
my $GBE_DPKG_REPLICA    = $ENV{'GBE_DPKG_REPLICA'} || "";       # Site Local Replica
64
my $GBE_BIN             = $ENV{'GBE_BIN'};
227 dpurdie 65
 
4688 dpurdie 66
my $istore;
227 dpurdie 67
my $gstore;
68
my $archive;
69
my $cache;
70
my $parchive;
71
my $local = '';
72
my $local_pkg;
73
my @package_list;
74
my @refresh_list;
5818 dpurdie 75
 
227 dpurdie 76
#
77
#   Globals for recursive copy
78
#
79
my $copyFind_dst;
80
my $copyFind_src;
81
my $copyFind_src_len;
5818 dpurdie 82
my $copyFind_touch;
5862 dpurdie 83
my $copyFind_time;
84
my @copyFindDups;
227 dpurdie 85
 
86
#
87
#   Globals for error recovery
88
#
89
my  $remove_on_error;
5818 dpurdie 90
my  $cacheMarker;
227 dpurdie 91
 
92
#-------------------------------------------------------------------------------
93
# Function        : Mainline Entry Point
94
#
95
# Description     :
96
#
97
# Inputs          :
98
#
99
my $result = GetOptions (
337 dpurdie 100
                "help|h:+"                  => \$opt_help,
101
                "manual:3"                  => \$opt_help,
102
                "verbose:+"                 => \$opt_verbose,           # flag, multiple use allowed
103
                "debug:+"                   => \$opt_debug,             # flag, multiple use allowed
227 dpurdie 104
                "flush"                     => \$opt_flush,             # flag
105
                "clear"                     => \$opt_clear,             # flag
5744 dpurdie 106
                "wait!"                     => \$opt_wait,              # [no]flag
5783 dpurdie 107
                "cache!"                    => \$opt_cache,             # [no]flag
5744 dpurdie 108
                "refresh!"                  => \$opt_refresh,           # [no]flag
227 dpurdie 109
                "refresh_all|refresh-all"   => \$opt_refresh_all,       # flag
110
                "update_all|update-all"     => \$opt_update_all,        # flag
111
                "list"                      => \$opt_list,              # flag
112
                "dir"                       => \$opt_list_short,        # flag
113
                "export"                    => \$opt_export,            # flag
114
                "quiet"                     => \$opt_quiet,             # flag
115
                "age=i"                     => \$opt_age,               # integer
361 dpurdie 116
                "test"                      => \$opt_test,              # flag
227 dpurdie 117
                );
118
 
119
                #
120
                #   UPDATE THE DOCUMENTATION AT THE END OF THIS FILE !!!
121
                #
122
 
123
#
124
#   Process help and manual options
125
#
126
pod2usage(-verbose => 0, -message => "Version: $VERSION")  if ($opt_help == 1  || ! $result);
127
pod2usage(-verbose => 1)  if ($opt_help == 2 );
337 dpurdie 128
pod2usage(-verbose => 2)  if ($opt_help > 2);
227 dpurdie 129
 
130
#
131
#   Configure the error reporting process now that we have the user options
132
#
133
ErrorConfig( 'name'    =>'CACHE',
134
             'verbose' => $opt_verbose,
135
             'debug'   => $opt_debug );
136
 
137
#
138
#   Validate user options
139
#
140
Error( "GBE_BIN not defined in the environment" ) unless ( defined($GBE_BIN) );
141
Error( "GBE_DPKG not defined in the environment" ) unless ( defined($GBE_DPKG) );
142
 
5744 dpurdie 143
$opt_refresh = 1
144
    if ( $opt_refresh_all );
145
 
227 dpurdie 146
#
5744 dpurdie 147
#   Locate the various package stores
148
#   Search order for package should be:
149
#       DPKG_SANDBOX (Not cached)
150
#       DPKG_LOCAL   (To be deprecated)
151
#       DPKG_CACHE
152
#       DPKG_REPLICA
153
#       DPKG         (Writable to build system)
154
#       DPKG_STORE
227 dpurdie 155
#
5744 dpurdie 156
$local = $GBE_DPKG_LOCAL;
157
$cache = $GBE_DPKG_CACHE;
158
$istore = $GBE_DPKG_REPLICA;
159
$archive = $GBE_DPKG;
160
$gstore = $GBE_DPKG_STORE;
161
 
162
Error ("dpkg_archive cache is the main archive")    if ( $cache eq $archive );
163
Error ("dpkg_archive replica is the main archive")  if ( $istore && $istore eq $archive );
164
Error ("main archive is local archive" )            if ( $local && $local eq $archive );
165
Warning  ("dpkg_archive_cache is local_archive" )   if ( $cache && $local && $local eq $cache );
166
 
167
#
168
#   Perform package/version replication wait
169
#   
170
#   Do not need a cache for this to be performed as the operation is used on build servers
171
#   that do not have a cache (most Unix build machines)
172
#   
173
#   Cannot wait if we don't have a REPLICA
174
#   
175
if ($opt_wait && $istore)
227 dpurdie 176
{
5744 dpurdie 177
    Verbose("Wait for package replication");
178
    #
179
    #   Scan the argument list
180
    #   Only wait for package/version. Cannot wait on an entire package
181
    #
182
    foreach (@ARGV)
183
    {
184
        unless (m~.+/.+~)
185
        {
186
            Verbose("Wait Skip $_");
187
            next;
188
        }
227 dpurdie 189
 
5744 dpurdie 190
        if (-d "$istore/$_")
191
        {
192
            Verbose("Wait not required. Version in replica: $_");
193
            next;
194
        }
227 dpurdie 195
 
5744 dpurdie 196
        unless (-d "$archive/$_")
197
        {
198
            Verbose("Wait not required. Version not in store: $_");
199
            next;
200
        }
227 dpurdie 201
 
5744 dpurdie 202
        #
203
        #   Wait for the package to be replicated
204
        #       Wait up to 10 minutes
205
        #       Or the package to appear in the replica
206
        #       Or the package to disappear from the main store
207
        #
208
        Verbose("Waiting for replication of: $_");
209
        my $waitStart = time();
210
        while (1)
211
        {
212
            sleep(30);
213
            my $delta = time() - $waitStart;
214
            if (-d "$istore/$_")
215
            {
216
                Message("PkgReplication. Package replicated ($delta): $_");
217
                last;
218
            }
219
            unless (-d "$archive/$_")
220
            {
221
                Message("PkgReplication. Package deleted from main archive: $_");
222
                last;
223
            }
224
            #
225
            #   Done wait forever
226
            #   If the package has not been replicate we will just have to copy it the slow way
227
            #   
228
            if ( $delta >= 600)
229
            {
230
                Message("PkgReplication. Package not replicated: $_");
231
                last;
232
            }
233
        }
234
    }
235
}
236
 
227 dpurdie 237
#
5783 dpurdie 238
#   Allow replication update without also caching the package
239
#       This is used to prevent ALL replicated packages beinng cached.
240
#       Packages do not need to be cached if they are not used by a build on the current machine
241
#       This will prevent the cache from filling up with unused packages.
242
#
243
unless ($opt_cache)
244
{
245
    Message("Packages not cached");
246
    exit 0;
247
}
248
 
249
#
5744 dpurdie 250
#   No cache - nothing to do
251
#   Generate a status message and exit
227 dpurdie 252
#
5744 dpurdie 253
unless ( $cache )
227 dpurdie 254
{
5744 dpurdie 255
    Warning ( "GBE_DPKG_CACHE not defined in the environment" );
256
    exit 0;
227 dpurdie 257
}
258
 
259
#
260
#   Export the list of cache entries
261
#
262
if ( $opt_export or $opt_refresh_all or $opt_list_short or $opt_update_all )
263
{
264
    opendir (DIR, $cache) || die "Cannot open $cache\n";
265
    my @dir_list = readdir(DIR);
266
    closedir DIR;
267
 
268
    for my $fn ( @dir_list)
269
    {
270
        my $count = 0;
271
        next if ( $fn =~ m/^\./ );
272
        next unless ( -d "$cache/$fn" );
273
 
274
        opendir( DIR, "$cache/$fn" ) || die "Cannot open $cache/$fn\n";
275
        while ( my $fn1 = readdir(DIR) )
276
        {
277
            next if ( $fn1 =~ m/^\./ );
278
            push @package_list, "$fn/$fn1";
279
        }
280
        closedir DIR;
281
    }
282
 
283
    if ( $opt_refresh_all or $opt_update_all )
284
    {
285
        @refresh_list = @package_list
286
    }
287
 
288
    #
289
    #   Simply export a list of packages in the cache
290
    #   Used by other programs
291
    #
292
    if ( $opt_export )
293
    {
294
        print join ' ', @package_list;
295
        exit 0;
296
    }
297
}
298
 
299
#
300
#   Display cache information
301
#   This is done AFTER the export - since the export command does not
302
#   need this header.
303
if ( $opt_verbose || $#ARGV < 0 )
304
{
305
    print  "dpkg_store        : $gstore\n" if ($gstore);
306
    print  "dpkg_archive      : $archive\n";
4688 dpurdie 307
    print  "dpkg_archive image: $istore\n";
227 dpurdie 308
    print  "dpkg_archive cache: $cache\n";
309
    print  "dpkg_archive local: $local\n";
310
    Verbose ("args:               @ARGV");
311
}
312
 
313
#
314
#   List the contents of the cache
315
#
316
if ( $opt_list_short )
317
{
318
    print join "\n", @package_list;
319
}
320
 
321
if ( $opt_list )
322
{
323
    #
324
    #   Process user commands
325
    #   List: A nice pretty display of the cache
326
    #
327
    opendir (DIR, $cache) || die "Cannot open $cache\n";
328
    my @dir_list = readdir(DIR);
329
    closedir DIR;
330
 
331
    #
332
    #   Determine max length of a name so that the display is nicely aligned
333
    #
334
    my $nl = 10;
335
    for my $fn ( @dir_list)
336
    {
337
        my $ns = length($fn);
338
        $nl = $ns if ( $ns > $nl )
339
    }
340
 
341
    #
342
    #   Display information by package
343
    #   A nicely formatted list with line wrapping
344
    #
345
    for my $fn ( @dir_list)
346
    {
347
        my $count = 0;
348
        next if ( $fn =~ m/^\./ );
349
        next unless ( -d "$cache/$fn" );
350
 
351
        opendir( DIR, "$cache/$fn" ) || die "Cannot open $cache/$fn\n";
352
        printf "%-*s :", $nl, $fn;
353
        while ( my $fn1 = readdir(DIR) )
354
        {
355
            next if ( $fn1 =~ m/^\./ );
356
            if ( $count++ >= 4 )
357
            {
358
                $count = 1;
359
                printf "\n%*s  ", $nl, "";
360
            }
361
            printf " %14.14s", $fn1;
362
        }
363
        closedir DIR;
364
        print "\n";
365
    }
366
}
367
 
368
#
369
#   Clear the cache
370
#
371
if ( $opt_clear )
372
{
373
    Warning   ( "Deleting the ENTIRE cache");
374
    rmtree( $cache, $opt_debug );
375
    mkpath( $cache, $opt_debug, 0777);
376
}
377
 
378
#
379
#   Perform cache aging
380
#
381
if ( $opt_age )
382
{
383
    Error ("Age parameter cannot be negative") if ( $opt_age < 0 );
384
    age_the_cache();
385
}
386
 
387
#
388
#   Process the command line arguments
389
#   These MUST be of the form
390
#       packagename
391
#       packagename/version
392
#
393
for (@ARGV, @refresh_list)
394
{
395
    $remove_on_error = undef;
5818 dpurdie 396
    $cacheMarker = undef;
397
 
227 dpurdie 398
    if ( $opt_flush )
399
    {
400
        unless ( -d "$cache/$_" )
401
        {
402
            Warning ("Package not in cache: $_")
403
                unless ( $opt_quiet );
404
        }
405
        else
406
        {
407
            rmtree( "$cache/$_", $opt_debug );
408
            print "Package removed: $_\n"
409
                unless ( $opt_quiet );
410
        }
411
    }
412
    else
413
    {
414
        #
415
        #   Speed up - for remote access
416
        #   If the package is found in the local archive and we aren't doing
417
        #   anything fancy then don't access the remote archive
418
        #
419
        my $local_pkg = ( -d "$local/$_" || -f "$local/$_.lnk" );
420
        if ( $local_pkg )
421
        {
422
            if ( !$opt_refresh )
423
            {
424
                print "Cache SkipLocal: $_ Local copy found\n";
425
 
426
                #
427
                # Delete the cache copy
428
                # The user is playing with local copy
429
                #
430
                if ( -d "$cache/$_" )
431
                {
432
                    print  "Cache SkipLocalDelete: $_ -> $cache\n";
433
                    Verbose ( "Remove: $_" );
434
                    rmtree( "$cache/$_", $opt_debug );
435
                    Verbose ( "Remove Complete: $_" );
436
                }
437
                next;
438
            }
439
        }
440
 
441
        #
442
        #   Locate package.
4688 dpurdie 443
        #   It may be in GBE_DPKG_REPLICA, GBE_DPKG or GBE_DPKG_STORE
227 dpurdie 444
        #
4688 dpurdie 445
        my $pkg_found;
446
        foreach my $store ( $istore, $archive, $gstore)
227 dpurdie 447
        {
4688 dpurdie 448
            $parchive = "$store/$_";
449
            if ( -d $parchive )
227 dpurdie 450
            {
4688 dpurdie 451
                $pkg_found = 1;
452
                last;
227 dpurdie 453
            }
454
        }
4688 dpurdie 455
 
456
        unless ($pkg_found )
457
        {
458
            Warning ("Package not in archive: $_")
459
                unless ( $opt_quiet );
460
            next;
461
        }
227 dpurdie 462
 
463
        ########################################################################
464
        #   We have a package to process
465
        #
5818 dpurdie 466
        my $dir_found = 0;
227 dpurdie 467
        my $force_update = 0;
468
        my $opr = "Update";
469
 
5818 dpurdie 470
        #
471
        #   Generate the 'name' of a marker file used to indicate that the version is being updated
472
        #   This is outside the target directory
473
        #   
474
        $cacheMarker = "$_/built.cache";
475
        $cacheMarker =~ tr~/~_~s;
476
        $cacheMarker =~ tr~_~_~s;
477
        $cacheMarker =  $cache . '/' . $cacheMarker;
478
        Verbose2("cacheMarker: $cacheMarker");
227 dpurdie 479
 
480
        #
481
        #   Setup error recovery
335 dpurdie 482
        #       1) Tag the directory to be deleted on error
227 dpurdie 483
        #
484
        $remove_on_error = "$cache/$_";
485
        ErrorConfig( 'on_exit' => \&error_recovery );
486
 
487
        #
335 dpurdie 488
        #   Not a forced refresh. Ensure that the cached copy is
227 dpurdie 489
        #   up to date. Examine descpkg
490
        #
5818 dpurdie 491
        $dir_found = waitForComplete($_,$cache);
492
        if ( $dir_found && !$opt_refresh )
227 dpurdie 493
        {
5818 dpurdie 494
            if ( FileIsNewer( "$parchive/descpkg", "$cache/$_/descpkg" ) )
227 dpurdie 495
            {
496
                $force_update = 1;
497
                $opr = "OutOfDate";
5818 dpurdie 498
                TouchFile ( $cacheMarker, "Marks the cache copy as incomplete");
227 dpurdie 499
                Verbose ("Cache out-of-date: $_");
500
            }
501
        }
502
 
503
        #
504
        #   If we need to refresh the cache copy - delete it first
505
        #
506
        if ( ($opt_refresh || $force_update) && $dir_found )
507
        {
508
            print  "Cache $opr: $_ -> $cache\n";
509
            Verbose ( "Remove: $_" );
510
            rmtree( "$cache/$_", $opt_debug );
511
            Verbose ( "Remove Complete: $_" );
229 dpurdie 512
 
513
            #
514
            #   Force transfer, but without the status message
515
            #
227 dpurdie 516
            $dir_found = 0;
229 dpurdie 517
            $opr = '';
227 dpurdie 518
        }
519
 
520
        #
5818 dpurdie 521
        #   If the directory exists, then we need to avoid a race condition
522
        #   where multiple instances are updating the same directory
523
        #       Need to aviod hanging forever if the primary updator does not
524
        #       complete the task
227 dpurdie 525
        #   If its not in the cache then copy it in
526
        #
5818 dpurdie 527
        $dir_found = waitForComplete($_,$cache);
227 dpurdie 528
        unless ( $dir_found )
529
        {
229 dpurdie 530
            print "Cache $opr: $_ -> $cache\n" if $opr;
227 dpurdie 531
            mkpath( "$cache/$_", $opt_debug, 0777);
5818 dpurdie 532
            TouchFile ( $cacheMarker, "Marks the cache copy as incomplete");
227 dpurdie 533
            Verbose ( "Copy in: $_" );
534
            $copyFind_dst = "$cache/$_";
535
            $copyFind_src = $parchive;
536
            $copyFind_src_len = length( $copyFind_src );
5818 dpurdie 537
            $copyFind_touch = $cacheMarker;
538
            $copyFind_time = 0;
5862 dpurdie 539
            undef @copyFindDups;
227 dpurdie 540
            File::Find::find( \&copyFind, $parchive );
5862 dpurdie 541
            if (@copyFindDups)
542
            {
543
                Warning("The following items where not transferred as they already existed",
544
                        "This may be due to symlinks or (Windows) case insensitive filename", @copyFindDups);
545
            }
5818 dpurdie 546
            rmtree( $cacheMarker, $opt_debug );  # Works on files too !!
227 dpurdie 547
        }
548
        else
549
        {
550
            $opr = "Skip";
551
            print "Cache $opr: $_ -> $cache\n";
552
        }
553
    }
554
}
555
 
556
#-------------------------------------------------------------------------------
5818 dpurdie 557
# Function        : waitForComplete 
558
#
559
# Description     : Wait for a package-version to complete
560
#                   At the end of this operation the directory will be 
561
#                   completly transferred, or it will be deleted
562
#                   
563
#                   If the directory exists, then we need to avoid a race condition
564
#                   where multiple instances are updating the same directory
565
#                   Need to avoid hanging forever if the primary updator does not
566
#                   complete the task
567
#
568
# Inputs          : $dir    - Subdir directory to monitor
569
#                   $cache  - Target cache
570
#
571
# Returns         : 0   - $dir does not exist
572
#                   1   - $dir does exist 
573
#
574
sub waitForComplete
575
{
576
    my ($dir, $cache) = @_;
577
    my $opr;
578
    my $tgtDir = "$cache/$dir";
579
    #
580
    #   Directory not preset
581
    #       All done - dir does not exist 
582
    #
583
    if (! -d $tgtDir)
584
    {
585
        return 0;
586
    }
587
 
588
    # If the package is not being updated, then we are done if the 
589
    # package being updated marker is NOT present
590
    #     
591
    if (  -d $tgtDir && ! -f $cacheMarker)
592
    {
593
        return 98;
594
    }
595
 
596
    #
597
    #   The package-version is being updated by another instance
598
    #   Hang around waiting for the update to complete or terminate
599
    #       Wait 10 minutes max
600
    #
601
    my  $waitStart = time();
602
    $opr = "Wait";
603
    print "Cache $opr: $dir -> $cache\n";
604
    while (1)
605
    {
606
        if ( ! -d $tgtDir )
607
        {
608
            # Directory has gone away
609
            #   Must have been a bad transfer by another instance
610
            #   Start the processing again
611
            $opr = "Retry";
612
            print "Cache $opr: $dir -> $cache\n";
613
            return 0;
614
        }
615
 
616
        if ( ! -f $cacheMarker )
617
        {
618
            #
619
            #   Update must have completed - marker file has been removed
620
            #
621
            $opr = "Found";
622
            print "Cache $opr: $_ -> $cache\n";
623
            return 97;
624
        }
625
 
626
        #
627
        #   Determine the 'age' of the marker file
628
        #       It will be updated after each file in the package has been copied
629
        #       If it is too old, then we must assume that the other instance died
630
        #   Complication. Windows does not correctly report the 'modified' time of a file
631
        #                 It gets messed up by the file system. Can't use the modified info
632
        #                 from Perls stat function to determine the absolute modified time of the
633
        #                 file.
634
        #                 
635
        #       Solution: Create a temp file and determine the age relative to that file
636
        #       
637
 
638
        my $cacheMarkerTmp = $cacheMarker . ".tmp";
639
        TouchFile($cacheMarkerTmp, "Relative time stamp");
640
        my $now   = (stat ($cacheMarkerTmp))[9];
641
        my $mtime = (stat ($cacheMarker))[9];
642
        rmtree( $cacheMarkerTmp, $opt_debug );
643
 
644
        #
645
        #   If the marker file is older than 5 minutes then we can consider the package as
646
        #   a dud. The marker file will be updated be every file write into the cache. Its
647
        #   an indication of activity.
648
        #   
649
        #   Assume that no file is going to take more then 5 minutes to copy
650
        #   Lets see how this goes.
651
        #   
652
        my $age = $now - $mtime;
653
        if ( $age > 5 * 60)
654
        {
655
            #
656
            #   Update taking too long
657
            #       Delete the package-version
658
            #       Try again
659
            #
660
            $opr = "Remove Bad";
661
            print "Cache $opr: $_ -> $cache\n";
662
            rmtree( "$cache/$_", $opt_debug );
663
 
664
            return 0;
665
        }
666
 
667
        #
668
        #   Wait a short while
669
        #
670
        Verbose("Waiting: " . (time() - $waitStart));
671
        sleep(5);
672
    }
673
}
674
 
675
 
676
#-------------------------------------------------------------------------------
227 dpurdie 677
# Function        : copyFind
678
#
679
# Description     : File:Find:find callback function to transfer files
680
#
681
# Inputs          : None
5862 dpurdie 682
#                   Global: $copyFind_dst       - Target directory
683
#                   Global: $copyFind_src       - Source directory
684
#                   Global: $copyFind_src_len   - Length of Source dir
685
#                   Global: $copyFind_touch     - File to touch after each operation
686
#                   Global: $copyFind_time      - Time of last touch
227 dpurdie 687
#
5862 dpurdie 688
# Returns         : Global: @copyFindDups       - Array of files not copied due to previous existence
227 dpurdie 689
#
690
 
691
sub copyFind
692
{
693
    my $item = $File::Find::name;
5818 dpurdie 694
#Debug0("Remove sleep in copy");
695
#sleep(1);
227 dpurdie 696
 
697
    #
698
    #   Calculate the target directory name
699
    #
5743 dpurdie 700
    my $tgt_path = substr($item, $copyFind_src_len );
701
    my $target = $copyFind_dst . $tgt_path;
702
 
703
    # Do not cache some parts of a package when being used by the build system
704
    #   /lcov ...
705
    #
706
    if ( $ENV{GBE_ABT})
707
    {
708
        if ($tgt_path =~ m~^/lcov(/|$)~)
709
        {
710
            Verbose("Prune directory: $tgt_path");
711
            $File::Find::prune = 1;
712
            return;
713
        }
714
    }
715
 
227 dpurdie 716
    if ( -d $item )
717
    {
718
        #
719
        #   Directories are handled differently
720
        #       - Directories are created with nice permissions
721
        #
722
        if ( ! -d $target )
723
        {
724
            mkpath( $target, $opt_debug, 0755);
725
        }
726
    }
727
    else
728
    {
729
        #
730
        #   File copy
731
        #
732
        #
733
        #   Copy file to destination
734
        #   If the file is a link, then duplicate the link contents
735
        #   Use: Unix libraries are created as two files:
736
        #        lib.xxxx.so -> libxxxx.so.vv.vv.vv
737
        #
738
        if ( -l $item )
739
        {
740
            Debug("Clone Link: $target");
741
            my $link = readlink $item;
742
            Debug( "Link: $item, $link");
743
            symlink ($link, $target );
744
            unless ( $link && -l $target )
745
            {
746
                Error("Failed to copy link [$item] to [$target]: $!");
747
            }
748
        }
5862 dpurdie 749
        elsif (-f $target)
750
        {
751
            # File already exists
752
            #   Most likely Windows filename clash. Windows files are case-insensitive
753
            push @copyFindDups, $tgt_path
754
        }
227 dpurdie 755
        elsif (File::Copy::copy($item, $target))
756
        {
757
            Debug("Copying File: $target");
758
 
759
            #   Make the file ReadOnly
760
            my $perm = (stat $item)[2] & 07777;
761
            CORE::chmod $perm & 0555, $target;
762
        }
763
        else
764
        {
765
            Error("Failed to copy file [$item] to [$target]: $!");
766
        }
767
    }
5818 dpurdie 768
 
769
    #
770
    #   Touch this file to indicate that the copy operation is still in progress
771
    #   Assists in early detection of partial copies
772
    #   
773
    #   Only touch once a minute to prevent hammering the file system
774
    #
775
    my $age = time() - $copyFind_time;
776
    if ($age > 60)
777
    {
778
        $copyFind_time = time();
779
        TouchFile ( $copyFind_touch, "Marks the cache copy as incomplete");
780
    }
227 dpurdie 781
}
782
 
783
#-------------------------------------------------------------------------------
784
# Function        : error_recovery
785
#
786
# Description     : Error recovery routine
787
#                   Delete the cached entry
788
#
789
# Inputs          :  Globals
790
#
791
# Returns         : None
792
#
793
sub error_recovery
794
{
795
    if ( $remove_on_error )
796
    {
345 dpurdie 797
        ReportError ("Error cleanup. Delete cache entry: $remove_on_error");
227 dpurdie 798
        rmtree( $remove_on_error, $opt_debug );
799
        $remove_on_error = undef;
800
    }
5818 dpurdie 801
 
802
    if ($cacheMarker)
803
    {
804
        ReportError ("Error cleanup. Delete cache marker: $cacheMarker");
805
        rmtree( $cacheMarker, $opt_debug );
806
        $cacheMarker = undef;
807
    }
227 dpurdie 808
}
809
 
810
 
811
#-------------------------------------------------------------------------------
812
# Function        : age_the_cache
813
#
814
# Description     : Age cache entries
815
#                   Determine the age by:
816
#                       Use used.cache file if present
817
#                       Use descpkg file
818
#                       Not a proper entry - delete
819
#
820
# Inputs          : opt_age         - Delete packages older than XX days
821
#
822
# Returns         : Nothing
823
#
824
sub age_the_cache
825
{
826
    my $now = time;
827
 
828
    opendir (DIR, $cache) || die "Cannot open $cache\n";
829
    my @dir_list = readdir(DIR);
830
    closedir DIR;
831
 
832
    for my $fn ( @dir_list)
833
    {
834
        my $keep_dir = 0;
835
        next if ( $fn =~ m/^\./ );
836
        next unless ( -d "$cache/$fn" );
361 dpurdie 837
        next if ( $fn eq "core_devl" );
227 dpurdie 838
 
839
        opendir( DIR, "$cache/$fn" ) || die "Cannot open $cache/$fn\n";
840
        while ( my $fn1 = readdir(DIR) )
841
        {
842
            next if ( $fn1 =~ m/^\./ );
843
            my $dir = "$cache/$fn/$fn1";
844
            my $file = "$dir/used.cache" ;
845
            $file = "$dir/descpkg" unless ( -f $file );
846
 
847
            if ( ! -f $file )
848
            {
849
                #
850
                #   No descpkg file
851
                #   This is a badly formed entry - so delete it
852
                #
361 dpurdie 853
                if ( $opt_test )
854
                {
855
                    Message("Would Purge: $fn/$fn1" );
856
                    $keep_dir = 1;
857
                }
858
                else
859
                {
860
                    Message("Purging: $fn/$fn1" );
861
                    rmtree( $dir, $opt_debug );
862
                }
227 dpurdie 863
            }
864
            else
865
            {
866
                #
867
                #   used.cache or descpkg file found
868
                #   How old is it
869
                #
870
                my $timestamp = (stat($file))[9] || 0;
871
                my $age = int( ($now - $timestamp) / (60 * 60 * 24));
872
 
873
                if ( $age > $opt_age )
874
                {
361 dpurdie 875
                    if ( $opt_test )
876
                    {
877
                        Message("Could Age: $fn/$fn1, $age" );
878
                        $keep_dir = 1;
879
                    } else {
880
                        Message("Aging: $fn/$fn1, $age" );
881
                        rmtree( $dir, $opt_debug );
882
                    }
227 dpurdie 883
                }
884
                else
885
                {
886
                    Verbose("Age of: $fn/$fn1, $age" );
887
                    $keep_dir = 1;
888
                }
889
            }
890
        }
891
        closedir DIR;
892
 
893
        #
894
        #   Delete the entire directory if is is empty
895
        #
896
        unless ( $keep_dir )
897
        {
898
            Message("Remove Empty Dir: $cache/$fn"  );
899
            rmtree( "$cache/$fn", $opt_debug );
900
        }
901
    }
902
}
903
 
904
#-------------------------------------------------------------------------------
905
#   Documentation
906
#
907
 
908
=pod
909
 
361 dpurdie 910
=for htmltoc    SYSUTIL::
911
 
227 dpurdie 912
=head1 NAME
913
 
914
cache_dpkg - Maintain a local cache of packages
915
 
916
=head1 SYNOPSIS
917
 
918
 jats cache_dpkg.pl [options] package/version ...
919
 
920
 Options:
921
    -help              - brief help message
922
    -help -help        - Detailed help message
923
    -man               - Full documentation
924
    -clear             - Delete the entire cache
925
    -flush             - Flush all named packages
926
    -[no]refresh       - Refresh package in cache
927
    -list              - List cache contents with nice format
928
    -dir               - List cache contents
929
    -export            - Generate a list of cached packages
930
    -refresh_all       - Refresh all packages within the cache
931
    -update_all        - Update all packages within the cache as required
5744 dpurdie 932
    -[no]wait          - Wait for package replication
5783 dpurdie 933
    -[no]cache         - Cache packages [default]
227 dpurdie 934
    -quiet             - Suppress warnings
361 dpurdie 935
    -age=nn            - Remove all packages older than nn days
936
    -test              - Use with -age to report ages
227 dpurdie 937
 
938
=head1 OPTIONS
939
 
940
=over 8
941
 
942
=item B<-help>
943
 
944
Print a brief help message and exits.
945
 
946
=item B<-help -help>
947
 
948
Print a detailed help message with an explanation for each option.
949
 
950
=item B<-man>
951
 
952
Prints the manual page and exits.
953
 
954
=item B<-clear>
955
 
956
Delete the B<entire> contents of the dpkg_archive cache. This will occur before
957
any new packages are copied into the cache.
958
 
959
=item B<-flush>
960
 
961
If set then the utility will delete the named packages from the cache. All named
962
packaged will be deleted. This option affects all named packages. It is not
963
possible to flush and copy packages with the same command.
964
 
965
=item B<-[no]refresh>
966
 
967
If the B<refresh> option has been specified then packages will be deleted, from
968
the cache and then a new copy will be copied in. If not specified then no copy
969
will occur if the package is present in the cache.
970
 
971
=item B<-list>
972
 
973
Display a list of all packages in the cache. A formatted display is generated.
974
 
975
This will be done before any packages are transferred.
976
 
977
=item B<-dir>
978
 
979
Display a list of all packages in the cache. This is a raw directory like
980
listing of the cache.
981
 
982
This will be done before any packages are transferred.
983
 
984
=item B<-export>
985
 
986
Generate a space separated list of cached packages as a single line. This is
987
intended to allow a list be exported for later import.
988
 
989
=item B<-refresh_all>
990
 
991
This option will force the program to refresh all the packages in the cache.
992
This forces a B<-refresh> and may be combined with other packages specified
993
on the command line.
994
 
995
=item B<-update_all>
996
 
997
This option will force the program to examine all packages within the cache and
998
refresh packages that are out of date. This option may be combined with other
999
packages specified on the command line.
1000
 
1001
A package is deemed to be out-of-date if the modification time of the package's
335 dpurdie 1002
descpkg file in the cache is older than the one in the archive.
227 dpurdie 1003
 
5744 dpurdie 1004
=item B<-[no]wait>
1005
 
1006
This option will cause the utility to wait for specified package versions to be replicated
1007
into a package replica. The dafult mode is to not-wait, unless the operation is invoked 
1008
from within the build phase.
1009
 
1010
The utility will wait upto 10 minutes (600 seconds) for a named version to be replicated
1011
from the main archive to a replica.
1012
 
1013
A dpkg_cache need not be present for the replication-wait to be performed.
1014
 
1015
The wait-for replication step is designed to address an issue where the build system
1016
is remotely located from the main archive. If required package versions are not in the
1017
replica then this utility would copy them from the main archive. This can be very very 
1018
slow - much slower than waiting for the replication to complete.
1019
 
5783 dpurdie 1020
=item B<-[no]cache>
1021
 
1022
This option can be used to prevent the named packages from being cached. The default mode
1023
is to cache the named packages.
1024
 
1025
It is intended to be used with the '-wait' option so that replication is complete
1026
but the local caching is not perform. To be used for non-ant builds to prevent local caching
1027
of packages that are not used in a build.
1028
 
227 dpurdie 1029
=item B<-quiet>
1030
 
1031
This option will suppress almost all of the progress messages, except for a single
1032
copy message. It is intended to be used when the program is called from another
1033
script.
1034
 
1035
=item B<-age=nn>
1036
 
1037
This option will delete all package versions that are older than the nn days.
1038
The age of a package is calculated from the timestamp of the descpkg file.
1039
 
361 dpurdie 1040
=item B<-test>
1041
 
1042
This option modifies the operation of the B<-age=nn> option such that it will not
1043
delete old package-versions. It will simply report what would be deleted.
1044
 
227 dpurdie 1045
=back
1046
 
1047
=head1 DESCRIPTION
1048
 
1049
This program simplifies the operation of maintaining a local copy of
1050
used packages from the maintaining dpkg_archive store. The cache should be
1051
stored on your local disk for speed.
1052
 
1053
=head2 Location of the cache
1054
 
4688 dpurdie 1055
The local cache is specified with the EnvVar GBE_DPKG_CACHE
227 dpurdie 1056
 
4688 dpurdie 1057
=head2 Location of the maintaining archive
227 dpurdie 1058
 
4688 dpurdie 1059
The required pacjage version can be found in three archives. These are:
227 dpurdie 1060
 
4688 dpurdie 1061
=over 4
227 dpurdie 1062
 
4688 dpurdie 1063
=item *
227 dpurdie 1064
 
4688 dpurdie 1065
GBE_DPKG_REPLICA. A local image of the main dpkg_archive. This is used when dpkg_archive is synced locally and is expected to be faster than GBE_DPKG
1066
 
1067
=item *
1068
 
1069
GBE_DPKG. The main dpkg_archive
1070
 
1071
=item *
1072
 
1073
GBE_DPKG_STORE. A global package archive. The search repository of last choice.
1074
 
1075
=back
1076
 
227 dpurdie 1077
=head2 Interaction with local_dpkg_archive
1078
 
1079
If a package is located in the users local_dpkg_archive and we are doing a
1080
simple cache update then the package will be deleted from the cache. This is
1081
done to speed use on slow remote links and ensure cache consistency.
1082
 
5743 dpurdie 1083
=head2 Interaction with build system
1084
 
1085
If the cache operating is done withinthe context of the Build System (GBE_ABT is not zero), 
1086
then some parts of the package will not be transferred. This is done to speed up the caching.
1087
 
1088
Directories that will not be cached:
1089
 
1090
=over 4
1091
 
1092
=item * 
1093
 
1094
lcov - A directory that contains code coverage information. This directory normally 
1095
contains a large number of files, none of which are used by the build system. Over long 
1096
links the trasnfer time of the 'lcov' directory can take hours.
1097
 
1098
=back
1099
 
227 dpurdie 1100
=head1 EXAMPLE
1101
 
1102
=head2 jats dpkg_cache -list
1103
 
1104
This will list the current contents of the cache.
1105
 
1106
=head2 jats dpkg_cache -refresh crc/1.0.4.cr
1107
 
1108
This will delete any cached copy of the package crc/1.0.4.cr, if one exists,
1109
and then copy in a new version.
1110
 
1111
=head2 jats dpkg_cache crc
1112
 
1113
This will copy in all versions of the crc package. This may not be desirable.
1114
 
1115
=head2 jats dpkg_cache -update_all
1116
 
1117
This will examine all packages in the cache and refresh those packages that are
1118
out of date.
1119
 
1120
=cut
1121