Subversion Repositories DevTools

Rev

Rev 5209 | Rev 5212 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
5209 dpurdie 1
########################################################################
2
# Copyright (c) VIX TECHNOLOGY (AUST) LTD
3
#
4
# Module name   : jats_lxr.pl
5
# Module type   : JATS Utility
6
# Compiler(s)   : Perl
7
# Environment(s): jats
8
#
9
# Description   : Tiools to maintain LXR information
10
#
11
# Usage         : See POD at the end of this file
12
#
13
#......................................................................#
14
 
15
require 5.008_002;
16
use strict;
17
use warnings;
18
 
19
use Pod::Usage;
20
use Getopt::Long;
21
use File::Path;
22
use FindBin;
23
use Socket;
24
 
25
use JatsError;
26
use JatsRmApi;
27
use DBI;
28
use JatsSystem;
29
use FileUtils;
30
 
31
#
32
#   Config Options
33
#
34
my $VERSION = "1.0.0";                      # Update this
35
my $opt_help = 0;
36
my $opt_verbose = $ENV{'GBE_VERBOSE'};      # Allow global verbose
37
my $opt_createVersions = 1;
38
my $opt_index = 1;
39
my $opt_purge = 1;
40
my $opt_extract = 1;
41
my $opt_logfile = 1;
5211 dpurdie 42
my $opt_config;
5209 dpurdie 43
 
44
 
45
#
46
#   Globals
47
#
48
my $scriptDir;
49
my $RM_DB;
50
our $config;
51
my @addressList;
52
my %ReleaseData;
53
my %Packages;
54
my $PackageStore;
55
my $ReleaseStore;
56
my $stamptime = time;
57
 
58
#-------------------------------------------------------------------------------
59
# Function        : Main
60
#
61
# Description     : Main entry point
62
#                   Parse user options
63
#
64
# Inputs          :
65
#
66
# Returns         :
67
#
68
 
69
my $result = GetOptions (
70
                "help:+"            => \$opt_help,              # flag, multiple use allowed
71
                "manual:3"          => \$opt_help,              # flag, multiple use allowed
72
                "verbose:+"         => \$opt_verbose,           # flag
73
                "createVersions!"   => \$opt_createVersions,
74
                "purge!"            => \$opt_purge,
75
                "index!"            => \$opt_index,
76
                "extract!"          => \$opt_extract,
5211 dpurdie 77
                "logfile!"          => \$opt_logfile,
78
                "config:s"          => \$opt_config,
5209 dpurdie 79
                );
80
 
81
#
82
#   Process help and manual options
83
#
84
pod2usage(-verbose => 0, -message => "Version: $VERSION")  if ($opt_help == 1  || ! $result);
85
pod2usage(-verbose => 1)  if ($opt_help == 2 );
86
pod2usage(-verbose => 2)  if ($opt_help > 2);
87
 
88
#
89
#   Capture all logging to a file
90
#   Could be done by the jats wrapper, but this way we can control its location
91
#
92
ErrorConfig( 'name'    => 'LXR',
93
             'verbose' => $opt_verbose );
94
 
95
#
96
#   Sanity test
97
#
98
 
99
$scriptDir = $FindBin::Bin;
100
InitFileUtils();
101
 
102
#
103
#   Read the tool configuration
104
#
105
readConfig();
106
Error ("No LXR Data directory defined", $config->{lxrFiles}) unless( exists $config->{lxrFiles});
107
Error ("No LXR Data directory found", $config->{lxrFiles}) unless( -d $config->{lxrFiles});
108
Error ("No LXR Install directory defined",$config->{lxr}) unless( exists $config->{lxr});
109
Error ("No LXR Install directory found",$config->{lxr}) unless( -d $config->{lxr});
110
 
111
#
112
#   Start logging
113
#
114
startLogFile();
115
Message ("Start of LXR. " . localtime(time) ) if $opt_logfile;
116
Message("ScriptDir: $scriptDir");
117
 
118
#
119
#   Check required paths exist
120
#
121
$PackageStore = catdir($config->{lxrFiles}, 'Packages');
122
$ReleaseStore = catdir($config->{lxrFiles}, 'Releases');
123
mkpath($PackageStore);
124
mkpath($ReleaseStore);
125
Error ("Package Store not found ",$PackageStore) unless( -d $PackageStore);
126
Error ("Release Store not found ",$ReleaseStore) unless( -d $ReleaseStore);
127
 
128
#
5211 dpurdie 129
#   Default config
130
#
131
$config->{'releaseAge'} = 0 unless exists $config->{'releaseAge'};
132
$config->{'packageAge'} = 0 unless exists $config->{'packageAge'};
133
 
134
#
5209 dpurdie 135
#   Determine the various names for this machine
136
#   Include all the IP addresses too
137
#
138
my ($canonname, $aliases, $addrtype, $length, @addrs) = gethostbyname($ENV{GBE_HOSTNAME});
139
push @addressList, $canonname ;
140
push (@addressList, $aliases) if ($aliases) ;
141
push (@addressList, $ENV{GBE_HOSTNAME});
142
foreach ( @addrs) {
143
    push @addressList, inet_ntoa($_); 
144
}
145
Message("IpAddress: @addressList");
146
 
147
#
148
#   Perform the hard work
149
#
150
getReleaseData();
151
determineDataToExtract();
152
createReleaseViews() if $opt_createVersions;
153
cleanReleaseViews();
154
rebuildLxrConfig();
155
buildIndexes() if $opt_index;
156
cleanPackageStore() if $opt_purge;
157
cleanupLogFiles() if $opt_purge;
158
 
159
#
160
#   All done
161
#
162
Message ("End of LXR. " . localtime(time) ) if $opt_logfile;
163
exit 0;
164
 
165
#-------------------------------------------------------------------------------
166
# Function        : determineDataToExtract 
167
#
168
# Description     : Determine the packages that need to be extracted
169
#                   Don't extract them if they already exist
170
#
171
# Inputs          : 
172
#
173
# Returns         : 
174
#
175
sub determineDataToExtract
176
{
177
    foreach my $pvid ( keys{%Packages})
178
    {
179
        my $entry = $Packages{$pvid};
180
        #DebugDumpData("Entry", \$entry);
181
        my $fullName = join('_', $entry->{name}, $entry->{ver});
182
        my $fullPath = catfile($PackageStore , $fullName);
183
        next unless defined $entry->{vcs};
184
        next if ( $entry->{vcs} ) =~ m~^UC::~;
5211 dpurdie 185
        unless (-d $fullPath ) 
186
        {
5209 dpurdie 187
            Message("Need to extract: $entry->{name}, $entry->{ver}");
188
            if ($opt_extract)
189
            {
190
                Verbose0("Extracting into: $fullPath");
5211 dpurdie 191
                if ($entry->{vcs} ~= m~/MASS_Dev_Crypto/~)
192
                {
193
                    print "$fullName : SUPPRESSED\n";
194
                }
195
                else
196
                {
197
                    my $rv = JatsCmd ('jats_vcsrelease', '-devmode=escrow', '-extractfiles', "-view=$fullName", "-label=$entry->{vcs}", "-root=$PackageStore", "-noprefix");
198
                    print "$fullName : SUCCESS\n" unless $rv;
199
                    print "$fullName : ERROR\n" if $rv;
200
                    $entry->{bad} = 1;
201
                }
5209 dpurdie 202
            }
203
        }
5211 dpurdie 204
        else
205
        {
206
            #   Package already extracted
207
            #   Ensure that it does not get aged out
208
            deleteAgeMarker($fullPath);
209
        }
5209 dpurdie 210
    }
211
}
212
 
213
#-------------------------------------------------------------------------------
214
# Function        : createReleaseViews 
215
#
216
# Description     : Create a new view for each Release
217
#                   Each view is basically a bunch of symlinks
218
#
219
# Inputs          : 
220
#
221
# Returns         : 
222
#
223
sub createReleaseViews
224
{
225
    #
226
    #   Create a name for the releases - based on a date-time
227
    #   Will be unqiue(ish)
228
    #
229
    my $dateTag = localtime($stamptime);
230
 
231
    foreach my $rtagid (keys \%ReleaseData)
232
    {
233
        my $entry = $ReleaseData{$rtagid};
234
        my $latestView = getLatestVersion(catdir($ReleaseStore, $rtagid));
235
        my  $needNewView = 1;
236
 
237
        #
238
        #   Check to see if we really need to create a new view
239
        #   If the LATEST view contins all the package-versions that we need then
240
        #   don't create a new one.
241
        #
242
        #   Scan each entry within the Last View
243
        #
244
        if ($latestView)
245
        {
246
            my %pkgsUsed;
247
            opendir (my $ldir, $latestView) || Warning ("Cannot open directory: $latestView", $!);
248
            while (my $ldirEntry = readdir($ldir))
249
            {
250
                #   Skip hidden files
251
                next if ($ldirEntry =~ m~^\.~);
252
                my $dirName = catdir($latestView, $ldirEntry );
253
 
254
                #
255
                #   Process each entry within the Version
256
                #
257
                my $pkgName = $ldirEntry;
258
                if (-l $dirName)
259
                {
260
                    $pkgName = readlink($dirName);
261
                    $pkgName =~ s~.*/~~;
262
                }
263
                $pkgsUsed{$pkgName} = 2;
264
            }
265
            close ($ldir);
266
 
267
            #
268
            #   Compare Package-Versions against those we need
269
            # 
270
            foreach my $pvid (keys $entry->{data})
271
            {
272
                my $entry = $Packages{$pvid};
273
                my $fullName = join('_', $entry->{name}, $entry->{ver});
274
                $pkgsUsed{$fullName}++;
275
            }
276
 
277
            #
278
            #   Scan the pkgUsed
279
            #   A value of 1 indicates that it is used only in the New Version
280
            #   A value of 2 indicates that it is only used on the Last Version
281
            #   A value of 3 indicates that its used in both
282
            #   Detect those that are not a 3
283
            #
284
            $needNewView = 0;
285
            foreach ( keys %pkgsUsed)
286
            {
287
                if ($pkgsUsed{$_} != 3)
288
                {
289
                    $needNewView = 1;
290
                    last;
291
                }
292
            }
293
            #DebugDumpData("pkgsUsed",\%pkgsUsed);
294
        }
295
 
296
        unless ($needNewView)
297
        {
298
            Message("No Changes to LXR View: $rtagid");
299
        }
300
        else
301
        {
302
            Message("Creating LXR View: $rtagid, $dateTag");
303
            my $releaseDir = catdir($ReleaseStore, $rtagid, $dateTag);
304
            mkpath($releaseDir);
305
            if (-d $releaseDir)
306
            {
307
                #
308
                #   Insert metadata
309
                #   Used to allow ordering and aging of the releases
310
                #    
311
                my $metaDataName = catfile( $releaseDir, '.jatslxr');
312
                open (my $md, '>' , $metaDataName) || Error ("Cannot create $metaDataName, $!");
313
                print $md $stamptime;
314
                close $md;
315
 
316
                #
317
                #   Populate with symlinks to the actual package-versions
318
                #
319
                foreach my $pvid (keys $entry->{data})
320
                {
321
                    my $entry = $Packages{$pvid};
322
                    my $alias = join('', $entry->{name}, $entry->{ext});
323
                    my $fullName = join('_', $entry->{name}, $entry->{ver});
324
                    my $PackageStore = catdir($PackageStore , $fullName);
325
                    my $releaseDir = catdir($releaseDir, $alias );
326
                    next if -l $releaseDir;
327
 
328
                    Verbose("Symlink $PackageStore, $releaseDir");
329
                    my $rv = symlink ($PackageStore, $releaseDir);
330
                    unless ($rv)
331
                    {
332
                        Warning("Could not link $PackageStore, $releaseDir")
333
                    }
334
                }
335
            }
336
        }
337
    }
338
}
339
 
340
#-------------------------------------------------------------------------------
341
# Function        : rebuildLxrConfig 
342
#
343
# Description     : Rebuild the LXR Configuration file
344
#                   This MAY be a bit LXR version specific, but since LXR doesn't
345
#                   provide a scriptable way to update configuration
346
#
347
#                   Uses template files that have been handcrafted after taken from
5211 dpurdie 348
#                   LXR. Basically we to a text replace and a glue together
5209 dpurdie 349
#
350
#                   For each release we need
351
#                       Long Release Name
352
#                       Short Release Name
353
#                       List of Versions
354
#
355
#
356
# Inputs          : Assumes Data has been added to %ReleaseData by other subroutines
357
#
358
# Returns         : 
359
#
360
sub rebuildLxrConfig
361
{
362
    my @lxrTreeText;
5211 dpurdie 363
 
5209 dpurdie 364
    #
5211 dpurdie 365
    #   Sort Sub
366
    #   Sort ReleaseData by Project and Name
367
    #   $a and $b are special to the sort
368
    #
369
    sub ReleaseDataSort
370
    {
371
        my $rv = lc($ReleaseData{$a}{release}{Project}) cmp lc($ReleaseData{$b}{release}{Project});
372
        if ($rv == 0)
373
        {
374
            $rv = lc($ReleaseData{$a}{release}{Name}) cmp lc($ReleaseData{$b}{release}{Name});
375
        }
376
        return $rv;
377
    }
378
 
379
    #
5209 dpurdie 380
    #   Process configured releases
5211 dpurdie 381
    #   Generate inthe order we wich to display the Releases
5209 dpurdie 382
    #
383
    #DebugDumpData("ReleaseData", \%ReleaseData);
5211 dpurdie 384
    foreach my $rtagid ( sort ReleaseDataSort keys %ReleaseData )
5209 dpurdie 385
    {
386
        my $entry = $ReleaseData{$rtagid};
5211 dpurdie 387
        Information("Entry: $entry->{release}{Project}, $entry->{release}{Name}, $rtagid");
5209 dpurdie 388
        $entry->{release}{VersionsString} = join( ',', map { '"' . $_ .'"'} @{$entry->{Versions}} );
389
        $entry->{release}{dbName} = genDatabaseName($rtagid);
390
        #DebugDumpData("ENTRY", \$entry);
391
 
392
        my $tfileName = 'lxr.tree.template';
393
        open( my $tf, '<', $tfileName) || Error ("Cannot open $tfileName. $!");
394
        while (my $line = <$tf>)
395
        {
396
            # Chomp trailing write space
397
            $line =~ s~\s+$~~;
398
 
399
            #   Replace known bits in the template
400
            $line =~ s~\@CAPTION\@~$entry->{release}{Project}::$entry->{release}{Name}~g;
401
            $line =~ s~\@SHORT_NAME\@~$rtagid~g;
402
            $line =~ s~\@RELEASE_ROOT\@~$entry->{release}{root}~g;
403
            $line =~ s~\@VERSIONS_LIST\@~$entry->{release}{VersionsString}~g;
404
            $line =~ s~\@DBNAME\@~$entry->{release}{dbName}~g;
405
            push @lxrTreeText, $line
406
        }
407
        close $tf;
408
    }
409
 
410
    #
411
    #   Insert tree sections into the main config file template
412
    #
413
    my $hostList = join( ',', map { '\'http://' . $_ .'\''} @addressList );
414
 
415
    my $tfileName = catfile($scriptDir, 'lxr.template');
416
    my $lxrFileName = catfile($config->{lxr}, 'lxr.new.conf');
417
    unlink $lxrFileName;
418
    open( my $tf, '<', $tfileName) || Error ("Cannot open $tfileName. $!");
419
    open( my $to, '>', $lxrFileName) || Error ("Cannot open $lxrFileName. $!");
420
    while (my $line = <$tf>)
421
    {
422
        # Chomp trailing write space
423
        $line =~ s~\s+$~~;
424
 
425
        #   Replace known bits in the template
426
        if ($line =~ m~\@TREE_SECTIONS\@~)
427
        {
428
            foreach (@lxrTreeText)
429
            {
430
                print $to $_, "\n";
431
            }
432
        }
433
        else
434
        {
435
            $line =~ s~\@HOSTLIST\@~$hostList~g;
436
            print $to $line, "\n";
437
        }
438
    }
439
    close $tf;
440
    close $to;
441
 
442
    #
443
    #   Install the new config files
444
    #
445
    my $lxrLive = catfile($config->{lxr}, 'lxr.conf');
446
    my $lxrBackup = catfile($config->{lxr}, 'lxr.conf.bak');
447
    unlink $lxrBackup;
448
    rename ($lxrLive, $lxrBackup) || Warning("Renaming $lxrLive, $lxrBackup", $!);
449
    rename ($lxrFileName, $lxrLive) || Warning("Renaming $lxrFileName, $lxrLive", $!);
450
 
451
    #
452
    #   Create new database tables if required
453
    #   Use a customized shell script to do the hard work
454
    #
455
    foreach my $rtagid ( keys %ReleaseData )
456
    {
457
        my $entry = $ReleaseData{$rtagid};
458
        Information("Database:$entry->{release}{Name}, $entry->{release}{dbName} ");
459
        System('--NoExit', '--NoShell', catfile($scriptDir, 'lxr.initdb.sh'), $entry->{release}{dbName});
460
    }
461
}
462
 
463
#-------------------------------------------------------------------------------
464
# Function        : buildIndexes 
465
#
466
# Description     : Build (Generate) indexes for all versions of all releases
467
#                   that don't have an index
468
#                   
469
#                   Place a marker file in the 'version' directory when the
470
#                   index has been created
471
#
472
#                   Notes:
473
#                   The 'genxref' program needs to be run from the lxr insatll directory
474
#                   chdir to that directory for each invocation
475
#
476
#                   genxref uses DBI - and must find the proper PERL DBI and not the one
477
#                   within JATS.. Solution:
478
#                       Kill the PERL5LIB EnvVar
479
#
480
#
481
#
482
# Inputs          : 
483
#
484
# Returns         : 
485
#
486
sub buildIndexes
487
{
488
    #
489
    #   Prep envonment for calling genxref
490
    #   See notes above
491
    #
492
    chdir ($config->{lxr}) || Error ("Cannot chnage directory:$config->{lxr}, $!");
493
    delete $ENV{PERL5LIB};
494
 
495
    #
496
    #   Examine each Version in each Release
497
    #   Generate command line like:
498
    #   genxref --tree RTAGID --url http://HOSTNAME/lxr --version 'VERSION'
499
    foreach my $rtagid ( keys %ReleaseData )
500
    {
501
        my $entry = $ReleaseData{$rtagid};
502
        foreach my $version (@{$entry->{Versions}})
503
        {
504
            my $markerFile = catfile($entry->{release}{root}, $version, '.lxrIndexed');
505
            unless (-f $markerFile) {
506
                Message("Must index: $rtagid, $version");
507
                my $rv = System('--NoExit', 
508
                       '--NoShell', 
509
                       catfile($config->{lxr}, 'genxref'),
510
                       '--url', 'http://' . $ENV{GBE_HOSTNAME} . '/lxr',
511
                       '--tree', $rtagid,
512
                       '--version', $version
513
                       );
514
               Verbose("genxref exit: $rv");
515
               unless ($rv) {
516
                   TouchFile($markerFile);
517
               }
518
            }
519
            else
520
            {
521
                Verbose("Already indexed: $rtagid, $version");
522
            }
523
        }
524
    }
525
    #
526
    #   Restore current directory
527
    #
528
    chdir($FileUtils::CwdFull);
529
}
530
 
531
 
532
#-------------------------------------------------------------------------------
533
# Function        : cleanReleaseViews 
534
#
535
# Description     : Clean up unused Releases and Release Views
536
#                   Maintain the list of retained versions
537
#
538
#                   Two classes
539
#                   Active - Marked as having LXR support
540
#                       Retain the last 5 Versions
541
#
542
#                  InActive - Not having LXR support
543
#                       Retain for 10 days then delete all versions
544
#
545
# Inputs          : 
546
#
547
# Returns         : 
548
#
549
sub cleanReleaseViews
550
{
551
    #
552
    #   Scan Releases and delete all that are not currently configured
553
    #
554
    opendir (my $rdir, $ReleaseStore) || Error ("Cannot open directory: $ReleaseStore", $!);
555
    while (my $rdirEntry = readdir($rdir))
556
    {
557
        #   Skip hidden files and directories
558
        next if ($rdirEntry =~ m~^\.~);
559
        my $vdirName = catdir($ReleaseStore, $rdirEntry );
560
        next unless ( -d $vdirName );
561
        unless(exists $ReleaseData{$rdirEntry} && $ReleaseData{$rdirEntry}{release}{LXR} )
562
        {
5211 dpurdie 563
            #   Release is no longer configured - age it out
5209 dpurdie 564
            #   Assume $rdirEntry is an rtag_id
5211 dpurdie 565
            if (processAgeMarker($vdirName, $config->{'releaseAge'} ))
566
            {
567
                Message("Delete Release: $rdirEntry");
568
                RmDirTree($vdirName);
569
                System('--NoExit', '--NoShell', catfile($scriptDir, 'lxr.dropdb.sh'), genDatabaseName($rdirEntry));
570
            }
5209 dpurdie 571
        }
572
        else
573
        {
574
            $ReleaseData{$rdirEntry}{release}{root} = $vdirName;
5211 dpurdie 575
            deleteAgeMarker($vdirName);
5209 dpurdie 576
 
577
            #   Release is configured
578
            #   Keep the last x created
579
            #   Note: Create time is a kludge
580
            #
581
            #   Process each version within the Release
582
            #
583
            my @versionData;
584
            opendir (my $vdir, $vdirName) || Warning ("Cannot open directory: $vdirName", $!);
585
            while (my $vdirEntry = readdir($vdir))
586
            {
587
                #   Skip hidden files and directories
588
                next if ($vdirEntry =~ m~^\.~);
589
                my $ldirName = catdir($vdirName, $vdirEntry );
590
                next unless ( -d $ldirName );
591
                my $data;
592
                $data->{name} = $vdirEntry; 
593
                $data->{path} = $ldirName; 
594
                $data->{ctime} = (stat $ldirName)[9];
595
                push @versionData  , $data;
596
            }
597
            close ($vdir);
598
            #DebugDumpData("versionData",\@versionData);
599
 
600
            my $keepCount = 0;
601
            foreach my $entry ( sort { $b->{ctime} <=> $a->{ctime}} @versionData )
602
            {
603
                #DebugDumpData("Entry", $entry);
604
                $keepCount++;
605
                if ($keepCount > 5)
606
                {
607
                    #   Version is no longer needed - remove it
608
                    Message("Delete Version: $rdirEntry, $entry->{name}, $entry->{ctime}");
609
                    RmDirTree($entry->{path});
610
                }
611
                else
612
                {
613
                    #   Note which versions we have
614
                    push @{$ReleaseData{$rdirEntry}{Versions}}, $entry->{name};
615
                }
616
            }
617
        }
618
    }
619
    close ($rdir);
620
}
621
 
622
#-------------------------------------------------------------------------------
623
# Function        : getLatestVersion 
624
#
625
# Description     : For a specified directory return the newest subdir
626
#
627
#                   Used to determine the most recent version
628
#
629
# Inputs          : Dir to process - expecting a Release directory
630
#
631
# Returns         : 
632
#
633
sub getLatestVersion
634
{
635
    my  ($vdirName) = @_;
636
    my $latestName;
637
    my $latestAge = 0;
638
 
639
    opendir (my $vdir, $vdirName) || Warning ("Cannot open directory: $vdirName", $!);
640
    while (my $vdirEntry = readdir($vdir))
641
    {
642
        #   Skip hidden files and directories
643
        next if ($vdirEntry =~ m~^\.~);
644
 
645
        my $ldirName = catdir($vdirName, $vdirEntry );
646
        next unless ( -d $ldirName );
647
 
648
        my $age = (stat $ldirName)[9];
649
        Verbose3("Age: $ldirName, $age");
650
 
651
        if  ($age > $latestAge )
652
        {
653
            $latestAge = $age;
654
            $latestName = $ldirName;
655
        }
656
    }
657
    close ($vdir);
658
 
659
    Verbose("Latest: $latestName, $latestAge");
660
    #DebugDumpData("versionDataSorted",\@versionDataSorted);
661
    return $latestName;
662
}
663
 
664
 
665
#-------------------------------------------------------------------------------
666
# Function        : cleanPackageStore 
667
#
668
# Description     : Delete unused PAckages fromthe package store
669
#
670
# Inputs          : 
671
#
672
# Returns         : 
673
#
674
sub cleanPackageStore
675
{
676
    my %pkgsUsed;
677
    #
678
    #   Examime ALL versions in ALL Releases and build up a hash of
679
    #   Package Versions used
680
    #
681
    opendir (my $rdir, $ReleaseStore) || Error ("Cannot open directory: $ReleaseStore", $!);
682
    while (my $rdirEntry = readdir($rdir))
683
    {
684
        #   Skip hidden files and directories
685
        next if ($rdirEntry =~ m~^\.~);
686
        my $vdirName = catdir($ReleaseStore, $rdirEntry );
687
        next unless ( -d $vdirName );
688
 
689
        #
690
        #   Process each version within the Release
691
        #
692
        opendir (my $vdir, $vdirName) || Warning ("Cannot open directory: $vdirName", $!);
693
        while (my $vdirEntry = readdir($vdir))
694
        {
695
            #   Skip hidden files and directories
696
            next if ($vdirEntry =~ m~^\.~);
697
            my $ldirName = catdir($vdirName, $vdirEntry );
698
            next unless ( -d $ldirName );
699
 
700
            #
701
            #   Process each entry within the Version
702
            #
703
            opendir (my $ldir, $ldirName) || Warning ("Cannot open directory: $ldirName", $!);
704
            while (my $ldirEntry = readdir($ldir))
705
            {
706
                #   Skip hidden files and directories (also borken symlinks )
707
                next if ($ldirEntry =~ m~^\.~);
708
                my $dirName = catdir($ldirName, $ldirEntry );
709
                next unless ( -d $dirName );
710
 
711
                #
712
                #   Process each entry within the Version
713
                #
714
                my $pkgName = $ldirEntry;
715
                if (-l $dirName)
716
                {
717
                    $pkgName = readlink($dirName);
718
                    $pkgName =~ s~.*/~~;
719
                }
720
                $pkgsUsed{$pkgName}++;
721
            }
722
            close ($ldir);
723
        }
724
        close ($vdir);
725
    }
726
    close ($rdir);
727
 
728
    #
729
    #   Process the Packages directory and remove those not currently used
730
    #
731
    #
732
    #   Process each entry within the Version
733
    #
734
    opendir (my $pdir, $PackageStore) || Error ("Cannot open directory: $PackageStore", $!);
735
    while (my $pdirEntry = readdir($pdir))
736
    {
737
        #   Skip hidden files and directories
738
        next if ($pdirEntry =~ m~^\.~);
739
        my $pdirName = catdir($PackageStore, $pdirEntry );
740
        next unless ( -d $pdirName );
741
        next if (exists $pkgsUsed{$pdirEntry} );
742
 
5211 dpurdie 743
        if (processAgeMarker($pdirName, $config->{packageAge})) 
744
        {
745
            Message("Purge Package: $pdirEntry");
746
            RmDirTree($pdirName);
747
        }
5209 dpurdie 748
    }
749
    close ($pdir);
750
 
751
    #DebugDumpData("pkgsUsed", \%pkgsUsed);
752
}
753
 
754
 
755
#-------------------------------------------------------------------------------
756
# Function        : getReleaseData 
757
#
758
# Description     : Get all the required Release Data 
759
#
760
# Inputs          :  
761
#
762
# Returns         : 
763
#
764
sub getReleaseData
765
{
766
    my ($rtagid) = @_;
767
    my (@row);
768
 
769
    Information("getReleaseData");
770
    connectRM(\$RM_DB) unless $RM_DB;
771
 
772
 
773
    #
774
    # Determine which Releases need LXR support
775
    #
776
    my $m_sqlstr = 
777
        "SELECT rtag_id, " .
778
        "  rt.proj_id, " .
779
        "  p.PROJ_NAME, " .
780
        "  rtag_name, " .
781
        "  official, " .
782
        "  NVL(TRUNC (SYSDATE - rt.official_stamp),0) AS OFFICIAL_STAMP_DAYS " .
783
        "FROM release_tags rt, " .
784
        "  projects p " .
785
        "WHERE rt.lxr  = 'Y' " .
786
        "AND p.PROJ_ID = rt.proj_id ";
787
 
788
    my $sth = $RM_DB->prepare($m_sqlstr);
789
    if ( defined($sth) )
790
    {
791
        if ( $sth->execute( ) )
792
        {
793
            if ( $sth->rows )
794
            {
795
                while ( @row = $sth->fetchrow_array )
796
                {
797
                    my $rtagid = $row[0];
798
                    my $data;
799
                    $data->{Project} = $row[2];
800
                    $data->{Name} = $row[3];
801
                    $data->{official} = substr($row[4],0,1);
802
                    $data->{official_stamp_days} = $row[5];
803
 
804
                    #
805
                    #   Determine if this request for an LXR release is OK
806
                    #   Ok If the release is Open, CCB or Restricted
807
                    #   Ok if closed and has been closed to < 10 days
808
                    #
809
                    if (index('NRC', $data->{official}) >= 0)
810
                    {
811
                        $data->{LXR} = 1;
812
                    }
813
                    elsif ($data->{official} eq 'Y' && $data->{official_stamp_days} < 10)
814
                    {
815
                        $data->{LXR} = 2;
816
                    }
817
 
818
                    $ReleaseData{$rtagid}{release} = $data;
819
                }
820
            }
821
            $sth->finish();
822
        }
823
    }
824
    else
825
    {
826
        Error("getReleaseData:Prepare failure" );
827
    }
828
 
829
    DebugDumpData("ReleaseData", \%ReleaseData);
830
    foreach my $rtagid ( keys %ReleaseData)
831
    {
832
        next unless $ReleaseData{$rtagid}{release}{LXR};
833
        Information("Entry: RtagId $rtagid");
834
        getOneRelease($rtagid);
835
    }
836
}
837
 
838
#-------------------------------------------------------------------------------
839
# Function        : getOneRelease
840
#
841
# Description     : Get data for one Release
842
#
843
# Inputs          : rtagid
844
#
845
# Returns         : 
846
#
847
sub getOneRelease
848
{
849
    my ($rtagid) = @_;
850
    my (@row);
851
 
852
    connectRM(\$RM_DB) unless $RM_DB;
853
 
854
 
855
    # Get details for each package in the Release
856
    #   Don't worry about dependent packages
857
    #
858
    my $m_sqlstr = "SELECT pv.PV_ID, pkg.PKG_NAME, pv.PKG_VERSION, pv.V_EXT, release_manager.PK_RMAPI.return_vcs_tag(pv.PV_ID)".
859
                   " FROM RELEASE_MANAGER.RELEASE_CONTENT rc, RELEASE_MANAGER.PACKAGE_VERSIONS pv, RELEASE_MANAGER.PACKAGES pkg" .
860
                   " WHERE rc.RTAG_ID = $rtagid AND rc.PV_ID = pv.PV_ID AND pv.PKG_ID = pkg.PKG_ID";
861
    my $sth = $RM_DB->prepare($m_sqlstr);
862
    if ( defined($sth) )
863
    {
864
        if ( $sth->execute( ) )
865
        {
866
            if ( $sth->rows )
867
            {
868
                while ( @row = $sth->fetchrow_array )
869
                {
870
                    my $data;
871
                    my $pvid = $row[0];
872
                    unless (exists $Packages{$pvid}) {
873
                        $data->{pv_id} = $row[0];
874
                        $data->{name} = $row[1];
875
                        $data->{ver} = $row[2];
876
                        $data->{ext} = $row[3] || '';
877
                        $data->{vcs} = $row[4] || '';
878
                        $Packages{$pvid} = $data;
879
                    }
880
                    my $rdata;
881
                    $rdata->{required} = 1;
882
                    $ReleaseData{$rtagid}{data}{$pvid} = $rdata;
883
                }
884
            }
885
            $sth->finish();
886
        }
887
    }
888
    else
889
    {
890
        Error("getOneRelease:Prepare failure" );
891
    }
892
}
893
 
894
#-------------------------------------------------------------------------------
895
# Function        : genDatabaseName 
896
#
897
# Description     : Genertate the name of a database
898
#
899
# Inputs          : rtag_id 
900
#
901
# Returns         : Text Name of the database
902
#
903
 
904
sub genDatabaseName
905
{
906
    my ($rtag_id) = @_;
907
    return 'LXR_' . $rtag_id;
908
}
909
 
910
#-------------------------------------------------------------------------------
911
# Function        : readConfig 
912
#
913
# Description     : Read the basic LXR config
914
#                   This is held in a Perl structure 
915
#
916
# Inputs          : None 
917
#
918
# Returns         : Populate Global Data
919
#
920
sub readConfig
921
{
922
    my $cfile = catfile($scriptDir,'jats_lxr.conf');
5211 dpurdie 923
    if ($opt_config)
924
    {
925
        Message ("Using alternate config: $opt_config");
926
        $cfile = $opt_config;
927
    }
5209 dpurdie 928
    Error ("Expected config file not found: $cfile") unless ( -f $cfile );
929
    require $cfile;
930
 
931
#   #
932
#   #   Create data
933
#   #
934
#   my %hash = (
935
#   number => 42,
936
#   string => 'This is a string',
937
#   array  => [ 1 .. 10 ],
938
#   hash   => { apple => 'red', banana => 'yellow' },
939
#   );
940
#
941
#   # Print structure to file
942
#   open (my $out, '>', $cfile) || Error("Can't create $cfile. $!");
943
#   DebugDumpData("Hash", \%hash);
944
#   print {$out} Data::Dumper->Dump([\%hash], ['$config']);
945
#   close $out;
946
 
947
    #DebugDumpData("config", \$config);
948
 
949
}
950
 
951
#-------------------------------------------------------------------------------
952
# Function        : startLogFile 
953
#
954
# Description     : Start logging to a log file
955
#                   Generate a nice name for the log file
956
#
957
# Inputs          : 
958
#
959
# Returns         : 
960
#
961
sub startLogFile
962
{
963
    if ( $opt_logfile )
964
    {
965
        if (exists $config->{lxrLogDir})
966
        {
967
            my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime($stamptime);
968
            my $name = sprintf("jats-lxr-%4.4d%2.2d%2.2d.%2.2d%2.2d%2.2d.log", 1900+$year, 1+$mon, $mday, $hour, $min, $sec);
969
            $name = catdir($config->{lxrLogDir}, $name);
970
 
971
            mkpath($config->{lxrLogDir}); 
972
            if (-d $config->{lxrLogDir} )
973
            {
974
                open STDOUT, '>', $name  or die "Can't redirect STDOUT: $!";
975
                open STDERR, ">&STDOUT"  or die "Can't dup STDOUT: $!";
976
            }
977
            else
978
            {
979
                Warning("Can't create log dir: $config->{lxrLogDir}. $!");
980
            }
981
        }
982
    }
983
}
984
 
985
#-------------------------------------------------------------------------------
986
# Function        : cleanupLogFiles 
987
#
988
# Description     : Remove old log files
989
#
990
# Inputs          : 
991
#
992
# Returns         : 
993
#
994
sub cleanupLogFiles
995
{
996
    if (exists $config->{lxrLogDir} && exists $config->{logAge}  && $config->{logAge} > 0 )
997
    {
998
        Verbose("cleanupLogFiles:$config->{lxrLogDir}, $config->{logAge}");
999
        if ( opendir my $logDir, $config->{lxrLogDir} )
1000
        {
1001
            foreach my $fileName (readdir $logDir)
1002
            {
1003
                my $file = catfile($config->{lxrLogDir}, $fileName);
1004
                next unless -f $file;
1005
                next unless -M $file > $config->{logAge};
1006
                Verbose("Purge logfile: $fileName");
1007
                unlink $file;
1008
            }
1009
            closedir $logDir;
1010
        }
1011
    }
1012
}
1013
 
5211 dpurdie 1014
#-------------------------------------------------------------------------------
1015
# Function        : processAgeMarker 
1016
#
1017
# Description     : Age out directories
1018
#                   Will create age markers as required
1019
#
1020
# Inputs          : $tdir       - Target Directory
1021
#                   $age        - Configured age 
1022
#
1023
# Returns         : true        - Directory has reached age
1024
#
1025
sub processAgeMarker
1026
{
1027
    my ($tdir, $age) = @_;
1028
 
1029
    unless (-d $tdir) {
1030
        Warning ("Expected directory not found: $tdir");
1031
        return 0; 
1032
    }
5209 dpurdie 1033
 
5211 dpurdie 1034
    #   A configured age of 0 implies delete immediatly
1035
    if ($age == 0)
1036
    {
1037
        return 1;
1038
    }
1039
 
1040
    #
1041
    #   Create the file ONCE
1042
    #
1043
    my $markerfile = catfile($tdir, '.lxrAge');
1044
    unless (-f $markerfile)
1045
    {
1046
        TouchFile($markerfile);
1047
    }
1048
    else
1049
    {
1050
        if (-M $markerfile > $age)
1051
        {
1052
            return 1
1053
        }
1054
    }
1055
    return 0;
1056
}
1057
 
1058
 
5209 dpurdie 1059
#-------------------------------------------------------------------------------
5211 dpurdie 1060
# Function        : deleteAgeMarker 
1061
#
1062
# Description     : Delete any age marker
1063
#                   Used when a (potentially) agable directory is used to remove
1064
#                   the aging marker
1065
#
1066
# Inputs          :  $tdir      - Directory
1067
#
1068
# Returns         : 
1069
#
1070
sub deleteAgeMarker
1071
{
1072
    my ($tdir) = @_;
1073
 
1074
    unless (-d $tdir) {
1075
        Warning ("Expected directory not found: $tdir");
1076
        return 0; 
1077
    }
1078
 
1079
    #
1080
    #   Use same file name as in processAgeMarker
1081
    #
1082
    my $markerfile = catfile($tdir, '.lxrAge');
1083
    unlink $markerfile;
1084
}
1085
 
1086
 
1087
#-------------------------------------------------------------------------------
5209 dpurdie 1088
#   Documentation
1089
#
1090
 
1091
=pod
1092
 
1093
=head1 NAME
1094
 
1095
jats_lxr - Maintain LXR Releases
1096
 
1097
=head1 SYNOPSIS
1098
 
1099
  jats jats_lxr [options]
1100
 
1101
 Options:
1102
    -help               - brief help message
1103
    -help -help         - Detailed help message
1104
    -man                - Full documentation
1105
    -[no]createVersions - Create new versions. Default:Create
1106
    -[no]extract        - Extract source code. Default:Extract
1107
    -[no]index          - Index new LXR versions. Default:Index
1108
    -[no]purge          - Purge unused packages. Default:Purge
1109
    -[no]logfile        - Capture out to a log file. Default:Log
5211 dpurdie 1110
    -config=file        - Alternate config file
5209 dpurdie 1111
 
1112
=head1 OPTIONS
1113
 
1114
=over 8
1115
 
1116
=item B<-help>
1117
 
1118
Print a brief help message and exits.
1119
 
1120
=item B<-help -help>
1121
 
1122
Print a detailed help message with an explanation for each option.
1123
 
1124
=item B<-man>
1125
 
1126
Prints the manual page and exits.
1127
 
1128
=item B<-verbose>
1129
 
1130
This option will display progress information as the program executes.
1131
 
1132
=item B<-[no]createVersions>
1133
 
1134
This option can be used to suppress the creation of new views.
1135
 
1136
=item B<-[no]extract>
1137
 
1138
This option can be used to suppress the extraction of source.
1139
 
1140
=item B<-[no]index>
1141
 
1142
This option can be used to suppress the indexing of newly created views.
1143
 
1144
=item B<-[no]purge>
1145
 
1146
This option can be used to suppress purging of packages that are no longer used by any of the LXR Trees.
1147
 
5211 dpurdie 1148
=item B<-config=file>
1149
 
1150
This option can be used to override the standard config file. Used in testing.
1151
 
5209 dpurdie 1152
=back
1153
 
1154
=head1 DESCRIPTION
1155
 
1156
This program is a tool for creating and maintaining an LXR instance within the VIX Build System.
1157
 
1158
The program can:
1159
 
1160
=over 8
1161
 
1162
=item * 
1163
 
1164
Examine the Release Manager Database and determine which Releases are a part of the set to have LXR Views created.
1165
 
1166
Releases that are Open (or Restricted or in CCB Mode) or that have been closed for less than 10 days will be processed.
1167
 
1168
=item *
1169
 
1170
Examine the Release Manager Database and determine the package-versions that are a part of the LXR Release Set.
1171
 
1172
The process will only examine the contents of a Release. It will not descend the version dependency tree.
1173
 
1174
The program will then extract new package-versions into it's Package Store. Multiple LXR Tree's will share the one 
1175
instance of the extracted source code.
1176
 
1177
=item *
1178
 
1179
Create a new 'Version' for each Release.
1180
 
1181
The Version Name is based on the date-time at which the process is run.
1182
 
1183
Each Version is a symlink to the required package-version held in the Package Store.
1184
 
1185
This step can be suppressed though a command line option.
1186
 
1187
=item *
1188
 
1189
Remove unused LXR Trees (Releases no longer being processed) and unused Versions within each Tree.
1190
 
1191
Releases that are no longer eligable for processing will be retained for 10 days and then deleted. 
1192
During this time the existing versions will not be aged out.
1193
 
1194
At the moment this tool simply retains the last 5 Versions of each Release. If the tool 
1195
is run nightly then this will translate into  5 days.
1196
 
1197
=item *
1198
 
1199
Regenerate the LXR Configuration file
1200
 
1201
Create new database tables for new Releases
1202
 
1203
=item *
1204
 
1205
Run the LXR indexer for each LXR Version that has not been processed. Once processed the Version will 
1206
be tagged to prevent further indexing.
1207
 
1208
=item *
1209
 
1210
Remove unused Package-Versions from the Package Store.
1211
 
1212
=back
1213
 
1214
=cut
1215