Subversion Repositories DevTools

Rev

Rev 5212 | Rev 5231 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 5212 Rev 5213
Line 4... Line 4...
4
# Module name   : jats_lxr.pl
4
# Module name   : jats_lxr.pl
5
# Module type   : JATS Utility
5
# Module type   : JATS Utility
6
# Compiler(s)   : Perl
6
# Compiler(s)   : Perl
7
# Environment(s): jats
7
# Environment(s): jats
8
#
8
#
9
# Description   : Tiools to maintain LXR information
9
# Description   : Tools to maintain LXR information
-
 
10
#
-
 
11
# Possible imporvements
-
 
12
#       Delete files we don't want to index
-
 
13
#           .DMP,.dmp,.so,.gz,.zip,.lib,.bz,.bz2,.tbz,.jar,.deb,.exe,.a,.msb
-
 
14
#
-
 
15
#       Merge ripples package versions
-
 
16
#           Just keep one
-
 
17
#
-
 
18
#       Cots packages
-
 
19
#           Use contents from dpkg_archive instead of the package
10
#
20
#
11
# Usage         : See POD at the end of this file
21
# Usage         : See POD at the end of this file
12
#
22
#
13
#......................................................................#
23
#......................................................................#
14
 
24
 
Line 19... Line 29...
19
use Pod::Usage;
29
use Pod::Usage;
20
use Getopt::Long;
30
use Getopt::Long;
21
use File::Path;
31
use File::Path;
22
use FindBin;
32
use FindBin;
23
use Socket;
33
use Socket;
-
 
34
use Fcntl ':flock';
24
 
35
 
25
use JatsError;
36
use JatsError;
26
use JatsRmApi;
37
use JatsRmApi;
27
use DBI;
38
use DBI;
28
use JatsSystem;
39
use JatsSystem;
Line 45... Line 56...
45
#
56
#
46
#   Globals
57
#   Globals
47
#
58
#
48
my $scriptDir;
59
my $scriptDir;
49
my $RM_DB;
60
my $RM_DB;
50
our $config;
61
my $Config;
51
my @addressList;
62
my @addressList;
52
my %ReleaseData;
63
my %ReleaseData;
53
my %Packages;
64
my %Packages;
54
my $PackageStore;
65
my $PackageStore;
55
my $ReleaseStore;
66
my $ReleaseStore;
56
my $stamptime = time;
67
my $StampTime = time;
-
 
68
my $dateTag = localtime($StampTime);
-
 
69
my $lockFile = '/tmp/JATSLXR_LOCK';
57
 
70
 
58
#-------------------------------------------------------------------------------
71
#-------------------------------------------------------------------------------
59
# Function        : Main
72
# Function        : Main
60
#
73
#
61
# Description     : Main entry point
74
# Description     : Main entry point
Line 99... Line 112...
99
$scriptDir = $FindBin::Bin;
112
$scriptDir = $FindBin::Bin;
100
InitFileUtils();
113
InitFileUtils();
101
 
114
 
102
#
115
#
103
#   Read the tool configuration
116
#   Read the tool configuration
-
 
117
#   Sed with Default config
104
#
118
#
-
 
119
$Config->{'releaseAge'} = 0;
-
 
120
$Config->{'packageAge'} = 0;
-
 
121
$Config->{'logAge'} = 0;
-
 
122
$Config->{'verbose'} = 0;
105
readConfig();
123
readConfig();
106
Error ("No LXR Data directory defined", $config->{lxrFiles}) unless( exists $config->{lxrFiles});
124
Error ("No LXR Data directory defined", $Config->{lxrFiles}) unless( exists $Config->{lxrFiles});
107
Error ("No LXR Data directory found", $config->{lxrFiles}) unless( -d $config->{lxrFiles});
125
Error ("No LXR Data directory found", $Config->{lxrFiles}) unless( -d $Config->{lxrFiles});
108
Error ("No LXR Install directory defined",$config->{lxr}) unless( exists $config->{lxr});
126
Error ("No LXR Install directory defined",$Config->{lxr}) unless( exists $Config->{lxr});
109
Error ("No LXR Install directory found",$config->{lxr}) unless( -d $config->{lxr});
127
Error ("No LXR Install directory found",$Config->{lxr}) unless( -d $Config->{lxr});
-
 
128
ErrorConfig( 'verbose' => $Config->{'verbose'} );
110
 
129
 
111
#
130
#
112
#   Start logging
131
#   Start logging
113
#
132
#
114
startLogFile();
133
startLogFile();
115
Message ("Start of LXR. " . localtime(time) ) if $opt_logfile;
134
Message ("Start of LXR. " . localtime(time) ) if $opt_logfile;
116
Message("ScriptDir: $scriptDir");
135
Message("ScriptDir: $scriptDir");
117
 
136
 
118
#
137
#
-
 
138
#   Prevent multiple instances of the program running
-
 
139
#
-
 
140
#   PerlMonks say to use $0 as the lock file, but that did not work.
-
 
141
#   Perhaps the file was open in an editor
-
 
142
#   Solution: Create my own file
-
 
143
#
-
 
144
open( FW, '>',$lockFile);
-
 
145
print FW '';
-
 
146
close FW;
-
 
147
 
-
 
148
open (my $self, '<', $lockFile)    || Error("Couldn't open self: $!");
-
 
149
flock ($self, (LOCK_EX | LOCK_NB)) || Error("This script is already running");
-
 
150
 
-
 
151
#
119
#   Check required paths exist
152
#   Check required paths exist
120
#
153
#
121
$PackageStore = catdir($config->{lxrFiles}, 'Packages');
154
$PackageStore = catdir($Config->{lxrFiles}, 'Packages');
122
$ReleaseStore = catdir($config->{lxrFiles}, 'Releases');
155
$ReleaseStore = catdir($Config->{lxrFiles}, 'Releases');
123
mkpath($PackageStore);
156
mkpath($PackageStore);
124
mkpath($ReleaseStore);
157
mkpath($ReleaseStore);
125
Error ("Package Store not found ",$PackageStore) unless( -d $PackageStore);
158
Error ("Package Store not found ",$PackageStore) unless( -d $PackageStore);
126
Error ("Release Store not found ",$ReleaseStore) unless( -d $ReleaseStore);
159
Error ("Release Store not found ",$ReleaseStore) unless( -d $ReleaseStore);
127
 
160
 
128
#
161
#
129
#   Default config
-
 
130
#
-
 
131
$config->{'releaseAge'} = 0 unless exists $config->{'releaseAge'};
-
 
132
$config->{'packageAge'} = 0 unless exists $config->{'packageAge'};
-
 
133
 
-
 
134
#
-
 
135
#   Determine the various names for this machine
162
#   Determine the various names for this machine
136
#   Include all the IP addresses too
163
#   Include all the IP addresses too
137
#
164
#
138
my ($canonname, $aliases, $addrtype, $length, @addrs) = gethostbyname($ENV{GBE_HOSTNAME});
165
my ($canonname, $aliases, $addrtype, $length, @addrs) = gethostbyname($ENV{GBE_HOSTNAME});
139
push @addressList, $canonname ;
166
push @addressList, $canonname ;
Line 146... Line 173...
146
 
173
 
147
#
174
#
148
#   Perform the hard work
175
#   Perform the hard work
149
#
176
#
150
getReleaseData();
177
getReleaseData();
151
extractPackages();
-
 
152
createReleaseViews() if $opt_createVersions;
178
updateReleaseViews() if $opt_createVersions;
153
cleanReleaseViews();
179
cleanReleaseViews();
154
rebuildLxrConfig();
180
rebuildLxrConfig()  if $opt_index;
155
buildIndexes() if $opt_index;
181
buildIndexes() if $opt_index;
156
cleanPackageStore() if $opt_purge;
182
cleanPackageStore() if $opt_purge;
157
cleanupLogFiles() if $opt_purge;
183
cleanupLogFiles() if $opt_purge;
158
 
184
 
159
#
185
#
160
#   All done
186
#   All done
161
#
187
#
-
 
188
unlink $lockFile;
162
Message ("End of LXR. " . localtime(time) ) if $opt_logfile;
189
Message ("End of LXR. " . localtime(time) ) if $opt_logfile;
163
exit 0;
190
exit 0;
164
 
191
 
165
#-------------------------------------------------------------------------------
192
#-------------------------------------------------------------------------------
166
# Function        : extractPackages 
193
# Function        : extractPackages 
Line 168... Line 195...
168
# Description     : Extract Packages from VCS
195
# Description     : Extract Packages from VCS
169
#                   Don't extract them if they already exist
196
#                   Don't extract them if they already exist
170
#   
197
#   
171
#                   Kludge: Prevent extraction from MASS_Dev_Crypto Repo
198
#                   Kludge: Prevent extraction from MASS_Dev_Crypto Repo
172
#
199
#
173
# Inputs          : 
200
# Inputs          : $rtagid         - Release To Process
-
 
201
#                                     Conatins PackageVersion data  
174
#
202
#
175
# Returns         : 
203
# Returns         : 
176
#
204
#
177
sub extractPackages
205
sub extractPackages
178
{
206
{
-
 
207
    my ($rtagid) = @_;
179
    foreach my $pvid ( keys{%Packages})
208
    foreach my $pvid (keys %{$ReleaseData{$rtagid}{data}})
180
    {
209
    {
181
        my $entry = $Packages{$pvid};
210
        my $entry = $Packages{$pvid};
182
        #DebugDumpData("Entry", \$entry);
211
        #DebugDumpData("Entry", \$entry);
183
        my $fullName = join('_', $entry->{name}, $entry->{ver});
212
        my $fullName = join('_', $entry->{name}, $entry->{ver});
184
        my $fullPath = catfile($PackageStore , $fullName);
213
        my $fullPath = catfile($PackageStore , $fullName);
Line 211... Line 240...
211
        }
240
        }
212
    }
241
    }
213
}
242
}
214
 
243
 
215
#-------------------------------------------------------------------------------
244
#-------------------------------------------------------------------------------
216
# Function        : createReleaseViews 
245
# Function        : updateReleaseViews 
217
#
246
#
218
# Description     : Create a new view for each Release
247
# Description     : 
219
#                   Each view is basically a bunch of symlinks
-
 
220
#
248
#
221
# Inputs          : 
249
# Inputs          : 
222
#
250
#
223
# Returns         : 
251
# Returns         : 
224
#
252
#
225
sub createReleaseViews
253
sub updateReleaseViews
226
{
254
{
227
    my $latestView;
-
 
228
    my $needNewView;
-
 
229
 
-
 
230
    #
255
    #
231
    #   Create a name for the releases - based on a date-time
-
 
232
    #   Will be unqiue(ish)
256
    #   Process each known Release
233
    #
257
    #
234
    my $dateTag = localtime($stamptime);
-
 
235
 
-
 
236
    foreach my $rtagid (keys \%ReleaseData)
258
    foreach my $rtagid (sort keys \%ReleaseData)
237
    {
259
    {
238
        $latestView = undef;
-
 
239
        $needNewView = 0;
-
 
240
 
-
 
241
        my $entry = $ReleaseData{$rtagid};
260
        if ($ReleaseData{$rtagid}{release}{ACTIVE})
242
        if ($entry->{data} )
-
 
243
        {
-
 
244
            $latestView = getLatestVersion(catdir($ReleaseStore, $rtagid));
-
 
245
            $needNewView = 1;
-
 
246
        }
-
 
247
 
-
 
248
        #
-
 
249
        #   Check to see if we really need to create a new view
-
 
250
        #   If the LATEST view contains all the package-versions that we need then
-
 
251
        #   don't create a new one.
-
 
252
        #
-
 
253
        #   Scan each entry within the Last View
-
 
254
        #
-
 
255
        if ($latestView)
-
 
256
        {
261
        {
-
 
262
            #
-
 
263
            #   Determinte last created view in this Release
257
            my %pkgsUsed;
264
            #
258
            opendir (my $ldir, $latestView) || Warning ("Cannot open directory: $latestView", $!);
265
            my ($latestView, $latestAge) = getLatestVersion(catdir($ReleaseStore, $rtagid ));
259
            while (my $ldirEntry = readdir($ldir))
266
            if ( ! defined($latestView) || $latestAge > 1 )
260
            {
267
            {
261
                #   Skip hidden files
-
 
262
                next if ($ldirEntry =~ m~^\.~);
-
 
263
                my $dirName = catdir($latestView, $ldirEntry );
-
 
264
 
-
 
265
                #
268
                #
-
 
269
                #   If there is no latest view, then we need to create a new view
-
 
270
                #   If there is a recent view, but its older than a day then we may need
266
                #   Process each entry within the Version
271
                #   to refresh it.
267
                #
272
                #
268
                my $pkgName = $ldirEntry;
273
                getReleasePakageData($rtagid);
269
                if (-l $dirName)
274
                if (checkViewDiffs($latestView,$rtagid) )
270
                {
275
                {
-
 
276
                    #
-
 
277
                    #   Need to create a view
-
 
278
                    #       Either one does not exist
-
 
279
                    #       Content has changed
-
 
280
                    #
271
                    $pkgName = readlink($dirName);
281
                    createReleaseView($rtagid);
-
 
282
                }
-
 
283
                else
-
 
284
                {
-
 
285
                    #   No need to create a new view
-
 
286
                    #   Do need to tag it so that we don't examine it again ( for a while )
-
 
287
                    Message("No Changes to LXR View: $rtagid");
272
                    $pkgName =~ s~.*/~~;
288
                    if (defined $latestView)
-
 
289
                    {
-
 
290
                        my $rv = utime $StampTime,$StampTime, $latestView; 
-
 
291
                        Debug0("Utime $rv: $latestView");
-
 
292
                    }
273
                }
293
                }
274
                $pkgsUsed{$pkgName} = 2;
-
 
275
            }
294
            }
276
            close ($ldir);
-
 
277
 
-
 
278
            #
-
 
279
            #   Compare Package-Versions against those we need
-
 
280
            # 
295
            else
281
            foreach my $pvid (keys $entry->{data})
-
 
282
            {
296
            {
283
                my $entry = $Packages{$pvid};
297
                Message("Recent LXR View: $rtagid");
284
                my $fullName = join('_', $entry->{name}, $entry->{ver});
-
 
285
                $pkgsUsed{$fullName}++;
-
 
286
            }
298
            }
-
 
299
        }
287
            
300
        else
-
 
301
        {
-
 
302
            Message("Inactive LXR View: $rtagid");
-
 
303
        }
-
 
304
    }
-
 
305
}
-
 
306
 
-
 
307
#-------------------------------------------------------------------------------
-
 
308
# Function        : checkViewDiffs 
-
 
309
#
-
 
310
# Description     : Check a view against the current package versions in the
-
 
311
#                   Release 
-
 
312
#
-
 
313
# Inputs          : $vdir       - View entry to process
-
 
314
#                   $rtagid     - RtagId
-
 
315
#
-
 
316
# Returns         : True, If we need to create a new view
-
 
317
#
-
 
318
sub checkViewDiffs
-
 
319
{
-
 
320
    my ($vdir, $rtagid) = @_;
-
 
321
 
-
 
322
    #   No entry to process, then we need to create a view
-
 
323
    return 1 if not defined $vdir;
-
 
324
 
-
 
325
    my %pkgsUsed;
-
 
326
    my $needNewView = 0;
-
 
327
 
-
 
328
    if (opendir (my $ldir, $vdir))
-
 
329
    {
-
 
330
        while (my $ldirEntry = readdir($ldir))
-
 
331
        {
-
 
332
            #   Skip hidden files
-
 
333
            next if ($ldirEntry =~ m~^\.~);
-
 
334
            my $dirName = catdir($vdir, $ldirEntry );
-
 
335
 
288
            #
336
            #
289
            #   Scan the pkgUsed
-
 
290
            #   A value of 1 indicates that it is used only in the New Version
-
 
291
            #   A value of 2 indicates that it is only used on the Last Version
-
 
292
            #   A value of 3 indicates that its used in both
-
 
293
            #   Detect those that are not a 3
337
            #   Process each entry within the Version
294
            #
338
            #
295
            $needNewView = 0;
339
            my $pkgName = $ldirEntry;
296
            foreach ( keys %pkgsUsed)
340
            if (-l $dirName)
297
            {
341
            {
298
                if ($pkgsUsed{$_} != 3)
342
                $pkgName = readlink($dirName);
299
                {
-
 
300
                    $needNewView = 1;
343
                $pkgName =~ s~.*/~~;
301
                    last;
-
 
302
                }
-
 
303
            }
344
            }
304
            #DebugDumpData("pkgsUsed",\%pkgsUsed);
345
            $pkgsUsed{$pkgName} = 2;
305
        }
346
        }
-
 
347
        close ($ldir);
306
 
348
 
-
 
349
        #
-
 
350
        #   Compare Package-Versions against those we need
-
 
351
        # 
-
 
352
        foreach my $pvid (keys %{$ReleaseData{$rtagid}{data}})
-
 
353
        {
-
 
354
            my $entry = $Packages{$pvid};
-
 
355
            my $fullName = join('_', $entry->{name}, $entry->{ver});
-
 
356
            $pkgsUsed{$fullName}++;
-
 
357
        }
-
 
358
 
-
 
359
        #
-
 
360
        #   Scan the pkgUsed
-
 
361
        #   A value of 1 indicates that it is used only in the New Version
-
 
362
        #   A value of 2 indicates that it is only used on the Last Version
-
 
363
        #   A value of 3 indicates that its used in both
-
 
364
        #   Detect those that are not a 3
-
 
365
        #
307
        unless ($needNewView)
366
        foreach ( keys %pkgsUsed)
308
        {
367
        {
-
 
368
            if ($pkgsUsed{$_} != 3)
-
 
369
            {
309
            Message("No Changes to LXR View: $rtagid");
370
                $needNewView = 1;
-
 
371
                last;
-
 
372
            }
310
        }
373
        }
-
 
374
    }
311
        else
375
    else
-
 
376
    {
-
 
377
        Warning ("Cannot open directory: $vdir", $!);
-
 
378
        $needNewView = 1;
-
 
379
    }
-
 
380
    #DebugDumpData("pkgsUsed",\%pkgsUsed);
-
 
381
    return $needNewView;
-
 
382
}
-
 
383
 
-
 
384
#-------------------------------------------------------------------------------
-
 
385
# Function        : createReleaseView 
-
 
386
#
-
 
387
# Description     : Create a new view for a single Release
-
 
388
#
-
 
389
# Inputs          : $rtagid     - RtagId
-
 
390
#
-
 
391
# Returns         : 
-
 
392
#
-
 
393
sub createReleaseView
-
 
394
{
-
 
395
    my ($rtagid) = @_;
-
 
396
 
-
 
397
    #
-
 
398
    #   Ensure that packages have been extracted
-
 
399
    #
-
 
400
    extractPackages($rtagid);
-
 
401
 
-
 
402
    #
-
 
403
    #   Create the actual view directory
-
 
404
    #   Its simply a bunch of symlinks back to the package store
-
 
405
    #
-
 
406
    Message("Creating LXR View: $rtagid, $dateTag");
-
 
407
    my $releaseDir = catdir($ReleaseStore, $rtagid, $dateTag);
-
 
408
    mkpath($releaseDir);
-
 
409
    if (-d $releaseDir)
-
 
410
    {
-
 
411
        foreach my $pvid (keys %{$ReleaseData{$rtagid}{data}})
312
        {
412
        {
-
 
413
            my $entry = $Packages{$pvid};
-
 
414
            my $alias = join('', $entry->{name}, $entry->{ext});
-
 
415
            my $fullName = join('_', $entry->{name}, $entry->{ver});
313
            Message("Creating LXR View: $rtagid, $dateTag");
416
            my $PackageStore = catdir($PackageStore , $fullName);
314
            my $releaseDir = catdir($ReleaseStore, $rtagid, $dateTag);
417
            my $releaseDir = catdir($releaseDir, $alias );
315
            mkpath($releaseDir);
418
            next if -l $releaseDir;
-
 
419
 
-
 
420
            Verbose("Symlink $PackageStore, $releaseDir");
-
 
421
            my $rv = symlink ($PackageStore, $releaseDir);
316
            if (-d $releaseDir)
422
            unless ($rv)
317
            {
423
            {
318
                foreach my $pvid (keys $entry->{data})
-
 
319
                {
-
 
320
                    my $entry = $Packages{$pvid};
-
 
321
                    my $alias = join('', $entry->{name}, $entry->{ext});
-
 
322
                    my $fullName = join('_', $entry->{name}, $entry->{ver});
-
 
323
                    my $PackageStore = catdir($PackageStore , $fullName);
-
 
324
                    my $releaseDir = catdir($releaseDir, $alias );
-
 
325
                    next if -l $releaseDir;
-
 
326
 
-
 
327
                    Verbose("Symlink $PackageStore, $releaseDir");
-
 
328
                    my $rv = symlink ($PackageStore, $releaseDir);
-
 
329
                    unless ($rv)
-
 
330
                    {
-
 
331
                        Warning("Could not link $PackageStore, $releaseDir")
424
                Warning("Could not link $PackageStore, $releaseDir")
332
                    }
-
 
333
                }
-
 
334
            }
425
            }
335
        }
426
        }
336
    }
427
    }
337
}
428
}
338
 
429
 
Line 384... Line 475...
384
    {
475
    {
385
        my $entry = $ReleaseData{$rtagid};
476
        my $entry = $ReleaseData{$rtagid};
386
        Information("Entry: $entry->{release}{Project}, $entry->{release}{Name}, $rtagid");
477
        Information("Entry: $entry->{release}{Project}, $entry->{release}{Name}, $rtagid");
387
        $entry->{release}{VersionsString} = join( ',', map { '"' . $_ .'"'} @{$entry->{Versions}} );
478
        $entry->{release}{VersionsString} = join( ',', map { '"' . $_ .'"'} @{$entry->{Versions}} );
388
        $entry->{release}{dbName} = genDatabaseName($rtagid);
479
        $entry->{release}{dbName} = genDatabaseName($rtagid);
-
 
480
        $entry->{release}{root} = catdir($ReleaseStore, $rtagid );
389
        #DebugDumpData("ENTRY", \$entry);
481
        #DebugDumpData("ENTRY", \$entry);
390
 
482
 
391
        my $tfileName = 'lxr.tree.template';
483
        my $tfileName = 'lxr.tree.template';
392
        open( my $tf, '<', $tfileName) || Error ("Cannot open $tfileName. $!");
484
        open( my $tf, '<', $tfileName) || Error ("Cannot open $tfileName. $!");
393
        while (my $line = <$tf>)
485
        while (my $line = <$tf>)
Line 410... Line 502...
410
    #   Insert tree sections into the main config file template
502
    #   Insert tree sections into the main config file template
411
    #
503
    #
412
    my $hostList = join( ',', map { '\'http://' . $_ .'\''} @addressList );
504
    my $hostList = join( ',', map { '\'http://' . $_ .'\''} @addressList );
413
 
505
 
414
    my $tfileName = catfile($scriptDir, 'lxr.template');
506
    my $tfileName = catfile($scriptDir, 'lxr.template');
415
    my $lxrFileName = catfile($config->{lxr}, 'lxr.new.conf');
507
    my $lxrFileName = catfile($Config->{lxr}, 'lxr.new.conf');
416
    unlink $lxrFileName;
508
    unlink $lxrFileName;
417
    open( my $tf, '<', $tfileName) || Error ("Cannot open $tfileName. $!");
509
    open( my $tf, '<', $tfileName) || Error ("Cannot open $tfileName. $!");
418
    open( my $to, '>', $lxrFileName) || Error ("Cannot open $lxrFileName. $!");
510
    open( my $to, '>', $lxrFileName) || Error ("Cannot open $lxrFileName. $!");
419
    while (my $line = <$tf>)
511
    while (my $line = <$tf>)
420
    {
512
    {
Line 439... Line 531...
439
    close $to;
531
    close $to;
440
 
532
 
441
    #
533
    #
442
    #   Install the new config files
534
    #   Install the new config files
443
    #
535
    #
444
    my $lxrLive = catfile($config->{lxr}, 'lxr.conf');
536
    my $lxrLive = catfile($Config->{lxr}, 'lxr.conf');
445
    my $lxrBackup = catfile($config->{lxr}, 'lxr.conf.bak');
537
    my $lxrBackup = catfile($Config->{lxr}, 'lxr.conf.bak');
446
    unlink $lxrBackup;
538
    unlink $lxrBackup;
447
    rename ($lxrLive, $lxrBackup) || Warning("Renaming $lxrLive, $lxrBackup", $!);
539
    rename ($lxrLive, $lxrBackup) || Warning("Renaming $lxrLive, $lxrBackup", $!);
448
    rename ($lxrFileName, $lxrLive) || Warning("Renaming $lxrFileName, $lxrLive", $!);
540
    rename ($lxrFileName, $lxrLive) || Warning("Renaming $lxrFileName, $lxrLive", $!);
449
 
541
 
450
    #
542
    #
451
    #   Create new database tables if required
543
    #   Create new database tables if required
452
    #   Use a customized shell script to do the hard work
544
    #   Use a customized shell script to do the hard work
453
    #
545
    #
454
    foreach my $rtagid ( keys %ReleaseData )
546
    foreach my $rtagid ( sort keys %ReleaseData )
455
    {
547
    {
456
        my $entry = $ReleaseData{$rtagid};
548
        my $entry = $ReleaseData{$rtagid};
457
        Information("Database:$entry->{release}{Name}, $entry->{release}{dbName} ");
549
        Verbose("Database:$entry->{release}{Name}, $entry->{release}{dbName} ");
458
        System('--NoExit', '--NoShell', catfile($scriptDir, 'lxr.initdb.sh'), $entry->{release}{dbName});
550
        System('--NoExit', '--NoShell', catfile($scriptDir, 'lxr.initdb.sh'), $entry->{release}{dbName});
459
    }
551
    }
460
}
552
}
461
 
553
 
462
#-------------------------------------------------------------------------------
554
#-------------------------------------------------------------------------------
Line 486... Line 578...
486
{
578
{
487
    #
579
    #
488
    #   Prep envonment for calling genxref
580
    #   Prep envonment for calling genxref
489
    #   See notes above
581
    #   See notes above
490
    #
582
    #
491
    chdir ($config->{lxr}) || Error ("Cannot chnage directory:$config->{lxr}, $!");
583
    chdir ($Config->{lxr}) || Error ("Cannot chnage directory:$Config->{lxr}, $!");
492
    delete $ENV{PERL5LIB};
584
    delete $ENV{PERL5LIB};
493
 
585
 
494
    #
586
    #
495
    #   Examine each Version in each Release
587
    #   Examine each Version in each Release
496
    #   Generate command line like:
588
    #   Generate command line like:
497
    #   genxref --tree RTAGID --url http://HOSTNAME/lxr --version 'VERSION'
589
    #   genxref --tree RTAGID --url http://HOSTNAME/lxr --version 'VERSION'
498
    foreach my $rtagid ( keys %ReleaseData )
590
    foreach my $rtagid ( sort keys %ReleaseData )
499
    {
591
    {
500
        my $entry = $ReleaseData{$rtagid};
592
        my $entry = $ReleaseData{$rtagid};
501
        foreach my $version (@{$entry->{Versions}})
593
        foreach my $version (@{$entry->{Versions}})
502
        {
594
        {
503
            my $markerFile = catfile($entry->{release}{root}, $version, '.lxrIndexed');
595
            my $markerFile = catfile(catdir($ReleaseStore, $rtagid ), $version, '.lxrIndexed');
504
            unless (-f $markerFile) {
596
            unless (-f $markerFile) {
505
                Message("Must index: $rtagid, $version");
597
                Message("Must index: $rtagid, $version");
506
                my $rv = System('--NoExit', 
598
                my $rv = System('--NoExit', 
507
                       '--NoShell', 
599
                       '--NoShell', 
508
                       catfile($config->{lxr}, 'genxref'),
600
                       catfile($Config->{lxr}, 'genxref'),
509
                       '--url', 'http://' . $ENV{GBE_HOSTNAME} . '/lxr',
601
                       '--url', 'http://' . $ENV{GBE_HOSTNAME} . '/lxr',
510
                       '--tree', $rtagid,
602
                       '--tree', $rtagid,
511
                       '--version', $version
603
                       '--version', $version
512
                       );
604
                       );
513
               Verbose("genxref exit: $rv");
605
               Verbose("genxref exit: $rv");
Line 559... Line 651...
559
        next unless ( -d $vdirName );
651
        next unless ( -d $vdirName );
560
        unless(exists $ReleaseData{$rdirEntry} && $ReleaseData{$rdirEntry}{release}{ACTIVE} )
652
        unless(exists $ReleaseData{$rdirEntry} && $ReleaseData{$rdirEntry}{release}{ACTIVE} )
561
        {
653
        {
562
            #   Release is no longer configured - age it out
654
            #   Release is no longer configured - age it out
563
            #   Assume $rdirEntry is an rtag_id
655
            #   Assume $rdirEntry is an rtag_id
564
            if (processAgeMarker($vdirName, $config->{'releaseAge'} ))
656
            if (processAgeMarker($vdirName, $Config->{'releaseAge'} ))
565
            {
657
            {
566
                Message("Delete Release: $rdirEntry");
658
                Message("Delete Release: $rdirEntry");
567
                RmDirTree($vdirName);
659
                RmDirTree($vdirName);
568
                System('--NoExit', '--NoShell', catfile($scriptDir, 'lxr.dropdb.sh'), genDatabaseName($rdirEntry));
660
                System('--NoExit', '--NoShell', catfile($scriptDir, 'lxr.dropdb.sh'), genDatabaseName($rdirEntry));
569
            }
661
            }
Line 624... Line 716...
624
#
716
#
625
#                   Used to determine the most recent version
717
#                   Used to determine the most recent version
626
#
718
#
627
# Inputs          : $vdirName - Dir to process - expecting a Release directory
719
# Inputs          : $vdirName - Dir to process - expecting a Release directory
628
#
720
#
629
# Returns         : 
721
# Returns         : latestName  - Patch to the latest directory
-
 
722
#                   Age (days ) - Of the named directory
630
#
723
#
631
sub getLatestVersion
724
sub getLatestVersion
632
{
725
{
633
    my  ($vdirName) = @_;
726
    my  ($vdirName) = @_;
634
    my $latestName;
727
    my $latestName;
635
    my $latestAge = 0;
728
    my $latestAge = 0;
636
 
729
 
637
    opendir (my $vdir, $vdirName) || Warning ("Cannot open directory: $vdirName", $!);
-
 
638
    while (my $vdirEntry = readdir($vdir))
730
    if (-d $vdirName )
639
    {
731
    {
-
 
732
        opendir (my $vdir, $vdirName) || Warning ("Cannot open directory: $vdirName", $!);
-
 
733
        while (my $vdirEntry = readdir($vdir))
-
 
734
        {
640
        #   Skip hidden files and directories
735
            #   Skip hidden files and directories
641
        next if ($vdirEntry =~ m~^\.~);
736
            next if ($vdirEntry =~ m~^\.~);
642
 
737
 
643
        my $ldirName = catdir($vdirName, $vdirEntry );
738
            my $ldirName = catdir($vdirName, $vdirEntry );
644
        next unless ( -d $ldirName );
739
            next unless ( -d $ldirName );
645
 
740
 
646
        my $age = (stat $ldirName)[9];
741
            my $age = (stat $ldirName)[9];
647
        Verbose3("Age: $ldirName, $age");
742
            Verbose3("Age: $ldirName, $age");
648
 
743
 
649
        if  ($age > $latestAge )
744
            if  ($age > $latestAge )
650
        {
745
            {
651
            $latestAge = $age;
746
                $latestAge = $age;
652
            $latestName = $ldirName;
747
                $latestName = $ldirName;
-
 
748
            }
653
        }
749
        }
-
 
750
        close ($vdir);
-
 
751
        
-
 
752
        #   Convert to Days ago
-
 
753
        $latestAge = ($StampTime - $latestAge) / (60*60*24);
-
 
754
        #DebugDumpData("versionDataSorted",\@versionDataSorted);
-
 
755
        Verbose("Latest: $latestName, $latestAge");
-
 
756
    }
-
 
757
    else
-
 
758
    {
-
 
759
        Verbose("Latest: No directory found: $vdirName");
654
    }
760
    }
655
    close ($vdir);
-
 
656
 
761
 
657
    Verbose("Latest: $latestName, $latestAge");
762
    return $latestName, $latestAge;
658
    #DebugDumpData("versionDataSorted",\@versionDataSorted);
-
 
659
    return $latestName;
-
 
660
}
763
}
661
 
764
 
662
 
-
 
663
#-------------------------------------------------------------------------------
765
#-------------------------------------------------------------------------------
664
# Function        : cleanPackageStore 
766
# Function        : cleanPackageStore 
665
#
767
#
666
# Description     : Delete unused PAckages fromthe package store
768
# Description     : Delete unused PAckages fromthe package store
667
#
769
#
Line 736... Line 838...
736
        next if ($pdirEntry =~ m~^\.~);
838
        next if ($pdirEntry =~ m~^\.~);
737
        my $pdirName = catdir($PackageStore, $pdirEntry );
839
        my $pdirName = catdir($PackageStore, $pdirEntry );
738
        next unless ( -d $pdirName );
840
        next unless ( -d $pdirName );
739
        next if (exists $pkgsUsed{$pdirEntry} );
841
        next if (exists $pkgsUsed{$pdirEntry} );
740
 
842
 
741
        if (processAgeMarker($pdirName, $config->{packageAge})) 
843
        if (processAgeMarker($pdirName, $Config->{packageAge})) 
742
        {
844
        {
743
            Message("Purge Package: $pdirEntry");
845
            Message("Purge Package: $pdirEntry");
744
            RmDirTree($pdirName);
846
            RmDirTree($pdirName);
745
        }
847
        }
746
    }
848
    }
Line 764... Line 866...
764
    my ($rtagid) = @_;
866
    my ($rtagid) = @_;
765
    my (@row);
867
    my (@row);
766
    my @releaseList;
868
    my @releaseList;
767
    my $partSql = '';
869
    my $partSql = '';
768
 
870
 
769
    Information("getReleaseData");
871
    Verbose("getReleaseData");
770
    connectRM(\$RM_DB) unless $RM_DB;
872
    connectRM(\$RM_DB) unless $RM_DB;
771
 
873
 
772
    #
874
    #
773
    #   Determine list of existing Releases
875
    #   Determine list of existing Releases
774
    #   Build up a Clause for the extraction SQL
876
    #   Build up a Clause for the extraction SQL
Line 842... Line 944...
842
                        {
944
                        {
843
                            $data->{ACTIVE} = 2;
945
                            $data->{ACTIVE} = 2;
844
                        }
946
                        }
845
                    }
947
                    }
846
 
948
 
847
                    $data->{root} = catdir($ReleaseStore, $rtagid );
-
 
848
                    $ReleaseData{$rtagid}{release} = $data;
949
                    $ReleaseData{$rtagid}{release} = $data;
849
                }
950
                }
850
            }
951
            }
851
            $sth->finish();
952
            $sth->finish();
852
        }
953
        }
Line 854... Line 955...
854
    else
955
    else
855
    {
956
    {
856
        Error("getReleaseData:Prepare failure" );
957
        Error("getReleaseData:Prepare failure" );
857
    }
958
    }
858
 
959
 
-
 
960
    if (IsVerbose(1))
-
 
961
    {
-
 
962
        DebugDumpData("ReleaseData", \%ReleaseData);
-
 
963
    }
859
 
964
 
-
 
965
    #
860
    DebugDumpData("ReleaseData", \%ReleaseData);
966
    #   Just a summary display for logging
-
 
967
    #
861
    foreach my $rtagid ( keys %ReleaseData)
968
    foreach my $rtagid ( sort keys %ReleaseData)
862
    {
969
    {
863
        next unless $ReleaseData{$rtagid}{release}{ACTIVE};
970
        my $state = $ReleaseData{$rtagid}{release}{ACTIVE} ? 'ACTIVE' : 'InActive';
864
        Information("Active Release: RtagId $rtagid");
971
        Information("Release: RtagId $rtagid, $state");
865
        getOneRelease($rtagid);
-
 
866
    }
972
    }
867
}
973
}
868
 
974
 
869
#-------------------------------------------------------------------------------
975
#-------------------------------------------------------------------------------
870
# Function        : getOneRelease
976
# Function        : getReleasePakageData
871
#
977
#
872
# Description     : Get PackgeVersion information for one Release
978
# Description     : Get PackgeVersion information for one Release
873
#
979
#
874
# Inputs          : rtagid
980
# Inputs          : rtagid
875
#
981
#
876
# Returns         : 
982
# Returns         : 
877
#
983
#
878
sub getOneRelease
984
sub getReleasePakageData
879
{
985
{
880
    my ($rtagid) = @_;
986
    my ($rtagid) = @_;
881
    my (@row);
987
    my (@row);
882
 
988
 
883
    connectRM(\$RM_DB) unless $RM_DB;
989
    connectRM(\$RM_DB) unless $RM_DB;
Line 914... Line 1020...
914
            $sth->finish();
1020
            $sth->finish();
915
        }
1021
        }
916
    }
1022
    }
917
    else
1023
    else
918
    {
1024
    {
919
        Error("getOneRelease:Prepare failure" );
1025
        Error("getReleasePakageData:Prepare failure" );
920
    }
1026
    }
921
}
1027
}
922
 
1028
 
923
#-------------------------------------------------------------------------------
1029
#-------------------------------------------------------------------------------
924
# Function        : genDatabaseName 
1030
# Function        : genDatabaseName 
Line 952... Line 1058...
952
    if ($opt_config)
1058
    if ($opt_config)
953
    {
1059
    {
954
        Message ("Using alternate config: $opt_config");
1060
        Message ("Using alternate config: $opt_config");
955
        $cfile = $opt_config;
1061
        $cfile = $opt_config;
956
    }
1062
    }
957
    Error ("Expected config file not found: $cfile") unless ( -f $cfile );
-
 
958
    require $cfile;
-
 
959
     
-
 
960
#   #
-
 
961
#   #   Create data
-
 
962
#   #
-
 
963
#   my %hash = (
-
 
964
#   number => 42,
-
 
965
#   string => 'This is a string',
-
 
966
#   array  => [ 1 .. 10 ],
-
 
967
#   hash   => { apple => 'red', banana => 'yellow' },
-
 
968
#   );
-
 
969
#
-
 
970
#   # Print structure to file
-
 
971
#   open (my $out, '>', $cfile) || Error("Can't create $cfile. $!");
-
 
972
#   DebugDumpData("Hash", \%hash);
-
 
973
#   print {$out} Data::Dumper->Dump([\%hash], ['$config']);
-
 
974
#   close $out;
-
 
975
 
1063
 
-
 
1064
    #
-
 
1065
    #   Slurp in the file and evaluate it as a perl expression
-
 
1066
    #
-
 
1067
    if (open(my $CONFIG, '<', $cfile))
-
 
1068
    {
-
 
1069
        local ($/) = undef;
-
 
1070
        my $config_contents = <$CONFIG>;
-
 
1071
        $config_contents =~ m/(.*)/s;
-
 
1072
        $config_contents = $1;    #untaint it
-
 
1073
        my $config = eval("\n#line 1 \"configuration file\"\n" . $config_contents);
-
 
1074
        Error($@) if $@;
-
 
1075
        close $CONFIG;
-
 
1076
 
-
 
1077
        #
-
 
1078
        #   Merge read data with defaults
-
 
1079
        #
976
    #DebugDumpData("config", \$config);
1080
        @$Config{ keys %$config } = values %$config;
-
 
1081
    }
-
 
1082
    else
-
 
1083
    {
-
 
1084
        Error("Couldn't open configuration file \"$cfile\".", $!);
-
 
1085
    }
977
 
1086
 
-
 
1087
    if (IsVerbose(1))
-
 
1088
    {
-
 
1089
        DebugDumpData("Config", \$Config);
-
 
1090
    }
978
}
1091
}
979
 
1092
 
980
#-------------------------------------------------------------------------------
1093
#-------------------------------------------------------------------------------
981
# Function        : startLogFile 
1094
# Function        : startLogFile 
982
#
1095
#
Line 989... Line 1102...
989
#
1102
#
990
sub startLogFile
1103
sub startLogFile
991
{
1104
{
992
    if ( $opt_logfile )
1105
    if ( $opt_logfile )
993
    {
1106
    {
994
        if (exists $config->{lxrLogDir})
1107
        if (exists $Config->{lxrLogDir})
995
        {
1108
        {
996
            my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime($stamptime);
1109
            my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime($StampTime);
997
            my $name = sprintf("jats-lxr-%4.4d%2.2d%2.2d.%2.2d%2.2d%2.2d.log", 1900+$year, 1+$mon, $mday, $hour, $min, $sec);
1110
            my $name = sprintf("jats-lxr-%4.4d%2.2d%2.2d.%2.2d%2.2d%2.2d.log", 1900+$year, 1+$mon, $mday, $hour, $min, $sec);
998
            $name = catdir($config->{lxrLogDir}, $name);
1111
            $name = catdir($Config->{lxrLogDir}, $name);
999
 
1112
 
1000
            mkpath($config->{lxrLogDir}); 
1113
            mkpath($Config->{lxrLogDir}); 
1001
            if (-d $config->{lxrLogDir} )
1114
            if (-d $Config->{lxrLogDir} )
1002
            {
1115
            {
1003
                open STDOUT, '>', $name  or die "Can't redirect STDOUT: $!";
1116
                open STDOUT, '>', $name  or die "Can't redirect STDOUT: $!";
1004
                open STDERR, ">&STDOUT"  or die "Can't dup STDOUT: $!";
1117
                open STDERR, ">&STDOUT"  or die "Can't dup STDOUT: $!";
1005
            }
1118
            }
1006
            else
1119
            else
1007
            {
1120
            {
1008
                Warning("Can't create log dir: $config->{lxrLogDir}. $!");
1121
                Warning("Can't create log dir: $Config->{lxrLogDir}. $!");
1009
            }
1122
            }
1010
        }
1123
        }
1011
    }
1124
    }
1012
}
1125
}
1013
 
1126
 
Line 1020... Line 1133...
1020
#
1133
#
1021
# Returns         : 
1134
# Returns         : 
1022
#
1135
#
1023
sub cleanupLogFiles
1136
sub cleanupLogFiles
1024
{
1137
{
1025
    if (exists $config->{lxrLogDir} && exists $config->{logAge}  && $config->{logAge} > 0 )
1138
    if (exists $Config->{lxrLogDir} && exists $Config->{logAge}  && $Config->{logAge} > 0 )
1026
    {
1139
    {
1027
        Verbose("cleanupLogFiles:$config->{lxrLogDir}, $config->{logAge}");
1140
        Verbose("cleanupLogFiles:$Config->{lxrLogDir}, $Config->{logAge}");
1028
        if ( opendir my $logDir, $config->{lxrLogDir} )
1141
        if ( opendir my $logDir, $Config->{lxrLogDir} )
1029
        {
1142
        {
1030
            foreach my $fileName (readdir $logDir)
1143
            foreach my $fileName (readdir $logDir)
1031
            {
1144
            {
1032
                my $file = catfile($config->{lxrLogDir}, $fileName);
1145
                my $file = catfile($Config->{lxrLogDir}, $fileName);
1033
                next unless -f $file;
1146
                next unless -f $file;
1034
                next unless -M $file > $config->{logAge};
1147
                next unless -M $file > $Config->{logAge};
1035
                Verbose("Purge logfile: $fileName");
1148
                Verbose("Purge logfile: $fileName");
1036
                unlink $file;
1149
                unlink $file;
1037
            }
1150
            }
1038
            closedir $logDir;
1151
            closedir $logDir;
1039
        }
1152
        }
Line 1074... Line 1187...
1074
    {
1187
    {
1075
        TouchFile($markerfile);
1188
        TouchFile($markerfile);
1076
    }
1189
    }
1077
    else
1190
    else
1078
    {
1191
    {
-
 
1192
        my $fileAge = -M $markerfile;
-
 
1193
        Verbose ("Age: $fileAge, $tdir");
1079
        if (-M $markerfile > $age)
1194
        if ($fileAge > $age)
1080
        {
1195
        {
1081
            return 1
1196
            return 1
1082
        }
1197
        }
1083
    }
1198
    }
1084
    return 0;
1199
    return 0;