Subversion Repositories DevTools

Rev

Go to most recent revision | Details | Last modification | View Log | RSS feed

Rev Author Line No. Line
392 dpurdie 1
########################################################################
5709 dpurdie 2
# Copyright (c) VIX TECHNOLOGY (AUST) LTD
392 dpurdie 3
#
4
# Module name   : cc2svn_gendata.pl
5
# Module type   : Makefile system
6
# Compiler(s)   : Perl
7
# Environment(s): jats
8
#
9
# Description   : Get all packages that are used in all releases
10
#                 Create a data file that can be used offline
11
#
12
#                 The process will exclude some old releases
13
#
14
#                 Generate data on Essential Package Versions to be
15
#                 transferred from CC to Subversion
16
#
17
#......................................................................#
18
 
19
require 5.006_001;
20
use strict;
21
use warnings;
22
use JatsError;
23
use JatsSystem;
24
use Getopt::Long;
25
use Pod::Usage;                             # required for help support
26
use JatsRmApi;
27
use ConfigurationFile;
28
 
29
use DBI;
30
use HTTP::Date;
31
 
32
my $VERSION = "1.2.3";                      # Update this
33
my $opt_verbose = 0;
34
my $opt_help = 0;
35
my $opt_manual;
36
my $opt_test;
37
my $opt_limit;
38
my $opt_quick;
1197 dpurdie 39
my $opt_mode = '';
392 dpurdie 40
my $RM_DB;
41
my $now = time();
42
 
43
#
44
#   Package information
45
#
46
my %Releases;
47
my %Packages;
48
my %Suffixes;
49
my @StrayPackages;
50
my %AllPackages;
51
 
52
my $doAllReleases = 0;
53
my $doIncludeOnly = 0;
54
my @includedProjects = (
55
#        481,    # UK BUS HOPS
56
);
57
 
58
my @includedReleases = (
59
        6222,   # HOME > UK STAGE COACH (SSW) > Mainline
60
        14503,  # HOME > UK STAGE COACH (SSW) > ITSO_HOPS_3
61
        21303,  # HOME > UK STAGE COACH (SSW) > SUPPORT_HOPS_REPORTS
62
        21343,  # HOME > UK STAGE COACH (SSW) > SUPPORT_CIPP
63
        17223,  # HOME > UK STAGE COACH (SSW) > ITSO HOPS 4
1453 dpurdie 64
        21864,  # Hops3.6'
65
        22303   # Hops3.7
392 dpurdie 66
);
67
 
68
 
69
my @excludeProjects = ( 162,            # WASHINGTON (WDC)
70
                        341,            # TUTORIAL (TUT)
71
                        142,            # SYDNEY (SYD)
72
                        182 ,           # ROME (ROM)
73
                        6 ,             # GMPTE/PCL (GMP)
74
                        521,            # NSW CLUB CARD
75
                        221,            # NZ STAGE COACH (NZS)
1197 dpurdie 76
                        82,             # LVS
77
                        42,             # SFO
2478 dpurdie 78
#                        641,            # BCC Releaeses
1197 dpurdie 79
                        62,             # OSLO
80
                        4,              # Singapore
81
                        441,            # Tas
82
                        102,            # Ventura
392 dpurdie 83
                        );
84
my @excludeReleases = ( 20424,          # MASS_REF (MAS) > test
1197 dpurdie 85
                        # RJACK 9043,           # TECHNOLOGY GROUP > Development Environment - For Test Setup
86
                        # RJACK 14383,          # TECHNOLOGY GROUP > eBrio TDS
87
                        # RJACK 20463,          # TECHNOLOGY GROUP > TPIT - BackOffice Linux build
88
                        # RJACK 14603,          # TECHNOLOGY GROUP > TPIT - BackOffice 64 bit [CCB Mode!]
392 dpurdie 89
                        #9263,           # TECHNOLOGY GROUP > Buildtool DEVI&TEST
90
                        22163,          # GLOBAL PRODUCT MGMT > Rio Tinto - Remote Draught Survey
91
                        19483,          # SEATTLE (SEA) > Phase 2 - I18 [backup] [Restrictive Mode]
92
                        20403,          # SEATTLE (SEA) > Phase 2 - I19 [backup]
93
                        20983,          # ??? May have been deleted
395 dpurdie 94
                        13083,          # TECHNOLOGY GROUP > TRACS
95
                        15224,          # 64Bit Solaris Test
1197 dpurdie 96
 
392 dpurdie 97
                        );
98
 
99
my %sillyVersions =
100
(
101
    '2b6'           => '2.6.0.cots',
102
    '1.0b2'         => '1.0.2.cots',
103
    '1.6.x'         => '1.6.0.cots',
104
    '3.5beta12.5'   => '3.5.12.5.cots',
1197 dpurdie 105
    '1.0b1.1.mas'   => '1.1.1.mas',
392 dpurdie 106
);
1197 dpurdie 107
 
108
my %suffixFixup = (
109
    '.sf'           => '.sfo',
110
    '.vt'           => '.vtk',
111
    '.lv'           => '.lvs',
112
    '.was'          => '.wdc',
113
    '.uk.1'         => '.uk',
114
    '.ssts.demo'    => '.ssts',
115
    '.u244.syd'     => '.syd',
116
    '.pxxx.sea'     => '.sea',
117
    '.pxxx.syd'     => '.syd',
118
    '.pxxx.sydddd'  => '.syd',
119
    '.oslo'         => '.oso',
120
);
392 dpurdie 121
 
122
#-------------------------------------------------------------------------------
123
# Function        : Main Entry
124
#
125
# Description     :
126
#
127
# Inputs          :
128
#
129
# Returns         :
130
#
131
my $result = GetOptions (
132
                "help+"         => \$opt_help,          # flag, multiple use allowed
133
                "manual"        => \$opt_manual,        # flag
134
                "verbose+"      => \$opt_verbose,       # flag
135
                "test:s"        => \$opt_test,          # Test a version string
136
                "limit:n"       => \$opt_limit,         #
137
                "quick"         => \$opt_quick,         # Don't look for indirects
1197 dpurdie 138
                'mode:s'        => \$opt_mode,          # Mode of operation
392 dpurdie 139
                );
140
 
141
#
142
#   Process help and manual options
143
#
144
pod2usage(-verbose => 0, -message => "Version: $VERSION")  if ($opt_help == 1  || ! $result);
145
pod2usage(-verbose => 1)  if ($opt_help == 2 );
146
pod2usage(-verbose => 2)  if ($opt_manual || ($opt_help > 2));
1197 dpurdie 147
ErrorConfig( 'name'    =>'CC2SVN_GENDATA' );
392 dpurdie 148
 
149
if ( $opt_test )
150
{
151
    my @results = massageVersion( $opt_test, 'DummyName' );
152
    Message ("Version", $opt_test, @results);
153
    exit 1;
154
}
155
 
1197 dpurdie 156
#
157
#   Set up the mode
158
#   Must be specified
159
#
160
if ( $opt_mode eq 'all' ) {
161
    $doAllReleases = 1;
162
    $doIncludeOnly = 0;
163
 
164
} elsif ( $opt_mode eq 'hops' ) {
165
    $doAllReleases = 0;
166
    $doIncludeOnly = 1;
392 dpurdie 167
 
1197 dpurdie 168
} elsif ( $opt_mode eq 'standard' ) {
169
    $doAllReleases = 0;
170
    $doIncludeOnly = 0;
171
 
172
} else {
173
    Error ("Mode not specified: all, hops, standard");
174
}
175
 
392 dpurdie 176
GetAllPackageNames();
2764 dpurdie 177
getPackagesInSvn();
392 dpurdie 178
getReleaseDetails();
179
getPkgDetailsByRTAG_ID();
1197 dpurdie 180
my ($pcount, $vcount) = countPackages();
181
print "Directly referenced Packages: $pcount Versions: $vcount\n";
392 dpurdie 182
LocateStrays() unless ($opt_quick);
1197 dpurdie 183
($pcount, $vcount) = countPackages();
184
print "Indirectly referenced Packages: $pcount Versions: $vcount\n";
2764 dpurdie 185
getSourcePaths();
1197 dpurdie 186
processData();
392 dpurdie 187
outputData();
188
 
189
if ( $opt_verbose > 1 )
190
{
191
    print "=========================================================================\n";
192
    DebugDumpData("Releases", \%Releases);
193
    print "=========================================================================\n";
194
    DebugDumpData("Packages", \%Packages );
195
    print "=========================================================================\n";
196
    DebugDumpData("Suffixes", \%Suffixes );
197
}
198
 
1197 dpurdie 199
($pcount, $vcount) = countPackages();
200
print "Total References Packages: $pcount Versions: $vcount\n";
392 dpurdie 201
exit;
202
 
203
#-------------------------------------------------------------------------------
2764 dpurdie 204
# Function        : getPackagesInSvn
205
#
206
# Description     : Determine packages that have been migrated
207
#                   Done by looking up RM for any version that has been migrated
208
#
209
# Inputs          : 
210
#
211
# Returns         : 
212
#
213
sub getPackagesInSvn
214
{
215
    my (@row);
216
 
217
    # if we are not or cannot connect then return 0 as we have not found anything
218
    connectRM(\$RM_DB) unless $RM_DB;
219
 
220
    # First get all packages that are referenced in a Release
221
    # This will only get the top level packages
222
    # From non-archived releases
223
 
224
    my $m_sqlstr = "SELECT UNIQUE" .
225
                        " pv.PKG_ID ".                                         #[0]
226
                   " FROM RELEASE_MANAGER.PACKAGE_VERSIONS pv".
227
                   " WHERE pv.VCS_TYPE_ID=23";
228
    my $sth = $RM_DB->prepare($m_sqlstr);
229
    if ( defined($sth) )
230
    {
231
        if ( $sth->execute( ) )
232
        {
233
#            print "--- Execute\n";
234
            if ( $sth->rows )
235
            {
236
                #print "--- Execute ROWS\n";
237
                while ( @row = $sth->fetchrow_array )
238
                {
239
                    my $id = $row[0];
240
                    next unless ( $id );
241
                    $AllPackages{$id}{inSvn} = 1;
242
#print "Data: @row\n";
243
                }
244
            }
245
#            print "--- Finish\n";
246
            $sth->finish();
247
        }
248
        else
249
        {
250
            Error("Execute failure: $m_sqlstr", $sth->errstr() );
251
        }
252
    }
253
    else
254
    {
255
        Error("Prepare failure" );
256
    }
257
 
258
}
259
 
260
#-------------------------------------------------------------------------------
392 dpurdie 261
# Function        : getReleaseDetails
262
#
263
# Description     : Determine all candiate releases
264
#
265
# Inputs          : 
266
#
267
# Returns         : 
268
#
269
sub getReleaseDetails
270
{
271
    my (@row);
272
 
273
    # if we are not or cannot connect then return 0 as we have not found anything
274
    connectRM(\$RM_DB) unless $RM_DB;
275
 
276
    # First get all packages that are referenced in a Release
277
    # This will only get the top level packages
278
    # From non-archived releases
279
 
280
    my $m_sqlstr = "SELECT prj.PROJ_NAME, rt.RTAG_NAME, rt.PROJ_ID, rt.RTAG_ID, rt.official" .
281
                   " FROM release_manager.release_tags rt, release_manager.projects prj" .
282
                   " WHERE prj.PROJ_ID = rt.PROJ_ID " .
1197 dpurdie 283
#                   "   AND rt.official != 'A' ".
284
#                   "   AND rt.official != 'Y'" .
392 dpurdie 285
                   " order by prj.PROJ_NAME";
286
    my $sth = $RM_DB->prepare($m_sqlstr);
287
    if ( defined($sth) )
288
    {
289
        if ( $sth->execute( ) )
290
        {
291
#            print "--- Execute\n";
292
            if ( $sth->rows )
293
            {
294
#                print "--- Execute ROWS\n";
295
                while ( @row = $sth->fetchrow_array )
296
                {
297
                    my $rtag_id =$row[3];
298
                    my $proj_id = $row[2];
299
 
300
                    $Releases{$rtag_id}{pName} = $row[0];
301
                    $Releases{$rtag_id}{name} = $row[1];
302
                    $Releases{$rtag_id}{proj_id} = $proj_id;
303
                    $Releases{$rtag_id}{rtag_id} = $rtag_id;
304
                    $Releases{$rtag_id}{official} = $row[4];
305
 
306
                    unless ( $doAllReleases )
307
                    {
308
                        if (grep {$_ eq $proj_id} @excludeProjects) {
309
                            $Releases{$rtag_id}{excluded} = 'E';
310
                        }
311
 
312
                        if (grep {$_ eq $rtag_id} @excludeReleases) {
313
                            $Releases{$rtag_id}{excluded} = 'E';
314
                        }
315
                    }
316
 
317
                    if ( $doIncludeOnly )
318
                    {
319
 
320
                        if (grep {$_ eq $proj_id} @includedProjects)
321
                        {
322
                            delete $Releases{$rtag_id}{excluded};
323
                        }
324
                        else
325
                        {
326
                            $Releases{$rtag_id}{excluded} = 'E';
327
                        }
328
 
329
                        if (grep {$_ eq $rtag_id} @includedReleases)
330
                        {
331
                            delete $Releases{$rtag_id}{excluded};
332
                        }
333
                    }
334
 
335
                    unshift @row, $Releases{$rtag_id}{excluded} || ' ';
336
                    print join (',',@row), "\n" if ($opt_verbose);
337
                }
338
            }
339
#            print "--- Finish\n";
340
            $sth->finish();
341
        }
342
        else
343
        {
344
            Error("Execute failure: $m_sqlstr", $sth->errstr() );
345
        }
346
    }
347
    else
348
    {
349
        Error("Prepare failure" );
350
    }
351
}
352
 
353
 
1197 dpurdie 354
sub getPkgDetailsByPVID
355
{
2764 dpurdie 356
    my (@plist) = @_;
1197 dpurdie 357
    my (@row);
358
 
359
    #
360
    #   Only do once
361
    #
2764 dpurdie 362
#    return if ( exists $Packages{$pv_id}{name} );
363
 
1197 dpurdie 364
    # if we are not or cannot connect then return 0 as we have not found anything
365
    connectRM(\$RM_DB) unless $RM_DB;
366
 
367
    my $m_sqlstr = "SELECT" .
368
                        " pv.PV_ID, ".                                          #[0]
369
                        " pkg.PKG_NAME, ".                                      #[1]
370
                        " pv.PKG_VERSION, ".                                    #[2]
371
                        " pv.DLOCKED," .                                        #[3]
372
                        " release_manager.PK_RMAPI.return_vcs_tag(pv.PV_ID), ". #[4]
373
                        " pv.PKG_ID," .                                         #[5]
2429 dpurdie 374
                        " pv.MODIFIED_STAMP, ".                                 #[6]
375
                        " pv.CREATOR_ID  ".                                     #[7]
1197 dpurdie 376
                   " FROM RELEASE_MANAGER.PACKAGE_VERSIONS pv,".
377
                   "      RELEASE_MANAGER.PACKAGES pkg ".
2764 dpurdie 378
                   " WHERE pv.PKG_ID = pkg.PKG_ID ".
379
                   "   AND pv.PV_ID in ( " . join(',', @plist) . "  )" ;
1197 dpurdie 380
    my $sth = $RM_DB->prepare($m_sqlstr);
381
    if ( defined($sth) )
382
    {
383
        if ( $sth->execute( ) )
384
        {
385
#            print "--- Execute\n";
386
            if ( $sth->rows )
387
            {
388
#                print "--- Execute ROWS\n";
389
                while ( @row = $sth->fetchrow_array )
390
                {
391
                    print join (',',@row), "\n" if ($opt_verbose);
392
 
393
                    my $pvid = $row[0];
2764 dpurdie 394
                    my $pkgid = $row[5];
1197 dpurdie 395
                    $Packages{$pvid}{name} = $row[1];
2764 dpurdie 396
                    next if ( exists $Packages{$pvid}{version} );
397
 
1197 dpurdie 398
                    $Packages{$pvid}{version} = $row[2];
399
                    $Packages{$pvid}{locked} = $row[3];
400
                    $row[4] =~ tr~\\/~/~;
401
                    $Packages{$pvid}{vcstag} = $row[4];
2764 dpurdie 402
                    $Packages{$pvid}{pkgid} = $pkgid;
1197 dpurdie 403
#                    $Packages{$pvid}{tlp} = 1;
404
                    ($Packages{$pvid}{suffix}, $Packages{$pvid}{fullVersion},$Packages{$pvid}{isaRipple} ) = massageVersion( $Packages{$pvid}{version}, $Packages{$pvid}{name} );
405
                    $Suffixes{$Packages{$pvid}{suffix}}++;
406
                    $Packages{$pvid}{Age} = ($now - str2time( $row[6] )) / (60 * 60 * 24);
2429 dpurdie 407
                    $Packages{$pvid}{Creator} = $row[7];
2764 dpurdie 408
 
409
                    $AllPackages{$pkgid}{essential} = 1;
1197 dpurdie 410
                }
411
            }
412
#            print "--- Finish\n";
413
            $sth->finish();
414
        }
415
        else
416
        {
417
            Error("getPkgDetailsByPVID:Execute failure: $m_sqlstr", $sth->errstr() );
418
        }
419
    }
420
    else
421
    {
422
        Error("getPkgDetailsByPVID:Prepare failure" );
423
    }
424
}
425
 
426
 
392 dpurdie 427
sub getPkgDetailsByRTAG_ID
428
{
429
    my (@row);
430
    my $excludes = '';
431
    my $count = 0;
432
 
433
    # if we are not or cannot connect then return 0 as we have not found anything
434
    connectRM(\$RM_DB) unless $RM_DB;
435
 
436
    Message ("Extract toplevel dependencies");
437
 
438
    # First get all packages that are referenced in a Release
439
    # This will only get the top level packages
440
    # From non-archived releases
441
 
442
    unless ($doAllReleases)
443
    {
444
        foreach  ( @excludeProjects )
445
        {
446
            $excludes .= " AND prj.PROJ_ID != $_ ";
447
        }
448
        foreach  ( @excludeReleases )
449
        {
450
            $excludes .= " AND rt.RTAG_ID != $_ ";
451
        }
452
    }
453
 
1454 dpurdie 454
    my $m_sqlstr = "SELECT DISTINCT " .
455
                        "pv.PV_ID, " .                                          #[0]
456
                        "pkg.PKG_NAME, " .                                      #[1]
457
                        "pv.PKG_VERSION, " .                                    #[2]
458
                        "pv.DLOCKED, " .                                        #[3]
459
                        "release_manager.PK_RMAPI.return_vcs_tag(pv.PV_ID), " . #[4]
2016 dpurdie 460
                        "pv.PKG_ID," .                                          #[5]
1454 dpurdie 461
                        "rt.RTAG_ID, " .                                        #[6]
462
                        "rmv.VIEW_NAME, " .                                     #[7]
463
                        "pv.MODIFIED_STAMP, " .                                 #[8]
2429 dpurdie 464
                        "prj.PROJ_ID, " .                                       #[9]
465
                        "pv.CREATOR_ID " .                                     #[10]
392 dpurdie 466
                   " FROM RELEASE_MANAGER.RELEASE_CONTENT rc, RELEASE_MANAGER.PACKAGE_VERSIONS pv,".
467
                   "      RELEASE_MANAGER.PACKAGES pkg, release_manager.release_tags rt, release_manager.projects prj" .
468
                   "    , release_manager.views rmv" .
469
                   " WHERE rc.PV_ID = pv.PV_ID AND pv.PKG_ID = pkg.PKG_ID" .
470
                   "   AND rmv.VIEW_ID = rc.BASE_VIEW_ID" .
471
                   "   AND prj.PROJ_ID = rt.PROJ_ID and rt.RTAG_ID = rc.RTAG_ID" .
1197 dpurdie 472
#                   "   AND rt.official != 'A'" .
473
#                   "   AND rt.official != 'Y' " .
392 dpurdie 474
                   $excludes .
475
                   " order by pkg.PKG_NAME";
476
    my $sth = $RM_DB->prepare($m_sqlstr);
477
    if ( defined($sth) )
478
    {
479
        if ( $sth->execute( ) )
480
        {
481
#            print "--- Execute\n";
482
            if ( $sth->rows )
483
            {
484
#                print "--- Execute ROWS\n";
485
                while ( @row = $sth->fetchrow_array )
486
                {
2764 dpurdie 487
                    $count++;
392 dpurdie 488
                    print join (',',@row), "\n" if ($opt_verbose);
489
                    my $pvid = $row[0];
490
                    unless ( exists $Packages{$pvid}{name} )
491
                    {
2764 dpurdie 492
                        my $pkgid = $row[5];
392 dpurdie 493
                        $Packages{$pvid}{name} = $row[1];
494
                        $Packages{$pvid}{version} = $row[2];
495
                        $Packages{$pvid}{locked} = $row[3];
496
                        $row[4] =~ tr~\\/~/~;
497
                        $Packages{$pvid}{vcstag} = $row[4];
2764 dpurdie 498
                        $Packages{$pvid}{pkgid} = $pkgid;
392 dpurdie 499
                        $Packages{$pvid}{tlp} = 1;
500
                        ($Packages{$pvid}{suffix}, $Packages{$pvid}{fullVersion},$Packages{$pvid}{isaRipple} ) = massageVersion( $Packages{$pvid}{version}, $Packages{$pvid}{name} );
501
                        $Suffixes{$Packages{$pvid}{suffix}}++;
2764 dpurdie 502
                        $AllPackages{$pkgid}{essential} = 1;
392 dpurdie 503
 
504
                        push @StrayPackages, $pvid;
505
                    }
506
 
507
                    my $rtag_id = $row[6];
508
                    push @{$Packages{$pvid}{release}}, $rtag_id;
509
                    $Packages{$pvid}{view}{$row[7]}++ if ( $row[7] );
510
 
511
                    $Packages{$pvid}{Age} = ($now - str2time( $row[8] )) / (60 * 60 * 24);
2429 dpurdie 512
                    $Packages{$pvid}{Creator} = $row[10];
392 dpurdie 513
 
514
                    my $proj_id = $row[9];
1197 dpurdie 515
                    push @{$Packages{$pvid}{projects}}, $proj_id
516
                        unless (grep {$_ eq $proj_id} @{$Packages{$pvid}{projects}});
392 dpurdie 517
 
518
                    if ( $doIncludeOnly )
519
                    {
520
                        if (grep {$_ eq $proj_id} @includedProjects)
521
                        {
522
                            $Packages{$pvid}{NamedProject} = 1;
523
                        }
524
                        if (grep {$_ eq $rtag_id} @includedReleases)
525
                        {
1197 dpurdie 526
                            $Packages{$pvid}{NamedProject} = 2;
392 dpurdie 527
                        }
528
                    }
529
                    else
530
                    {
1197 dpurdie 531
                        $Packages{$pvid}{NamedProject} = 3;
392 dpurdie 532
                    }
533
 
534
 
535
                    if ( $opt_limit )
536
                    {
2764 dpurdie 537
                        last if ( $count > $opt_limit );
392 dpurdie 538
                    }
539
                }
540
            }
541
#            print "--- Finish\n";
542
            $sth->finish();
543
        }
544
        else
545
        {
546
            Error("Execute failure: $m_sqlstr", $sth->errstr() );
547
        }
548
    }
549
    else
550
    {
551
        Error("Prepare failure" );
552
    }
2764 dpurdie 553
 
554
    Message ("Extract toplevel dependencies: $count rows");
392 dpurdie 555
}
556
 
557
#-------------------------------------------------------------------------------
558
# Function        : GetDepends
559
#
560
# Description     :
561
#
2764 dpurdie 562
# Inputs          : @plist          - list of pvid's to process
392 dpurdie 563
#
564
# Returns         :
565
#
566
sub GetDepends
567
{
2764 dpurdie 568
    my (@plist) = @_;
392 dpurdie 569
 
570
    #
1197 dpurdie 571
    #   Ensure we have package information
572
    #
2764 dpurdie 573
    getPkgDetailsByPVID( @plist );
574
#    return if ( $Packages{$pv_id}{depend} );
575
#    $Packages{$pv_id}{depend} = 1;
1197 dpurdie 576
 
577
    #
392 dpurdie 578
    #   Now extract the package dependacies
1197 dpurdie 579
    #   There may not be any
392 dpurdie 580
    #
1197 dpurdie 581
    my $m_sqlstr = "SELECT ".
582
                    " pd.PV_ID, ".
583
                    " pd.DPV_ID " .
584
                  " FROM    RELEASE_MANAGER.PACKAGE_DEPENDENCIES pd ".
2764 dpurdie 585
                  " WHERE pd.PV_ID in ( " . join(',', @plist) . " )";
392 dpurdie 586
    my $sth = $RM_DB->prepare($m_sqlstr);
587
    if ( defined($sth) )
588
    {
589
        if ( $sth->execute( ) )
590
        {
591
            if ( $sth->rows )
592
            {
593
                while ( my @row = $sth->fetchrow_array )
594
                {
595
                    my $pvid = $row[0];
1197 dpurdie 596
                    my $dpvid = $row[1];
597
                    push @StrayPackages, $dpvid;
598
                    push @{$Packages{$dpvid}{usedBy}}, $pvid;
392 dpurdie 599
                }
600
            }
601
            $sth->finish();
602
        }
603
        else
604
        {
1197 dpurdie 605
            Error("GetDepends:Execute failure: $m_sqlstr", $sth->errstr() );
392 dpurdie 606
        }
607
    }
608
    else
609
    {
610
        Error("GetDepends:Prepare failure" );
611
    }
612
}
613
 
614
#-------------------------------------------------------------------------------
615
# Function        : GetAllPackageNames
616
#
617
# Description     :
618
#
619
# Inputs          : None
620
#
621
# Returns         :
622
#
623
sub GetAllPackageNames
624
{
625
    # if we are not or cannot connect then return 0 as we have not found anything
626
    connectRM(\$RM_DB) unless $RM_DB;
627
 
628
    #
629
    #   Now extract all the package names
630
    #
631
    my $m_sqlstr = "SELECT pkg.PKG_ID, pkg.PKG_NAME" .
632
                  " FROM RELEASE_MANAGER.PACKAGES pkg";
633
    my $sth = $RM_DB->prepare($m_sqlstr);
634
    if ( defined($sth) )
635
    {
636
        if ( $sth->execute( ) )
637
        {
638
            if ( $sth->rows )
639
            {
640
                while ( my @row = $sth->fetchrow_array )
641
                {
642
                    my $id = $row[0];
643
                    my $name = $row[1];
644
                    next unless ( $id );
2764 dpurdie 645
                    $AllPackages{$id}{name} = $name;
392 dpurdie 646
                }
647
            }
648
            $sth->finish();
649
        }
650
        else
651
        {
2764 dpurdie 652
            Error("GetAllPackageNames:Execute failure: $m_sqlstr", $sth->errstr() );
392 dpurdie 653
        }
654
    }
655
    else
656
    {
657
        Error("GetAllPackageNames:Prepare failure" );
658
    }
659
}
660
 
2764 dpurdie 661
#-------------------------------------------------------------------------------
662
# Function        : getSourcePaths
663
#
664
# Description     : Get source paths for all packages
665
#                   Really only want those not in SVN
666
#
667
# Inputs          : Globals
668
#
669
# Returns         : Globals
670
#
671
sub getSourcePaths
672
{
673
    Message("Get source paths for all packages");
674
    # if we are not or cannot connect then return 0 as we have not found anything
675
    connectRM(\$RM_DB) unless $RM_DB;
392 dpurdie 676
 
2764 dpurdie 677
    #
678
    #   Process on blocks for speed
679
    #
680
    my @needed;
681
    foreach my $pkgid ( keys %AllPackages, 0 )
682
    {
683
        if ( $pkgid )
684
        {
685
            next unless ( exists $AllPackages{$pkgid}{essential} );
686
            next if ( $AllPackages{$pkgid}{inSvn} );
687
            $AllPackages{$pkgid}{essential} = 2;
688
            push @needed, $pkgid;
689
#print "= $pkgid, $AllPackages{$pkgid}{name}\n";
690
        }
691
 
692
        #
693
        #   Have enough for a query
694
        #
695
        if ( scalar @needed > 100 || (! $pkgid && scalar @needed > 0) )
696
        {
697
            #
698
            #   Now extract all the source paths
699
            #
700
            my $m_sqlstr = "SELECT UNIQUE" .
701
                        " pv.PKG_ID, ".                                     #[0]
702
                        " pv.SRC_PATH, ".                                   #[1]
703
                        " pv.VCS_TYPE_ID ".                                 #[2]
704
                        " FROM RELEASE_MANAGER.PACKAGE_VERSIONS pv".
705
                        " WHERE pv.PKG_ID in (" . join(',', @needed) . ")";
706
 
707
            my $sth = $RM_DB->prepare($m_sqlstr);
708
            if ( defined($sth) )
709
            {
710
                if ( $sth->execute( ) )
711
                {
712
                    if ( $sth->rows )
713
                    {
714
                        while ( my @row = $sth->fetchrow_array )
715
                        {
716
#print "--- @row\n";
717
                            my $pkgid = $row[0];
718
                            my $type = $row[2] || '2';
719
                            if ( $row[1] )
720
                            {
721
                                #
722
                                #   Trivial cleanup
723
                                #
724
                                my $path = '/' . $row[1];   # Force initial /
725
                                $path =~ tr~\\/~/~s;        # Force / and single /
726
                                $path =~ s~/+$~~;           # Remove Trailing /
727
                                $AllPackages{$pkgid}{srcPath}{$path}++;
728
                            }
729
                        }
730
                    }
731
                    $sth->finish();
732
                }
733
                else
734
                {
735
                    Error("getSourcePaths:Execute failure: $m_sqlstr", $sth->errstr() );
736
                }
737
            }
738
            else
739
            {
740
                Error("getSourcePaths:Prepare failure" );
741
            }
742
 
743
            #
744
            #   Reset the list
745
            #
746
            @needed = ();
747
        }
748
    }
749
}
750
 
751
 
392 dpurdie 752
#-------------------------------------------------------------------------------
753
# Function        : massageVersion
754
#
755
# Description     : Process a version number and return usful bits
756
#
757
# Inputs          : Version Number
758
#                   Package Name - debug only
759
#
760
# Returns         : An array
761
#                       suffix
762
#                       multipart version string useful for text comparisons
763
#
764
sub massageVersion
765
{
766
    my ($version, $name) = @_;
767
    my ($major, $minor, $patch, $build, $suffix);
768
    my $result;
1197 dpurdie 769
    my $buildVersion;
392 dpurdie 770
    my $isaRipple;
771
    my $isaWIP;
772
    $build = 0;
773
 
1197 dpurdie 774
#print "--- $name, $version\n";
775
    $version =~ s~^_~~;
776
    $version =~ s~^\Q${name}\E_~~;
777
 
392 dpurdie 778
    #
779
    #   Pre-massage some silly ones
780
    #
781
    if ( exists $sillyVersions{$version} ) {
782
        $version = $sillyVersions{$version};
783
    }
784
 
1197 dpurdie 785
    if ( $name eq 'ReleaseName' ) {
786
        $version =~ s~[a-z]~.~g;
787
        $version =~ s~\.+~.~g;
788
        $version =~ s~\.$~~g
789
    }
392 dpurdie 790
 
1197 dpurdie 791
    #
792
    #   xxxxxxxxx.nnnn.cots
793
    #
392 dpurdie 794
    if ( $version =~ m~(.*)\.cots$~ ) {
795
        my $cots_base = $1;
796
        $suffix = '.cots';
797
        if ( $version =~ m~(.*?)\.([0-9]{4})\.cots$~ )
798
        {
799
            $result = $1 . sprintf (".%4.4d", $2) . $suffix;
800
        }
801
        else
802
        {
803
            $result = $cots_base . '.0000.cots';
804
        }
805
    }
806
    #
807
    #   Convert version into full form for comparisions
808
    #       nnn.nnn.nnn.[p]nnn.xxx
809
    #       nnn.nnn.nnn.[p]nnn-xxx
810
    #       nnn.nnn.nnn-[p]nnn.xxx
811
    #       nnn.nnn.nnn-[p]nnn-xxx
812
    #       nnn.nnn.nnn[p]nnn-xxx
1197 dpurdie 813
    #   Don't flag as ripples - they are patches
392 dpurdie 814
    #
815
    elsif ( $version =~ m~^(\d+)\.(\d+)\.(\d+)[-.p][p]?(\d+)([-.](.*))?$~ ) {
816
        $major = $1;
817
        $minor = $2;
818
        $patch = $3;
819
        $build = $4;
820
        $suffix = defined $6 ? ".$6" : '';
1197 dpurdie 821
        $isaRipple = 0;
392 dpurdie 822
    }
823
    #
824
    #       nn.nnn.nnnnn.xxx
825
    #       nn.nnn.nnnnn-xxx
1197 dpurdie 826
    #       nnn.nnn.nnnx.xxx
827
    #   Don't flag as ripples - they are patches
392 dpurdie 828
    #
1197 dpurdie 829
    elsif ( $version =~ m~^(\d+)\.(\d+)\.(\d+)\w?([-.](.*))?$~ ) {
392 dpurdie 830
        $major = $1;
831
        $minor = $2;
832
        $patch = $3;
833
        if ( length( $patch) >= 4 )
834
        {
835
            $build = substr( $patch, -3 ,3);
836
            $patch = substr( $patch,  0 ,length($patch)-3);
837
        }
838
        $suffix = defined $5 ? ".$5" : '';
839
    }
1197 dpurdie 840
 
392 dpurdie 841
    #
1197 dpurdie 842
    #       nnn.nnn.nnn
843
    #       nnn.nnn-nnn
844
    #       nnn.nnn_nnn
392 dpurdie 845
    #
1197 dpurdie 846
    elsif ( $version =~ m~^(\d+)\.(\d+)[-._](\d+)$~ ) {
392 dpurdie 847
        $major = $1;
848
        $minor = $2;
849
        $patch = $3;
850
        $suffix = '';
851
    }
1197 dpurdie 852
 
392 dpurdie 853
    #
1197 dpurdie 854
    #       nnn.nnn.nnn.nnn
855
    #       nnn.nnn.nnn-nnn
856
    #       nnn.nnn.nnn_nnn
392 dpurdie 857
    #
1197 dpurdie 858
    elsif ( $version =~ m~^(\d+)\.(\d+)\.(\d+)[-._](\d+)$~ ) {
392 dpurdie 859
        $major = $1;
860
        $minor = $2;
861
        $patch = $3;
1197 dpurdie 862
        $build = $4;
392 dpurdie 863
        $suffix = '';
1197 dpurdie 864
        $isaRipple = 0;
392 dpurdie 865
    }
1197 dpurdie 866
 
392 dpurdie 867
 
868
    #
869
    #       nnn.nnn
870
    #
871
    elsif ( $version =~ m~^(\d+)\.(\d+)$~ ) {
872
        $major = $1;
873
        $minor = $2;
874
        $patch = 0;
875
        $suffix = '';
876
    }
877
    #
1197 dpurdie 878
    #       nnn.nnn.xxx
879
    #
880
    elsif ( $version =~ m~^(\d+)\.(\d+)(\.\w+)$~ ) {
881
        $major = $1;
882
        $minor = $2;
883
        $patch = 0;
884
        $suffix = $3;
885
    }
886
 
887
    #
392 dpurdie 888
    #       nnn.nnn.nnnz
889
    #
890
    elsif ( $version =~ m~^(\d+)\.(\d+)\.(\d+)([a-z])$~ ) {
891
        $major = $1;
892
        $minor = $2;
893
        $patch = $3;
894
        $build = ord($4) - ord('a');
895
        $suffix = '.cots';
1197 dpurdie 896
        $isaRipple = 0;
392 dpurdie 897
    }
898
    #
899
    #       ???REV=???
900
    #
901
    elsif ( $version =~ m~REV=~ ) {
902
        $suffix = '.cots';
903
        $result = $version . '.0000.cots';
904
    }
905
 
906
    #
907
    #   Wip Packages
1197 dpurdie 908
    #   (nnnnnn).xxx
392 dpurdie 909
    #   Should be essential, but want to sort very low
910
    #
1197 dpurdie 911
    elsif ($version =~ m~\((.*)\)(\..*)?~) {
912
        $suffix = $2 || '';
913
        $result = "000.000.000.000$suffix";
392 dpurdie 914
        $isaWIP = 1;
915
    }
1197 dpurdie 916
 
392 dpurdie 917
    #
1197 dpurdie 918
    #   !current
392 dpurdie 919
    #
1197 dpurdie 920
    elsif ($version eq '!current' || $version eq 'current_$USER' || $version eq 'current' || $version eq 'beta' || $version eq 'latest' || $version eq 'beta.cr' || $version eq 'CREATE') {
921
        $suffix = '';
922
        $result = "000.000.000.000$suffix";
923
        $isaWIP = 1;
924
    }
925
 
926
    #
927
    #   Also WIP: FINRUN.103649.BEI.WIP
928
    elsif ($version =~ m~(\.[a-zA-Z]+)\.WIP$~) {
929
        $suffix = lc($1);
930
        $result = "000.000.000.000$suffix";
931
        $isaWIP = 1;
932
    }
933
 
934
    #
935
    #   Also ERGOFSSLS190100_015
936
    #   Don't flag as a ripple
937
    elsif ($version =~ m~^ERG[A-Z]+(\d\d)(\d\d)(\d\d)[-_](\d+)(\.\w+)?$~) {
938
        $major = $1;
939
        $minor = $2;
940
        $patch = $3;
941
        $build = $4;
942
        $suffix = $5 || '.sls';
943
        $isaRipple = 0;
944
    }
945
 
946
    #
947
    #   Stuff we don't yet handle
948
    #
392 dpurdie 949
    else  {
1197 dpurdie 950
        Warning ("Unknown version number: $name,$version");
392 dpurdie 951
        $version =~ m~(\.\w+)$~;
952
        $suffix = $1 || '';
953
        $result = $version;
954
    }
955
 
1197 dpurdie 956
    $isaRipple = ($build > 0) unless defined $isaRipple;
392 dpurdie 957
    unless ( $result )
958
    {
1197 dpurdie 959
        # Major and minor of 99.99 are normally funny versions
960
        # Don't make important decisions on them
961
        #
962
        if (defined $major && defined $minor && $major == 99 && $minor == 99 )
963
        {
964
            $major = 0;
965
            $minor = 0;
966
            $patch = 0;
967
        }
968
 
392 dpurdie 969
        $result = sprintf("%3.3d.%3.3d.%3.3d.%3.3d%s", $major,$minor,$patch,$build,$suffix || '.0000');
1197 dpurdie 970
        $buildVersion = [ $major, $minor, $patch, $build ];
392 dpurdie 971
    }
972
 
1197 dpurdie 973
    $suffix = lc( $suffix );
974
    if ( exists $suffixFixup{$suffix} )
975
    {
976
        $suffix = $suffixFixup{$suffix} ;
977
    }
392 dpurdie 978
 
1197 dpurdie 979
    return ($suffix, $result, $isaRipple, $isaWIP, $buildVersion );
392 dpurdie 980
}
981
 
982
 
983
#-------------------------------------------------------------------------------
984
# Function        : LocateStrays
985
#
2764 dpurdie 986
# Description     : Locate stray packages
987
#                   Try to do several at a time to speed up processing
392 dpurdie 988
#
989
# Inputs          :
990
#
991
# Returns         :
992
#
993
sub LocateStrays
994
{
995
    Message ("Locate indirectly referenced packages");
996
    while ( $#StrayPackages >= 0 )
997
    {
2764 dpurdie 998
#print "Strays Remaining: ", scalar @StrayPackages ,"\n";
392 dpurdie 999
 
2764 dpurdie 1000
        my @plist;
1001
        while ( $#plist <= 200 && @StrayPackages )
1002
        {
1003
            my $pv_id = pop @StrayPackages;
1004
            next if ( exists $Packages{$pv_id}{done} );
1005
            push @plist, $pv_id;
1006
        }
1007
 
1008
        GetDepends(@plist) if @plist;
1009
 
1010
        foreach ( @plist)
1011
        {
1012
            $Packages{$_}{done} = 1;
1013
        }
392 dpurdie 1014
    }
1015
}
1016
 
1017
#-------------------------------------------------------------------------------
1197 dpurdie 1018
# Function        : countPackages
392 dpurdie 1019
#
1197 dpurdie 1020
# Description     : 
392 dpurdie 1021
#
1022
# Inputs          : 
1023
#
1197 dpurdie 1024
# Returns         : Number of packages and number oof versions
392 dpurdie 1025
#
1197 dpurdie 1026
sub countPackages
392 dpurdie 1027
{
1197 dpurdie 1028
    my $v = 0;
1029
    my $p = 0;
1030
    my %names;
392 dpurdie 1031
 
1197 dpurdie 1032
    foreach ( keys %Packages )
1033
    {
1034
        my $name = $Packages{$_}{name};
1035
        next unless ( $name );
1036
        $names{$name} = 1;
1037
        $v++;
1038
    }
392 dpurdie 1039
 
1197 dpurdie 1040
    $p = keys %names;
1041
 
1042
    return $p,$v;
1043
 
1044
}
1045
 
1046
#-------------------------------------------------------------------------------
1047
# Function        : processData
1048
#
1049
# Description     : Process data before its written out
1050
#                       Remove a few packages that we do not want to now about
1051
#                       Determine Reason that a version is in the list
1052
#                       Finish taging packages in NamedProject
1053
#
1054
# Inputs          : 
1055
#
1056
# Returns         : 
1057
#
1058
sub processData
1059
{
392 dpurdie 1060
    foreach ( keys %Packages )
1061
    {
1062
        delete $Packages{$_}{done};
1063
        next if ( $Packages{$_}{name} =~ ~m~CSWcfengine~ );
1064
 
1065
        if ($Packages{$_}{name} eq 'Activestate Perl - Solaris')
1066
        {
1067
            delete $Packages{$_};
1068
            next;
1069
        }
1070
 
1071
        if ( $Packages{$_}{name} =~ m/^CSW/ || $Packages{$_}{name} =~ m/^Solaris$/)
1072
        {
1073
            delete $Packages{$_};
1074
            next;
1075
        }
1076
 
1077
        if ( $Packages{$_}{name} =~ m/^jats_/)
1078
        {
1079
            delete $Packages{$_};
1080
            next;
1081
        }
1197 dpurdie 1082
 
1083
 
1084
        #
1085
        #   Catch packages that are dependents of NamedProject's
1086
        #
1087
        if ( $doIncludeOnly )
1088
        {
1089
            if ( exists  $Packages{$_}{'sbomBase'} || exists  $Packages{$_}{'sbomOsid'} )
1090
            {
1091
                $Packages{$_}{NamedProject} = 4;
1092
            }
1093
 
1094
            unless ( $Packages{$_}{NamedProject}  )
1095
            {
1096
                my $named;
1097
                my %usedBy;
1098
 
1099
                if ( exists $Packages{$_}{'usedBy'})
1100
                {
1101
                    my @examineThese = @{$Packages{$_}{'usedBy'}};
1102
                    while ( @examineThese )
1103
                    {
1104
                        my $pvid = pop @examineThese;
1105
                        next if ( $usedBy{$pvid} );
1106
 
1107
                        if ( $Packages{$pvid}{NamedProject}  )
1108
                        {
1109
                            $named = 1;
1110
                            last;
1111
                        }
1112
 
1113
                        push @examineThese, @{$Packages{$pvid}{'usedBy'}}
1114
                            if (exists $Packages{$pvid}{'usedBy'});
1115
                    }
1116
                    $Packages{$_}{NamedProject} = 5
1117
                        if ( $named );
1118
                }
1119
#                else
1120
#                {
1121
#                    Warning("Not Named and not usedBy: $Packages{$_}{name} $Packages{$_}{'version'}");
1122
#                }
1123
            }
1124
        }
1125
        else
1126
        {
1127
            $Packages{$_}{NamedProject} = 6;
1128
        }
392 dpurdie 1129
    }
1197 dpurdie 1130
}
392 dpurdie 1131
 
1197 dpurdie 1132
#-------------------------------------------------------------------------------
1133
# Function        : outputData
1134
#
1135
# Description     : Write out data in a form to allow post processing
1136
#
1137
# Inputs          : 
1138
#
1139
# Returns         : 
1140
#
1141
sub outputData
1142
{
1143
    my $file = "cc2svn.raw.txt";
1144
    Message ("Create: $file");
1145
    my $fh = ConfigurationFile::New( $file );
1146
 
392 dpurdie 1147
    $fh->DumpData(
1197 dpurdie 1148
        "\n# Releases.\n#\n",
1149
        "ScmReleases", \%Releases );
1150
 
1151
    $fh->DumpData(
392 dpurdie 1152
        "\n# Packages.\n#\n",
1153
        "ScmPackages", \%Packages );
1154
 
1155
    $fh->DumpData(
1156
        "\n# Suffixes.\n#\n",
1157
        "ScmSuffixes", \%Suffixes );
1158
 
1159
    $fh->DumpData(
1160
        "\n# All Package Names.\n#\n",
1161
        "ScmAllPackages", \%AllPackages );
1162
 
1163
    #
1164
    #   Close out the file
1165
    #
1166
    $fh->Close();
1167
 
1168
#    #
1169
#    #   Split up package data into small files for easy consumption
1170
#    #
1171
#
1172
#    foreach ( keys %Packages )
1173
#    {
1174
#        my $file = "cc2svn.raw.${_}.txt";
1175
#        Message ("Create: $file");
1176
#        my $fh = ConfigurationFile::New( $file );
1177
#
1178
#        $fh->DumpData(
1179
#            "\n# Releases.\n#\n",
1180
#            "ScmReleases", \$Packages{$_} );
1181
#        $fh->Close();
1182
#    }
1183
 
1184
}
1185
 
1186
 
1187
#-------------------------------------------------------------------------------
1188
#   Documentation
1189
#
1190
 
1191
=pod
1192
 
1193
=for htmltoc    SYSUTIL::cc2svn::
1194
 
1195
=head1 NAME
1196
 
1197
cc2svn_gendata - Extract CC2SVN Essential Package Data from Release Manager
1198
 
1199
=head1 SYNOPSIS
1200
 
1201
  jats cc2svn_gendata [options]
1202
 
1203
 Options:
1204
    -help              - brief help message
1205
    -help -help        - Detailed help message
1206
    -man               - Full documentation
1207
    -test=version      - Test a version string, then exit
1208
    -limit=n           - Limit packages processed. Test only
1197 dpurdie 1209
    -mode=xxx          - Set Mode: all, hops, standard
392 dpurdie 1210
 
1211
=head1 OPTIONS
1212
 
1213
=over 8
1214
 
1215
=item B<-help>
1216
 
1217
Print a brief help message and exits.
1218
 
1219
=item B<-help -help>
1220
 
1221
Print a detailed help message with an explanation for each option.
1222
 
1223
=item B<-man>
1224
 
1225
Prints the manual page and exits.
1226
 
1227
=item B<-test=version>
1228
 
1229
Examine a package version string and report how the tool will parse it.
1230
 
1231
=item B<-limit=n>
1232
 
1233
Limit the number of packages processed by the tool. This is only used to
1234
simplify testing of the program
1235
 
1236
=back
1237
 
1238
=head1 DESCRIPTION
1239
 
1240
This program is a tool used in the conversion of ClearCase VOBS to subversion.
1241
It will:
1242
 
1243
=over 8
1244
 
1245
=item *
1246
 
1247
Determine all Releases in Release manager and mark those that
1248
are to be excluded.
1249
 
1250
=item *
1251
 
1252
Determine all the package-versions used by the releases that are
1253
not excluded. These are called 'direct' dependencies.
1254
 
1255
=item *
1256
 
1257
Recursively find all the dependent packages of all packages. New package
1258
versions are called 'indirect' dependencies. They are buried. This process can
1259
take several minutes.
1260
 
1261
=back
1262
 
1263
The data collected is dumped into a text file for later processing.
1264
 
1265
=cut
1266