Subversion Repositories DevTools

Rev

Rev 392 | Rev 1197 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | RSS feed 

########################################################################
# Copyright (C) 1998-2012 Vix Technology, All rights reserved
#
# Module name   : cc2svn_gendata.pl
# Module type   : Makefile system
# Compiler(s)   : Perl
# Environment(s): jats
#
# Description   : Get all packages that are used in all releases
#                 Create a data file that can be used offline
#
#                 The process will exclude some old releases
#
#                 Generate data on Essential Package Versions to be
#                 transferred from CC to Subversion
#
#......................................................................#

require 5.006_001;
use strict;
use warnings;
use JatsError;
use JatsSystem;
use Getopt::Long;
use Pod::Usage;                             # required for help support
use JatsRmApi;
use ConfigurationFile;

use DBI;
use HTTP::Date;

my $VERSION = "1.2.3";                      # Update this
my $opt_verbose = 0;
my $opt_help = 0;
my $opt_manual;
my $opt_test;
my $opt_limit;
my $opt_quick;
my $RM_DB;
my $now = time();

#
#   Package information
#
my %Releases;
my %Packages;
my %Suffixes;
my @StrayPackages;
my %AllPackages;

my $doAllReleases = 0;
my $doIncludeOnly = 0;
my @includedProjects = (
#        481,    # UK BUS HOPS
);

my @includedReleases = (
        6222,   # HOME > UK STAGE COACH (SSW) > Mainline
        14503,  # HOME > UK STAGE COACH (SSW) > ITSO_HOPS_3
        21303,  # HOME > UK STAGE COACH (SSW) > SUPPORT_HOPS_REPORTS
        21343,  # HOME > UK STAGE COACH (SSW) > SUPPORT_CIPP
        17223,  # HOME > UK STAGE COACH (SSW) > ITSO HOPS 4
);


my @excludeProjects = ( 162,            # WASHINGTON (WDC)
                        341,            # TUTORIAL (TUT)
                        142,            # SYDNEY (SYD)
                        182 ,           # ROME (ROM)
                        6 ,             # GMPTE/PCL (GMP)
                        521,            # NSW CLUB CARD
                        221,            # NZ STAGE COACH (NZS)
                        82              # LVS
                        );
my @excludeReleases = ( 20424,          # MASS_REF (MAS) > test
                        9043,           # TECHNOLOGY GROUP > Development Environment - For Test Setup
                        #9263,           # TECHNOLOGY GROUP > Buildtool DEVI&TEST
                        14383,          # TECHNOLOGY GROUP > eBrio TDS
                        20463,          # TECHNOLOGY GROUP > TPIT - BackOffice Linux build
                        14603,          # TECHNOLOGY GROUP > TPIT - BackOffice 64 bit [CCB Mode!]
                        22163,          # GLOBAL PRODUCT MGMT > Rio Tinto - Remote Draught Survey
                        19483,          # SEATTLE (SEA) > Phase 2 - I18 [backup] [Restrictive Mode]
                        20403,          # SEATTLE (SEA) > Phase 2 - I19 [backup]
                        20983,          # ??? May have been deleted
                        13083,          # TECHNOLOGY GROUP > TRACS
                        15224,          # 64Bit Solaris Test
                        );

my %sillyVersions =
(
    '2b6'           => '2.6.0.cots',
    '1.0b2'         => '1.0.2.cots',
    '1.6.x'         => '1.6.0.cots',
    '3.5beta12.5'   => '3.5.12.5.cots',
);
                        
#-------------------------------------------------------------------------------
# Function        : Main Entry
#
# Description     :
#
# Inputs          :
#
# Returns         :
#
my $result = GetOptions (
                "help+"         => \$opt_help,          # flag, multiple use allowed
                "manual"        => \$opt_manual,        # flag
                "verbose+"      => \$opt_verbose,       # flag
                "test:s"        => \$opt_test,          # Test a version string
                "limit:n"       => \$opt_limit,         #
                "quick"         => \$opt_quick,         # Don't look for indirects
                );

#
#   Process help and manual options
#
pod2usage(-verbose => 0, -message => "Version: $VERSION")  if ($opt_help == 1  || ! $result);
pod2usage(-verbose => 1)  if ($opt_help == 2 );
pod2usage(-verbose => 2)  if ($opt_manual || ($opt_help > 2));

if ( $opt_test )
{
    my @results = massageVersion( $opt_test, 'DummyName' );
    Message ("Version", $opt_test, @results);
    exit 1;
}


ErrorConfig( 'name'    =>'CC2SVN_GENDATA' );
GetAllPackageNames();
getReleaseDetails();
getPkgDetailsByRTAG_ID();
my $count;
$count = keys %Packages;
print "Directly referenced Packages: $count\n";
LocateStrays() unless ($opt_quick);
$count = keys %Packages;
print "Indirectly referenced Packages: $count\n";
outputData();

if ( $opt_verbose > 1 )
{
    print "=========================================================================\n";
    DebugDumpData("Releases", \%Releases);
    print "=========================================================================\n";
    DebugDumpData("Packages", \%Packages );
    print "=========================================================================\n";
    DebugDumpData("Suffixes", \%Suffixes );
}

$count = keys %Packages;
print "Total References Packages: $count\n";

exit;

#-------------------------------------------------------------------------------
# Function        : getReleaseDetails
#
# Description     : Determine all candiate releases
#
# Inputs          : 
#
# Returns         : 
#
sub getReleaseDetails
{
    my (@row);

    # if we are not or cannot connect then return 0 as we have not found anything
    connectRM(\$RM_DB) unless $RM_DB;

    # First get all packages that are referenced in a Release
    # This will only get the top level packages
    # From non-archived releases

    my $m_sqlstr = "SELECT prj.PROJ_NAME, rt.RTAG_NAME, rt.PROJ_ID, rt.RTAG_ID, rt.official" .
                   " FROM release_manager.release_tags rt, release_manager.projects prj" .
                   " WHERE prj.PROJ_ID = rt.PROJ_ID " .
                   "   AND rt.official != 'A' AND rt.official != 'Y'" .
                   " order by prj.PROJ_NAME";
    my $sth = $RM_DB->prepare($m_sqlstr);
    if ( defined($sth) )
    {
        if ( $sth->execute( ) )
        {
#            print "--- Execute\n";
            if ( $sth->rows )
            {
#                print "--- Execute ROWS\n";
                while ( @row = $sth->fetchrow_array )
                {
                    my $rtag_id =$row[3];
                    my $proj_id = $row[2];

                    $Releases{$rtag_id}{pName} = $row[0];
                    $Releases{$rtag_id}{name} = $row[1];
                    $Releases{$rtag_id}{proj_id} = $proj_id;
                    $Releases{$rtag_id}{rtag_id} = $rtag_id;
                    $Releases{$rtag_id}{official} = $row[4];

                    unless ( $doAllReleases )
                    {
                        if (grep {$_ eq $proj_id} @excludeProjects) {
                            $Releases{$rtag_id}{excluded} = 'E';
                        }

                        if (grep {$_ eq $rtag_id} @excludeReleases) {
                            $Releases{$rtag_id}{excluded} = 'E';
                        }
                    }

                    if ( $doIncludeOnly )
                    {

                        if (grep {$_ eq $proj_id} @includedProjects)
                        {
                            delete $Releases{$rtag_id}{excluded};
                        }
                        else
                        {
                            $Releases{$rtag_id}{excluded} = 'E';
                        }

                        if (grep {$_ eq $rtag_id} @includedReleases)
                        {
                            delete $Releases{$rtag_id}{excluded};
                        }
                    }

                    unshift @row, $Releases{$rtag_id}{excluded} || ' ';
                    print join (',',@row), "\n" if ($opt_verbose);
                }
            }
#            print "--- Finish\n";
            $sth->finish();
        }
        else
        {
            Error("Execute failure: $m_sqlstr", $sth->errstr() );
        }
    }
    else
    {
        Error("Prepare failure" );
    }
}


sub getPkgDetailsByRTAG_ID
{
    my (@row);
    my $excludes = '';
    my $count = 0;

    # if we are not or cannot connect then return 0 as we have not found anything
    connectRM(\$RM_DB) unless $RM_DB;

    Message ("Extract toplevel dependencies");

    # First get all packages that are referenced in a Release
    # This will only get the top level packages
    # From non-archived releases

    unless ($doAllReleases)
    {
        foreach  ( @excludeProjects )
        {
            $excludes .= " AND prj.PROJ_ID != $_ ";
        }
        foreach  ( @excludeReleases )
        {
            $excludes .= " AND rt.RTAG_ID != $_ ";
        }
    }

    my $m_sqlstr = "SELECT DISTINCT pv.PV_ID, pkg.PKG_NAME, pv.PKG_VERSION, pv.DLOCKED" .
                   "    , release_manager.PK_RMAPI.return_vcs_tag(pv.PV_ID), pv.PKG_ID" .
                   "    , rt.RTAG_ID, rmv.VIEW_NAME, pv.MODIFIED_STAMP, prj.PROJ_ID" .
                   " FROM RELEASE_MANAGER.RELEASE_CONTENT rc, RELEASE_MANAGER.PACKAGE_VERSIONS pv,".
                   "      RELEASE_MANAGER.PACKAGES pkg, release_manager.release_tags rt, release_manager.projects prj" .
                   "    , release_manager.views rmv" .
                   " WHERE rc.PV_ID = pv.PV_ID AND pv.PKG_ID = pkg.PKG_ID" .
                   "   AND rmv.VIEW_ID = rc.BASE_VIEW_ID" .
                   "   AND prj.PROJ_ID = rt.PROJ_ID and rt.RTAG_ID = rc.RTAG_ID" .
                   "   AND rt.official != 'A' AND rt.official != 'Y'" .
                   $excludes .
                   " order by pkg.PKG_NAME";
    my $sth = $RM_DB->prepare($m_sqlstr);
    if ( defined($sth) )
    {
        if ( $sth->execute( ) )
        {
#            print "--- Execute\n";
            if ( $sth->rows )
            {
#                print "--- Execute ROWS\n";
                while ( @row = $sth->fetchrow_array )
                {
                    print join (',',@row), "\n" if ($opt_verbose);
                    my $pvid = $row[0];
                    unless ( exists $Packages{$pvid}{name} )
                    {
                        $Packages{$pvid}{name} = $row[1];
                        $Packages{$pvid}{version} = $row[2];
                        $Packages{$pvid}{locked} = $row[3];
                        $row[4] =~ tr~\\/~/~;
                        $Packages{$pvid}{vcstag} = $row[4];
                        $Packages{$pvid}{pkgid} = $row[5];
                        $Packages{$pvid}{tlp} = 1;
                        ($Packages{$pvid}{suffix}, $Packages{$pvid}{fullVersion},$Packages{$pvid}{isaRipple} ) = massageVersion( $Packages{$pvid}{version}, $Packages{$pvid}{name} );
                        $Suffixes{$Packages{$pvid}{suffix}}++;

                        push @StrayPackages, $pvid;
                    }

                    my $rtag_id = $row[6];
                    push @{$Packages{$pvid}{release}}, $rtag_id;
                    $Packages{$pvid}{view}{$row[7]}++ if ( $row[7] );

                    $Packages{$pvid}{Age} = ($now - str2time( $row[8] )) / (60 * 60 * 24);

                    my $proj_id = $row[9];
                    push @{$Packages{$pvid}{projects}}, $proj_id;

                    if ( $doIncludeOnly )
                    {
                        if (grep {$_ eq $proj_id} @includedProjects)
                        {
                            $Packages{$pvid}{NamedProject} = 1;
                        }
                        if (grep {$_ eq $rtag_id} @includedReleases)
                        {
                            $Packages{$pvid}{NamedProject} = 1;
                        }
                    }
                    else
                    {
                        $Packages{$pvid}{NamedProject} = 1;
                    }


                    if ( $opt_limit )
                    {
                        last if ( $count++ > $opt_limit );
                    }
                }
            }
#            print "--- Finish\n";
            $sth->finish();
        }
        else
        {
            Error("Execute failure: $m_sqlstr", $sth->errstr() );
        }
    }
    else
    {
        Error("Prepare failure" );
    }
}

#-------------------------------------------------------------------------------
# Function        : GetDepends
#
# Description     :
#
# Inputs          : $pvid
#
# Returns         :
#
sub GetDepends
{
    my ($pv_id ) = @_;

    #
    #   Now extract the package dependacies
    #
    my $m_sqlstr = "SELECT pv.PV_ID, pkg.PKG_NAME, pv.PKG_VERSION, pv.DLOCKED, release_manager.PK_RMAPI.return_vcs_tag(pv.PV_ID), pv.PKG_ID, pv.MODIFIED_STAMP" .
                  " FROM RELEASE_MANAGER.PACKAGE_DEPENDENCIES pd, RELEASE_MANAGER.PACKAGE_VERSIONS pv, RELEASE_MANAGER.PACKAGES pkg" .
                  " WHERE pd.PV_ID = \'$pv_id\' AND pd.DPV_ID = pv.PV_ID" .
                  "       AND pv.PKG_ID = pkg.PKG_ID";
    my $sth = $RM_DB->prepare($m_sqlstr);
    if ( defined($sth) )
    {
        if ( $sth->execute( ) )
        {
            if ( $sth->rows )
            {
                while ( my @row = $sth->fetchrow_array )
                {
                    my $pvid = $row[0];
                    unless ( exists $Packages{$pvid}{name} )
                    {
                        print join (',',@row), "\n" if ($opt_verbose) ;
                        $Packages{$pvid}{name} = $row[1];
                        $Packages{$pvid}{version} = $row[2];
                        $Packages{$pvid}{locked} = $row[3];
                        $row[4] =~ tr~\\/~/~;
                        $Packages{$pvid}{vcstag} = $row[4];
                        $Packages{$pvid}{pkgid} = $row[5];
                        $Packages{$pvid}{Age} = ($now - str2time( $row[6] )) / (60 * 60 * 24);
                        
                        $Packages{$pvid}{depend} = 1;
                        ($Packages{$pvid}{suffix}, $Packages{$pvid}{fullVersion},$Packages{$pvid}{isaRipple} ) = massageVersion( $Packages{$pvid}{version},$Packages{$pvid}{name} );
                        $Suffixes{$Packages{$pvid}{suffix}}++;

                        push @StrayPackages, $pvid;
                    }

                    push @{$Packages{$pvid}{usedBy}}, $pv_id;
                }
            }
            $sth->finish();
        }
        else
        {
        Error("GetDepends:Execute failure" );
        }
    }
    else
    {
        Error("GetDepends:Prepare failure" );
    }
}

#-------------------------------------------------------------------------------
# Function        : GetAllPackageNames
#
# Description     :
#
# Inputs          : None
#
# Returns         :
#
sub GetAllPackageNames
{
    # if we are not or cannot connect then return 0 as we have not found anything
    connectRM(\$RM_DB) unless $RM_DB;

    #
    #   Now extract all the package names
    #
    my $m_sqlstr = "SELECT pkg.PKG_ID, pkg.PKG_NAME" .
                  " FROM RELEASE_MANAGER.PACKAGES pkg";
    my $sth = $RM_DB->prepare($m_sqlstr);
    if ( defined($sth) )
    {
        if ( $sth->execute( ) )
        {
            if ( $sth->rows )
            {
                while ( my @row = $sth->fetchrow_array )
                {
                    my $id = $row[0];
                    my $name = $row[1];
                    next unless ( $id );
                    $AllPackages{$id} = $name;
                }
            }
            $sth->finish();
        }
        else
        {
        Error("GetAllPackageNames:Execute failure" );
        }
    }
    else
    {
        Error("GetAllPackageNames:Prepare failure" );
    }
}


#-------------------------------------------------------------------------------
# Function        : massageVersion
#
# Description     : Process a version number and return usful bits
#
# Inputs          : Version Number
#                   Package Name - debug only
#
# Returns         : An array
#                       suffix
#                       multipart version string useful for text comparisons
#
sub massageVersion
{
    my ($version, $name) = @_;
    my ($major, $minor, $patch, $build, $suffix);
    my $result;
    my $isaRipple;
    my $isaWIP;
    $build = 0;

    #
    #   Pre-massage some silly ones
    #
    if ( exists $sillyVersions{$version} ) {
        $version = $sillyVersions{$version};
    }


    if ( $version =~ m~(.*)\.cots$~ ) {
        my $cots_base = $1;
        $suffix = '.cots';
        if ( $version =~ m~(.*?)\.([0-9]{4})\.cots$~ )
        {
            $result = $1 . sprintf (".%4.4d", $2) . $suffix;
        }
        else
        {
            $result = $cots_base . '.0000.cots';
        }
    }
    #
    #   Convert version into full form for comparisions
    #       nnn.nnn.nnn.[p]nnn.xxx
    #       nnn.nnn.nnn.[p]nnn-xxx
    #       nnn.nnn.nnn-[p]nnn.xxx
    #       nnn.nnn.nnn-[p]nnn-xxx
    #       nnn.nnn.nnn[p]nnn-xxx
    #
    elsif ( $version =~ m~^(\d+)\.(\d+)\.(\d+)[-.p][p]?(\d+)([-.](.*))?$~ ) {
        $major = $1;
        $minor = $2;
        $patch = $3;
        $build = $4;
        $suffix = defined $6 ? ".$6" : '';
    }
    #
    #       nn.nnn.nnnnn.xxx
    #       nn.nnn.nnnnn-xxx
    #
    elsif ( $version =~ m~^(\d+)\.(\d+)\.(\d+)([-.](.*))?$~ ) {
        $major = $1;
        $minor = $2;
        $patch = $3;
        if ( length( $patch) >= 4 )
        {
            $build = substr( $patch, -3 ,3);
            $patch = substr( $patch,  0 ,length($patch)-3);
        }
        $suffix = defined $5 ? ".$5" : '';
    }
    #
    #       nn.nnn.nnn_nnn
    #
    elsif ( $version =~ m~^(\d+)\.(\d+)\.(\d+)[-_.](\d+)$~ ) {
        $major = $1;
        $minor = $2;
        $patch = $3;
        $build = $4;
        $suffix = '';
    }
    
    #
    #       nnn.nnn.nnn
    #       nnn.nnn-nnn
    #
    elsif ( $version =~ m~^(\d+)\.(\d+)[-.](\d+)$~ ) {
        $major = $1;
        $minor = $2;
        $patch = $3;
        $suffix = '';
    }

    #
    #       nnn.nnn
    #
    elsif ( $version =~ m~^(\d+)\.(\d+)$~ ) {
        $major = $1;
        $minor = $2;
        $patch = 0;
        $suffix = '';
    }
    #
    #       nnn.nnn.nnnz
    #
    elsif ( $version =~ m~^(\d+)\.(\d+)\.(\d+)([a-z])$~ ) {
        $major = $1;
        $minor = $2;
        $patch = $3;
        $build = ord($4) - ord('a');
        $suffix = '.cots';
    }
    #
    #       ???REV=???
    #
    elsif ( $version =~ m~REV=~ ) {
        $suffix = '.cots';
        $result = $version . '.0000.cots';
    }

    #
    #   Wip Packages
    #   Should be essential, but want to sort very low
    #
    elsif ($version =~ m~\((.*)\)(\.*)~) {
        $suffix = $2;
        $result = "000.000.000.000.$suffix";
        $isaWIP = 1;
    }
    #
    #   zzzzzzzzzzzzzzz.cots
    #   zzzzzzzzzz.nnnn.cots
    #
    else  {
        Warning ("Unknown version number: '$name','$version'");
        $version =~ m~(\.\w+)$~;
        $suffix = $1 || '';
        $result = $version;
    }

    $isaRipple = ($build > 0);
    unless ( $result )
    {
        $result = sprintf("%3.3d.%3.3d.%3.3d.%3.3d%s", $major,$minor,$patch,$build,$suffix || '.0000');
    }

#    $suffix = $suffixFixup{$suffix} if ( exists $suffixFixup{$suffix} );

    return ($suffix, $result, $isaRipple, $isaWIP );
}


#-------------------------------------------------------------------------------
# Function        : LocateStrays
#
# Description     :
#
# Inputs          :
#
# Returns         :
#
sub LocateStrays
{
    Message ("Locate indirectly referenced packages");
    while ( $#StrayPackages >= 0 )
    {
        my $pv_id = pop @StrayPackages;

        next if ( exists $Packages{$pv_id}{done} );
#print "... ",$#StrayPackages,"\n";
        GetDepends( $pv_id);
        $Packages{$pv_id}{done} = 1;
    }
}

#-------------------------------------------------------------------------------
# Function        : outputData
#
# Description     : Write out data in a form to allow post processing
#
# Inputs          : 
#
# Returns         : 
#
sub outputData
{
    my $file = "cc2svn.raw.txt";
    Message ("Create: $file");
    my $fh = ConfigurationFile::New( $file );

    $fh->DumpData(
        "\n# Releases.\n#\n",
        "ScmReleases", \%Releases );

    foreach ( keys %Packages )
    {
        delete $Packages{$_}{done};
        next if ( $Packages{$_}{name} =~ ~m~CSWcfengine~ );

        if ($Packages{$_}{name} eq 'Activestate Perl - Solaris')
        {
            delete $Packages{$_};
            next;
        }

        if ( $Packages{$_}{name} =~ m/^CSW/ || $Packages{$_}{name} =~ m/^Solaris$/)
        {
            delete $Packages{$_};
            next;
        }

        if ( $Packages{$_}{name} =~ m/^jats_/)
        {
            delete $Packages{$_};
            next;
        }
        
    }

    $fh->DumpData(
        "\n# Packages.\n#\n",
        "ScmPackages", \%Packages );

    $fh->DumpData(
        "\n# Suffixes.\n#\n",
        "ScmSuffixes", \%Suffixes );

    $fh->DumpData(
        "\n# All Package Names.\n#\n",
        "ScmAllPackages", \%AllPackages );
    
    #
    #   Close out the file
    #
    $fh->Close();

#    #
#    #   Split up package data into small files for easy consumption
#    #
#
#    foreach ( keys %Packages )
#    {
#        my $file = "cc2svn.raw.${_}.txt";
#        Message ("Create: $file");
#        my $fh = ConfigurationFile::New( $file );
#
#        $fh->DumpData(
#            "\n# Releases.\n#\n",
#            "ScmReleases", \$Packages{$_} );
#        $fh->Close();
#    }
    
}


#-------------------------------------------------------------------------------
#   Documentation
#

=pod

=for htmltoc    SYSUTIL::cc2svn::

=head1 NAME

cc2svn_gendata - Extract CC2SVN Essential Package Data from Release Manager

=head1 SYNOPSIS

  jats cc2svn_gendata [options]

 Options:
    -help              - brief help message
    -help -help        - Detailed help message
    -man               - Full documentation
    -test=version      - Test a version string, then exit
    -limit=n           - Limit packages processed. Test only

=head1 OPTIONS

=over 8

=item B<-help>

Print a brief help message and exits.

=item B<-help -help>

Print a detailed help message with an explanation for each option.

=item B<-man>

Prints the manual page and exits.

=item B<-test=version>

Examine a package version string and report how the tool will parse it.

=item B<-limit=n>

Limit the number of packages processed by the tool. This is only used to
simplify testing of the program

=back

=head1 DESCRIPTION

This program is a tool used in the conversion of ClearCase VOBS to subversion.
It will:

=over 8

=item *

Determine all Releases in Release manager and mark those that
are to be excluded.

=item *

Determine all the package-versions used by the releases that are
not excluded. These are called 'direct' dependencies.

=item *

Recursively find all the dependent packages of all packages. New package
versions are called 'indirect' dependencies. They are buried. This process can
take several minutes.

=back

The data collected is dumped into a text file for later processing.

=cut