Subversion Repositories DevTools

########################################################################

# Copyright ( C ) 2010 ERG Limited, All rights reserved

#

# Module name   : jats_update_release.pl

# Module type   : Makefile system

# Compiler(s)   : Perl

# Environment(s): jats build system

#

# Description   : Extracts current package version list from Deployment Manager

#                 SBom(s) and copies resultant packages to release specific

#                 directory.

#......................................................................#

require 5.008_002;

use File::Basename;

use File::Copy;

use File::Path;

use strict;

use warnings;

use JatsEnv;

use JatsError;

use JatsSystem;

use JatsRmApi;

use ArrayHashUtils;

use DBI;

use Getopt::Long;

use Pod::Usage;                             # required for help support

use Storable qw (dclone);

#

#   Config Options

#

my $VERSION = "1.0.0";                      # Update this

my $opt_help = 0;

my $opt_verbose = $ENV{'GBE_VERBOSE'};      # Allow global verbose

my @opt_sbom_ids;

my $opt_rootdir;

my @opt_filters;

my $opt_projectdir;

my $opt_releasedir;

my $opt_test;

my @opt_addFilters;

my @opt_delFilters;

#

#   Constants

#

my $CONFFILE = ".updateRelease";

#

#   Data Base Interface

#

my $DM_DB;

#

#   The directory we copy to.

#

my $projectDestDir;

#

#   Configuration file vars

#

my @confFilters;

my $writeConf = 0;

my %filtersUsed;

# -------------------------------------------------------------------------

sub GetYesNo

#

# -------------------------------------------------------------------------

{

    my ($question) = @_;

    my ($u_tmp) = "";

    Question ("$question, (default: y) [y,n]: ");

    while ( <STDIN> )

    {

        $u_tmp = $_;

        chomp($u_tmp);

        return 1

            if ( "$u_tmp" eq "" );

        if( $u_tmp =~ /[yn]{1}/i )

        {

            return ( "$u_tmp" eq "y" );

        }

        else

        {

            Question("Please re-enter response? (default: y) [y,n]: ");

        }

    }

}

#==============================================================================

#   getSbomProjectAndRelease

#   Returns the Project Name and Release for supplied SBoms

#   Calls Error and exists on error condition

#==============================================================================

sub getSbomProjectAndRelease

{

    my ( $DB, $sboms ) = @_;

    my ( $bom, $proj, $rel, $lastProj, $lastRel );

    Error("getSbomProjectAndRelease: SBom Parameter Error, must pass array") if ( ref($sboms) ne "ARRAY" );

    # create a hash of sbom values so we can test after if any sboms could not be found

    my %sbomIdx = map { $_ => 1 } @{$sboms};

    my $m_sqlstr = "SELECT   boms.bom_id, dm_projects.proj_name, branches.branch_name " .

                   "FROM     deployment_manager.boms, deployment_manager.branches, deployment_manager.dm_projects " .

                   "WHERE    branches.branch_id = boms.branch_id AND " .

                   "         dm_projects.proj_id = branches.proj_id AND " .

                   "         boms.bom_id " . ( $#{$sboms} == 0 ? "= " . $sboms->[0] : "IN ( " . join(",", @{$sboms}) . ")" );

    my $sth = $DB->prepare($m_sqlstr);

    if ( defined($sth) )

    {

        if ( $sth->execute( ) )

        {

            if ( $sth->rows )

            {

                while ( ( $bom, $proj, $rel ) = $sth->fetchrow_array )

                {

                    if ( ! defined($proj) )

                    {

                        Error("getSbomProjectAndRelease: NULL Project Name from Deployment Manager Sbom : $bom");

                    }

                    elsif ( ! defined($rel) )

                    {

                        Error("getSbomProjectAndRelease: NULL Release Tag Name from Deployment Manager Sbom : $bom");

                    }

                    elsif ( defined($lastProj) && $proj ne $lastProj )

                    {

                        Error("getSbomProjectAndRelease: SBom Id [$bom] is in a different project [$proj]", "All SBom Id's must all be part of the same Deployment Manager Project");

                    }

                    elsif ( defined($lastRel) && $rel ne $lastRel )

                    {

                        Error("getSbomProjectAndRelease: SBom Id [$bom] is in a different project release [$rel]", "All SBom Id's must all be part of the same Deployment Manager Project Release");

                    }

                    $lastProj = $proj;

                    $lastRel  = $rel;

                    # delete sbom from idx, any remaining after loop will indicate we have an sbom that could not found

                    delete($sbomIdx{$bom});

                }

                my @sbomsNotFound = keys %sbomIdx;

                if ( $#sbomsNotFound > -1 )

                {

                    Error("getSbomProjectAndRelease: Could not find details for the following SBomId(s) " . join(",", @sbomsNotFound) );

                }

            }

            else

            {

                Error("getSbomProjectAndRelease: No SBom(s) found for Deployment Manager SBomId(s) " . join(",", @{$sboms}) );

            }

            $sth->finish();

        }

        else

        {

            Error("getSbomProjectAndRelease: Execute failure", $m_sqlstr );

        }

    }

    else

    {

        Error("getSbomProjectAndRelease: Prepare failure", $m_sqlstr );

    }

    return ( $lastProj, $lastRel );

}   # getSbomProjectAndRelease

#-------------------------------------------------------------------------------

# Function        : Main

#

# Description     : Main entry point

#                   Parse user options

#

# Inputs          :

#

# Returns         :

#

my $result = GetOptions (

                "help:+"            => \$opt_help,              # flag, multiple use allowed

                "manual:3"          => \$opt_help,              # flag, multiple use allowed

                "verbose:+"         => \$opt_verbose,           # flag

                "sbomid|sbom_id=s"  => \@opt_sbom_ids,          # multiple numbers

                "filter=s"          => \@opt_filters,           # multiple strings

                "addfilter=s"       => \@opt_addFilters,        # multiple strings

                "delfilter=s"       => \@opt_delFilters,        # multiple strings

                "rootdir=s"         => \$opt_rootdir,           # string

                "projectdir=s"      => \$opt_projectdir,        # string

                "releasedir=s"      => \$opt_releasedir,        # string

                "test"              => \$opt_test,              # flag

                );

#

#   Process help and manual options

#

pod2usage(-verbose => 0, -message => "Version: $VERSION")  if ($opt_help == 1  || ! $result);

pod2usage(-verbose => 1)  if ($opt_help == 2 );

pod2usage(-verbose => 2)  if ($opt_help > 2);

ErrorConfig( 'name'    => 'updateRelease',

             'verbose' => $opt_verbose );

#

#   Sanity tests

#

# Must supply at least one SbomId and the rootdir

if ( $#opt_sbom_ids == -1 || ! defined($opt_rootdir) )

{

    Error("Need -sbomid and -rootdir", "Example: -sbomid=2362 -rootdir=/export/devl/releases" ) ;

}

# Supplied rootdir must exists as a directory

elsif ( ! -d $opt_rootdir )

{

    Error("Root dir $opt_rootdir not a valid directory");

}

# Environment var GBE_DPKG must exists as a directory

elsif ( ! -d $ENV{GBE_DPKG} )

{

    Error("GBE_DPKG Environment var is not a directory") ;

}

# projectdir and releasedir must be specified together or not at all

elsif ( (  defined($opt_projectdir) && !defined($opt_releasedir) ) ||

        ( !defined($opt_projectdir) &&  defined($opt_releasedir) ) )

{

    Error("Both -projectdir and -releasedir are required if either one is specified") ;

}

# if projectdir is specified it must be a valid dir in the rootdir

elsif ( defined($opt_projectdir) && ! -d "$opt_rootdir/$opt_projectdir" )

{

    Error("The specified project directory does not exist in the root directory");

}

# if releasedir is specified it must be a valid dir in the projectdir

elsif ( defined($opt_releasedir) && ! -d "$opt_rootdir/$opt_projectdir/$opt_releasedir" )

{

    Error("The specified project/release directory does not exist in the root directory");

}

# Now lets connect and do our thing

connectRM(\$DM_DB);

if ( defined($opt_projectdir) && defined($opt_releasedir) )

{

    # If project and release dirs are specified we DONT need to get projectDir & releaseDir from DM but 

    # we do need to make sure that all sbomids specified have the same project & release in DM

    # so we call the getSbomProjectAndRelease but ignore the return values for project and release names

    getSbomProjectAndRelease($DM_DB, \@opt_sbom_ids);

}

else

{

    # If project and release dirs not specified then get the projectDir & releaseDir from DM and in addition 

    # we need to check that all sbomids specified have the same project & release in DM

    ( $opt_projectdir, $opt_releasedir ) = getSbomProjectAndRelease($DM_DB, \@opt_sbom_ids);

}

Message("Using Project [$opt_projectdir] Release [$opt_releasedir]");

# now set the projectDestDir 

$projectDestDir = "$opt_rootdir/$opt_projectdir/$opt_releasedir";

# load the config file if one exists

if ( -f "$projectDestDir/$CONFFILE" )

{

    Message("Loading release specific Config File");

    open(CONF, "<$projectDestDir/$CONFFILE") || Error("Failed to open config file");

    while( <CONF> )

    {

        chomp;

        if ( /^\s*filter\s*=\s*(.*)\s*$/ )

        {

            push(@confFilters, $1);

        }

    }

    close(CONF);

}

# validate config and command line options

if ( $#opt_filters > -1 && $#confFilters > -1 )

{

    Message("Filters supplied on Command line", @opt_filters);

    Message("Filters in release configuration file", @confFilters);

    if ( !GetYesNo("Replace Config Filters with command line Filters, be careful as this may change the copy rules") )

    {

        Error("Script terminated by user.");

    }

    @confFilters = ();

    foreach my $element (@opt_filters) {

        UniquePush (\@confFilters, $_ ) foreach  ( split(/,/, $element));

    }

    $writeConf = 1;

}

elsif ( $#opt_filters > -1 && $#confFilters == -1 )

{

    Message("Filters supplied on Command line will be written to config file for release", @opt_filters);

    @confFilters = ();

    foreach my $element (@opt_filters) {

        UniquePush (\@confFilters, $_ ) foreach  ( split(/,/, $element));

    }

    $writeConf = 1;

}

elsif ( $#opt_filters == -1 && $#confFilters > -1 )

{

    Message("Filters loaded from config file for release will be used", @confFilters) if ( IsVerbose(1) );

}

elsif ( $#opt_filters == -1 && $#confFilters == -1 )

{

    Error("No Filters supplied on command line or release config file");

}

if ( @opt_addFilters )

{

    Message ("Adding command line filters to the release config file");

    foreach my $element (@opt_addFilters) {

        UniquePush (\@confFilters, $_ ) foreach  ( split(/,/, $element));

    }

    $writeConf = 1;

}

if ( @opt_delFilters )

{

    Message ("Deleting command line filters to the release config file");

    foreach my $element (@opt_delFilters) {

        ArrayDelete (\@confFilters, $_ ) foreach  ( split(/,/, $element));

    }

    $writeConf = 1;

}

Message("Copying packages from $ENV{GBE_DPKG} to $projectDestDir");

if ( ! -d $projectDestDir )

{

    if ( defined($opt_test) )

    {

        Message("mkdir $projectDestDir");

    }

    else

    {

        eval { mkpath($projectDestDir) };

        Error("Failed to make project directory tree $projectDestDir") if ( $@ || ! -d $projectDestDir );

    }

}

my $m_sqlstr = "SELECT   packages.pkg_name, package_versions.pkg_version " .

               "FROM     deployment_manager.bom_contents, " .

               "         deployment_manager.network_nodes, " .

               "         deployment_manager.os_contents, " .

               "         deployment_manager.operating_systems, " .

               "         release_manager.package_versions, " .

               "         release_manager.packages " .

               "WHERE    network_nodes.node_id = bom_contents.node_id AND " .

               "         network_nodes.node_id = operating_systems.node_id AND " .

               "         operating_systems.os_id = os_contents.os_id AND " .

               "         os_contents.prod_id = package_versions.pv_id AND " .

               "         package_versions.pkg_id = packages.pkg_id AND " .

               "         bom_contents.bom_id " . ( $#opt_sbom_ids == 0 ? "= " . $opt_sbom_ids[0] : "IN ( " . join(",", @opt_sbom_ids) . ")" ) . " " .

               "GROUP BY packages.pkg_name, package_versions.pkg_version " .

               "ORDER BY packages.pkg_name ASC, package_versions.pkg_version ASC";

my ( $PKG_NAME, $PKG_VERSION );

my $sth = $DM_DB->prepare($m_sqlstr);

if ( defined($sth) )

{

    if ( $sth->execute( ) )

    {

        if ( $sth->rows )

        {

            while ( ( $PKG_NAME, $PKG_VERSION ) = $sth->fetchrow_array )

            {

                my $pkgDir = "$ENV{GBE_DPKG}/$PKG_NAME";

                my $srcDir = "$ENV{GBE_DPKG}/$PKG_NAME/$PKG_VERSION";

                my $dstDir = "$projectDestDir/$PKG_NAME/$PKG_VERSION";

                if ( -d "$srcDir" )

                {

                    my @filelist;

                    my $foundFiltered = 0;

                    # for each of the filter rules we glob the rule in the src pkg/version dir

                    # and if any of the globbed files dont exist in the dst dir add it to the 

                    # the filelist array of files to copy

                    foreach my $filter ( @confFilters )

                    {

                        foreach my $srcPath ( glob("$srcDir/$filter") )

                        {

                            next unless ( -f $srcPath );

                            $foundFiltered = 1;

                            $filtersUsed{$filter} = 1;

                            my $dstFile = basename($srcPath);

                            my $srcFile = $srcPath;

                            $srcFile =~ s~^$srcDir/~~;

                            push(@filelist, $srcFile) if ( ! -f "$dstDir/$dstFile" );

                        }

                    }

                    # if no files found using filters then issue warning

                    if ( $foundFiltered == 0 )

                    {

                        Warning("No Files found for Package Version $PKG_NAME/$PKG_VERSION using supplied filters");

                    }

                    # else we have found filtered files but they may already exist

                    # if so filelist may be empty, so check it b4 doing anything

                    elsif ( $#filelist > -1 )

                    {

                        Message("Copying files for package $PKG_NAME version $PKG_VERSION");

                        if ( defined($opt_test) )

                        {

                            Message( map("$_...", @filelist) );

                        }

                        else

                        {

                            eval { mkpath($dstDir) };

                            Error("Failed to make destination directory") if ( $@ || ! -d $dstDir );

                            foreach my $file ( @filelist )

                            {

                                Verbose("$file...");

                                if ( ! copy("$srcDir/$file", $dstDir) )

                                {

                                    Warning("Failed to copy $file ($!)");

                                }

                            }

                        }

                    }

                }

                elsif ( ! -d "$pkgDir" )

                {

                    # if srcDir and pkgDir dont exist then package is not in dpkg_archive so display message

                    Verbose("Skipping Package $PKG_NAME as it does not exist in dpkg_archive");

                }

                else

                {

                    # However if srcDir does not exist but pkgDir does exist then the package version is missing which maybe an issue

                    Warning("Missing Version $PKG_VERSION for Package $PKG_NAME in dpkg_archive");

                }

            }

            #

            #   Report filter elements that where not used.

            #

            my @notUsed;

            foreach my $filter ( @confFilters )

            {

                next if ( exists $filtersUsed{$filter} );

                push @notUsed, $filter

            }

            Warning ("Unused filter rules:", @notUsed )

                if ( @notUsed );

        }

        else

        {

            Error("No Boms found for Deployment Manager SBomId(s) " . join(",", @opt_sbom_ids) );

        }

        $sth->finish();

    }

    else

    {

        Error("Execute failure", $m_sqlstr );

    }

}

else

{

    Error("Prepare failure", $m_sqlstr );

}

if ( ! defined($opt_test) && $writeConf )

{

    open(CONF, ">$projectDestDir/$CONFFILE") || Error("Failed to open config file");

    print CONF map( "filter=$_\n", @confFilters);

    close CONF;

}

#-------------------------------------------------------------------------------

#   Documentation

#

=pod

=for htmltoc    DEPLOY::Update Release

=head1 NAME

jats_update_release - Extracts current package version list from Deployment Manager SBom(s)

                and copy resultant packages to release specific directory.

=head1 SYNOPSIS

  jats update_release -sbomid=xxx -rootdir=xxx [options]

 Options:

    -help              - brief help message

    -help -help        - Detailed help message

    -man               - Full documentation

    -sbomid=xxx        - Specify the Deployment Manager SBom(s) to process (Mandatory)

                       - Can be specified multiple times to combine SBoms

    -rootdir=xxx       - Specifies the root of the releases directory (Mandatory)

    -projectdir=xxx    - Override the project directory name that normally comes 

                       - from the Deployment Manager Project Name

    -releasedir=xxx    - Override the project release directory name that normally 

                       - comes from the Deployment Manager Project Release Name

    -filter=xxx        - Specifies a shell wildcard used to filter package files to copy

                       - Can be specified multiple times to use multiple filters

    -addfilter=xxx     - Add a new filter to the existing filter set

    -delfilter=xxx     - Delete a filter from the existing filter set

    -test              - Just log actions without copying files.

    -verbose           - Enable verbose output

=head1 OPTIONS

=over 8

=item B<-help>

Print a brief help message and exits.

=item B<-help -help>

Print a detailed help message with an explanation for each option.

=item B<-man>

Prints the manual page and exits.

=item B<-sbomid=xxx>

This option specifies one or more SBOM_ID's to use as the source of packages that will be copied.

The SBoms will be used to get a unique list of package/versions that can be copied from dpkg_archive.

This option is Mandatory and a minimum of one SBom must be supplied.  If more that one SBom is

supplied then all Sbom Ids must be of the same project and release with in that project.

=item B<-rootdir=xxx>

This option specifies the root directory where the packages will be copied to.

It is the top level directory in which the project/release directories will be created

and the packages copied to.

This option is mandatory and must specify a directory that exists.

=item B<-projectdir=xxx>

This option specifies the project directory that will be used in the rootdir for this release.

The project directory by default comes from the project name in Deployment Manager under which the 

SBom sbomid is under.

This option is provided to allow this script to be used against releases that are already 

populated and whose project directory does not match the Deployment Manager project name.

If the option is specified it must be used with the -releasedir option and must be a directory 

name that exists in the rootdir.

Additionally if used to populate a release then it must always be used to update the release.

=item B<-releasedir=xxx>

This option specifies the release directory that will be used in the projectdir for this release.

The release directory by default comes from the release name in Deployment Manager under 

which the SBom sbomid is under.

This option is provided to allow this script to be used against releases that are already 

populated and whose release directory does not match the Deployment Manager release name.

If the option is specified it must be used with the -projectdir option and must be a directory 

name that exists in the projectdir.

Additionally if used to populate a release then it must always be used to update the release.

=item B<-filter=xxx[,yyy]>

This option specifies a comma separated list of shell wildcard filter rule that

is used to determine which files are copied from package version directory in

GBE_DPKG to the release directory. This can be supplied multiple times to

specify rules for copying.

This must be specified on the command line the first time this command is run against a release 

and packages are copied to the project/release directory.  These values are then written to a 

config file in the project/release directory so the same values can be used on subsequent runs.  

In these subsequent runs this option need not be specified as the config items will be used, however

they can be changed by specifying them again on the command line and the config will be re-written.

The values of these will depend on what builds are required for each project.  Some examples are

--filter='*-SOLARIS10_SPARC64-[DP].pkg.gz'

--filter='*-SOLARIS10_SPARC86-[DP].pkg.gz'

--filter='*-SOLARIS10_X64-[DP].pkg.gz'

--filter='*-SOLARIS10_X86-[DP].pkg.gz'

--filter='*-WIN32.exe,*.deb'

--filter='scripts/*.sh'

=item B<-addFilter=xxx[,yyy]>

This option allows new filters to be added to an existing set of filters. This

option can be specified multiple times.

=item B<-delFilter=xxx[,yyy]>

This option deletes one or more filter rules from an existing set of filters. This

option can be specified multiple times.

=item B<-test>

This option will display what would be copied without actually copying anything

=item B<-verbose>

This option will display progress information as the program executes.

=back

=head1 DESCRIPTION

This program is used to update a projects release directory with the versions of

packages as indicated by the specified Deployment Manager SBoms.

Every invocation of this tool requires the -sbomid and -rootdir options specified.

The sbomid is used to get all the required information from Deployment Manager about

which package version are required, as well as the project name and release name under

which the Sboms are under.

The sbomid option can be specified multiple times to copy packages from multiple SBoms

to the Project Release directory.  All Sboms that are specified must be under the 

same Release under the same Project otherwise the script will abort.

The rootdir is used to specify the root location of the global releases directory.

This will be used as the root location from where the project and release specific

directories will be created to copy the files to.

By default the project and release names come from the project and release tree under

which the Sboms appear in Deployment Manager.  These values are used to populate the releasedir 

and projectdir respectively.

Alternatively you may override the projectdir and releasedir from Deployment Manager

by specifying the -projectdir=xxx and -releasedir=xxx options on the command line.

If they are used they must both be specified and must be valid directories of

rootdir/projectdir/releasedir.  These options are provided to allow existing release

directory structures to be used that don't match Deployment Managers project and release names.

The final release specific directory will be B<rootdir/projectdir/releasedir> and

all packages will be copied into this directory under a PkgName/PkgVersion directory tree.

For example running the command 

=over 8

=item jats updateRelease -sbomid=51904 -rootdir=/export/devl/releases

=back

will retrieve "BANGKOK (BKK)" as the project and "R1" as the release and will

copy packages into the "/export/devl/releases/BANGKOK (BKK)/R1" directory.

In addition to using Deployment Manager SBoms to determine which Package/Versions are

required to be copied this script also uses a set of shell wildcard filters that are

used to determine which files are actually copied when invoked.

The filter rules can be supplied on the command line if available read from a 

configuration file saved in the project/releasedir the last time the script was run

on this release directory.

One or more filter rules must be specified on the command line the first time this command 

is run against a project release directory.  These filter values are then written to a config

file in the project/release directory so the same values can be used on subsequent runs.  

In subsequent runs the filter rules will be loaded from the config file and need not be specified 

on the command line, however the filter rules in the config file can be changed by specifying 

them again on the command line and the config will be re-written.

=cut