Rev 4086 | Rev 5710 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | RSS feed
######################################################################### Copyright ( C ) 2010 ERG Limited, All rights reserved## Module name : jats_update_release.pl# Module type : Makefile system# Compiler(s) : Perl# Environment(s): jats build system## Description : Extracts current package version list from Deployment Manager# SBom(s) and copies resultant packages to release specific# directory.#......................................................................#require 5.008_002;use File::Basename;use File::Copy;use File::Path;use strict;use warnings;use JatsEnv;use JatsError;use JatsSystem;use JatsRmApi;use ArrayHashUtils;use DBI;use Getopt::Long;use Pod::Usage; # required for help supportuse Storable qw (dclone);## Config Options#my $VERSION = "1.0.0"; # Update thismy $opt_help = 0;my $opt_verbose = $ENV{'GBE_VERBOSE'}; # Allow global verbosemy @opt_sbom_ids;my $opt_rootdir;my @opt_filters;my $opt_projectdir;my $opt_releasedir;my $opt_test;my @opt_addFilters;my @opt_delFilters;## Constants#my $CONFFILE = ".updateRelease";## Data Base Interface#my $DM_DB;## The directory we copy to.#my $projectDestDir;## Configuration file vars#my @confFilters;my $writeConf = 0;my %filtersUsed;# -------------------------------------------------------------------------sub GetYesNo## -------------------------------------------------------------------------{my ($question) = @_;my ($u_tmp) = "";Question ("$question, (default: y) [y,n]: ");while ( <STDIN> ){$u_tmp = $_;chomp($u_tmp);return 1if ( "$u_tmp" eq "" );if( $u_tmp =~ /[yn]{1}/i ){return ( "$u_tmp" eq "y" );}else{Question("Please re-enter response? (default: y) [y,n]: ");}}}#==============================================================================# getSbomProjectAndRelease# Returns the Project Name and Release for supplied SBoms# Calls Error and exists on error condition#==============================================================================sub getSbomProjectAndRelease{my ( $DB, $sboms ) = @_;my ( $bom, $proj, $rel, $lastProj, $lastRel );Error("getSbomProjectAndRelease: SBom Parameter Error, must pass array") if ( ref($sboms) ne "ARRAY" );# create a hash of sbom values so we can test after if any sboms could not be foundmy %sbomIdx = map { $_ => 1 } @{$sboms};my $m_sqlstr = "SELECT boms.bom_id, dm_projects.proj_name, branches.branch_name " ."FROM deployment_manager.boms, deployment_manager.branches, deployment_manager.dm_projects " ."WHERE branches.branch_id = boms.branch_id AND " ." dm_projects.proj_id = branches.proj_id AND " ." boms.bom_id " . ( $#{$sboms} == 0 ? "= " . $sboms->[0] : "IN ( " . join(",", @{$sboms}) . ")" );my $sth = $DB->prepare($m_sqlstr);if ( defined($sth) ){if ( $sth->execute( ) ){if ( $sth->rows ){while ( ( $bom, $proj, $rel ) = $sth->fetchrow_array ){if ( ! defined($proj) ){Error("getSbomProjectAndRelease: NULL Project Name from Deployment Manager Sbom : $bom");}elsif ( ! defined($rel) ){Error("getSbomProjectAndRelease: NULL Release Tag Name from Deployment Manager Sbom : $bom");}elsif ( defined($lastProj) && $proj ne $lastProj ){Error("getSbomProjectAndRelease: SBom Id [$bom] is in a different project [$proj]", "All SBom Id's must all be part of the same Deployment Manager Project");}elsif ( defined($lastRel) && $rel ne $lastRel ){Error("getSbomProjectAndRelease: SBom Id [$bom] is in a different project release [$rel]", "All SBom Id's must all be part of the same Deployment Manager Project Release");}$lastProj = $proj;$lastRel = $rel;# delete sbom from idx, any remaining after loop will indicate we have an sbom that could not founddelete($sbomIdx{$bom});}my @sbomsNotFound = keys %sbomIdx;if ( $#sbomsNotFound > -1 ){Error("getSbomProjectAndRelease: Could not find details for the following SBomId(s) " . join(",", @sbomsNotFound) );}}else{Error("getSbomProjectAndRelease: No SBom(s) found for Deployment Manager SBomId(s) " . join(",", @{$sboms}) );}$sth->finish();}else{Error("getSbomProjectAndRelease: Execute failure", $m_sqlstr );}}else{Error("getSbomProjectAndRelease: Prepare failure", $m_sqlstr );}return ( $lastProj, $lastRel );} # getSbomProjectAndRelease#-------------------------------------------------------------------------------# Function : Main## Description : Main entry point# Parse user options## Inputs :## Returns :#my $result = GetOptions ("help:+" => \$opt_help, # flag, multiple use allowed"manual:3" => \$opt_help, # flag, multiple use allowed"verbose:+" => \$opt_verbose, # flag"sbomid|sbom_id=s" => \@opt_sbom_ids, # multiple numbers"filter=s" => \@opt_filters, # multiple strings"addfilter=s" => \@opt_addFilters, # multiple strings"delfilter=s" => \@opt_delFilters, # multiple strings"rootdir=s" => \$opt_rootdir, # string"projectdir=s" => \$opt_projectdir, # string"releasedir=s" => \$opt_releasedir, # string"test" => \$opt_test, # flag);## Process help and manual options#pod2usage(-verbose => 0, -message => "Version: $VERSION") if ($opt_help == 1 || ! $result);pod2usage(-verbose => 1) if ($opt_help == 2 );pod2usage(-verbose => 2) if ($opt_help > 2);ErrorConfig( 'name' => 'updateRelease','verbose' => $opt_verbose );## Sanity tests## Must supply at least one SbomId and the rootdirif ( $#opt_sbom_ids == -1 || ! defined($opt_rootdir) ){Error("Need -sbomid and -rootdir", "Example: -sbomid=2362 -rootdir=/export/devl/releases" ) ;}# Supplied rootdir must exists as a directoryelsif ( ! -d $opt_rootdir ){Error("Root dir $opt_rootdir not a valid directory");}# Environment var GBE_DPKG must exists as a directoryelsif ( ! -d $ENV{GBE_DPKG} ){Error("GBE_DPKG Environment var is not a directory") ;}# projectdir and releasedir must be specified together or not at allelsif ( ( defined($opt_projectdir) && !defined($opt_releasedir) ) ||( !defined($opt_projectdir) && defined($opt_releasedir) ) ){Error("Both -projectdir and -releasedir are required if either one is specified") ;}# if projectdir is specified it must be a valid dir in the rootdirelsif ( defined($opt_projectdir) && ! -d "$opt_rootdir/$opt_projectdir" ){Error("The specified project directory does not exist in the root directory");}# if releasedir is specified it must be a valid dir in the projectdirelsif ( defined($opt_releasedir) && ! -d "$opt_rootdir/$opt_projectdir/$opt_releasedir" ){Error("The specified project/release directory does not exist in the root directory");}## This command is destined to be used in a directory where group permissions# are important. Ensure that the user is not killing group access#umask 0002;# Now lets connect and do our thingconnectRM(\$DM_DB);if ( defined($opt_projectdir) && defined($opt_releasedir) ){# If project and release dirs are specified we DONT need to get projectDir & releaseDir from DM but# we do need to make sure that all sbomids specified have the same project & release in DM# so we call the getSbomProjectAndRelease but ignore the return values for project and release namesgetSbomProjectAndRelease($DM_DB, \@opt_sbom_ids);}else{# If project and release dirs not specified then get the projectDir & releaseDir from DM and in addition# we need to check that all sbomids specified have the same project & release in DM( $opt_projectdir, $opt_releasedir ) = getSbomProjectAndRelease($DM_DB, \@opt_sbom_ids);}Message("Using Project [$opt_projectdir] Release [$opt_releasedir]");# now set the projectDestDir$projectDestDir = "$opt_rootdir/$opt_projectdir/$opt_releasedir";# load the config file if one existsif ( -f "$projectDestDir/$CONFFILE" ){Message("Loading release specific Config File");open(CONF, "<$projectDestDir/$CONFFILE") || Error("Failed to open config file");while( <CONF> ){chomp;if ( /^\s*filter\s*=\s*(.*)\s*$/ ){push(@confFilters, $1);}}close(CONF);}# validate config and command line optionsif ( $#opt_filters > -1 && $#confFilters > -1 ){Message("Filters supplied on Command line", @opt_filters);Message("Filters in release configuration file", @confFilters);if ( !GetYesNo("Replace Config Filters with command line Filters, be careful as this may change the copy rules") ){Error("Script terminated by user.");}@confFilters = ();foreach my $element (@opt_filters) {UniquePush (\@confFilters, $_ ) foreach ( split(/,/, $element));}$writeConf = 1;}elsif ( $#opt_filters > -1 && $#confFilters == -1 ){Message("Filters supplied on Command line will be written to config file for release", @opt_filters);@confFilters = ();foreach my $element (@opt_filters) {UniquePush (\@confFilters, $_ ) foreach ( split(/,/, $element));}$writeConf = 1;}elsif ( $#opt_filters == -1 && $#confFilters > -1 ){Message("Filters loaded from config file for release will be used", @confFilters) if ( IsVerbose(1) );}elsif ( $#opt_filters == -1 && $#confFilters == -1 ){Error("No Filters supplied on command line or release config file");}if ( @opt_addFilters ){Message ("Adding command line filters to the release config file");foreach my $element (@opt_addFilters) {UniquePush (\@confFilters, $_ ) foreach ( split(/,/, $element));}$writeConf = 1;}if ( @opt_delFilters ){Message ("Deleting command line filters to the release config file");foreach my $element (@opt_delFilters) {ArrayDelete (\@confFilters, $_ ) foreach ( split(/,/, $element));}$writeConf = 1;}Message("Copying packages from $ENV{GBE_DPKG} to $projectDestDir");if ( ! -d $projectDestDir ){if ( defined($opt_test) ){Message("mkdir $projectDestDir");}else{eval { mkpath($projectDestDir) };Error("Failed to make project directory tree $projectDestDir") if ( $@ || ! -d $projectDestDir );}}my $m_sqlstr = "SELECT packages.pkg_name, package_versions.pkg_version " ."FROM deployment_manager.bom_contents, " ." deployment_manager.network_nodes, " ." deployment_manager.os_contents, " ." deployment_manager.operating_systems, " ." release_manager.package_versions, " ." release_manager.packages " ."WHERE network_nodes.node_id = bom_contents.node_id AND " ." network_nodes.node_id = operating_systems.node_id AND " ." operating_systems.os_id = os_contents.os_id AND " ." os_contents.prod_id = package_versions.pv_id AND " ." package_versions.pkg_id = packages.pkg_id AND " ." bom_contents.bom_id " . ( $#opt_sbom_ids == 0 ? "= " . $opt_sbom_ids[0] : "IN ( " . join(",", @opt_sbom_ids) . ")" ) . " " ."GROUP BY packages.pkg_name, package_versions.pkg_version " ."ORDER BY packages.pkg_name ASC, package_versions.pkg_version ASC";my ( $PKG_NAME, $PKG_VERSION );my $sth = $DM_DB->prepare($m_sqlstr);if ( defined($sth) ){if ( $sth->execute( ) ){if ( $sth->rows ){while ( ( $PKG_NAME, $PKG_VERSION ) = $sth->fetchrow_array ){my $pkgDir = "$ENV{GBE_DPKG}/$PKG_NAME";my $srcDir = "$ENV{GBE_DPKG}/$PKG_NAME/$PKG_VERSION";my $dstDir = "$projectDestDir/$PKG_NAME/$PKG_VERSION";if ( -d "$srcDir" ){my @filelist;my $foundFiltered = 0;# for each of the filter rules we glob the rule in the src pkg/version dir# and if any of the globbed files dont exist in the dst dir add it to the# the filelist array of files to copyforeach my $filter ( @confFilters ){foreach my $srcPath ( glob("$srcDir/$filter") ){next unless ( -f $srcPath );$foundFiltered = 1;$filtersUsed{$filter} = 1;my $dstFile = basename($srcPath);my $srcFile = $srcPath;$srcFile =~ s~^$srcDir/~~;push(@filelist, $srcFile) if ( ! -f "$dstDir/$dstFile" );}}# if no files found using filters then issue warningif ( $foundFiltered == 0 ){Warning("No Files found for Package Version $PKG_NAME/$PKG_VERSION using supplied filters");}# else we have found filtered files but they may already exist# if so filelist may be empty, so check it b4 doing anythingelsif ( $#filelist > -1 ){Message("Copying files for package $PKG_NAME version $PKG_VERSION");if ( defined($opt_test) ){Message( map("$_...", @filelist) );}else{eval { mkpath($dstDir) };Error("Failed to make destination directory") if ( $@ || ! -d $dstDir );foreach my $file ( @filelist ){Verbose("$file...");if ( ! copy("$srcDir/$file", $dstDir) ){Warning("Failed to copy $file ($!)");}}}}}elsif ( ! -d "$pkgDir" ){# if srcDir and pkgDir dont exist then package is not in dpkg_archive so display messageVerbose("Skipping Package $PKG_NAME as it does not exist in dpkg_archive");}else{# However if srcDir does not exist but pkgDir does exist then the package version is missing which maybe an issueWarning("Missing Version $PKG_VERSION for Package $PKG_NAME in dpkg_archive");}}## Report filter elements that where not used.#my @notUsed;foreach my $filter ( @confFilters ){next if ( exists $filtersUsed{$filter} );push @notUsed, $filter}Warning ("Unused filter rules:", @notUsed )if ( @notUsed );}else{Error("No Boms found for Deployment Manager SBomId(s) " . join(",", @opt_sbom_ids) );}$sth->finish();}else{Error("Execute failure", $m_sqlstr );}}else{Error("Prepare failure", $m_sqlstr );}if ( ! defined($opt_test) && $writeConf ){open(CONF, ">$projectDestDir/$CONFFILE") || Error("Failed to open config file");print CONF map( "filter=$_\n", @confFilters);close CONF;}#-------------------------------------------------------------------------------# Documentation#=pod=for htmltoc DEPLOY::Update Release=head1 NAMEjats_update_release - Extracts current package version list from Deployment Manager SBom(s)and copy resultant packages to release specific directory.=head1 SYNOPSISjats update_release -sbomid=xxx -rootdir=xxx [options]Options:-help - brief help message-help -help - Detailed help message-man - Full documentation-sbomid=xxx - Specify the Deployment Manager SBom(s) to process (Mandatory)- Can be specified multiple times to combine SBoms-rootdir=xxx - Specifies the root of the releases directory (Mandatory)-projectdir=xxx - Override the project directory name that normally comes- from the Deployment Manager Project Name-releasedir=xxx - Override the project release directory name that normally- comes from the Deployment Manager Project Release Name-filter=xxx - Specifies a shell wildcard used to filter package files to copy- Can be specified multiple times to use multiple filters-addfilter=xxx - Add a new filter to the existing filter set-delfilter=xxx - Delete a filter from the existing filter set-test - Just log actions without copying files.-verbose - Enable verbose output=head1 OPTIONS=over 8=item B<-help>Print a brief help message and exits.=item B<-help -help>Print a detailed help message with an explanation for each option.=item B<-man>Prints the manual page and exits.=item B<-sbomid=xxx>This option specifies one or more SBOM_ID's to use as the source of packages that will be copied.The SBoms will be used to get a unique list of package/versions that can be copied from dpkg_archive.This option is Mandatory and a minimum of one SBom must be supplied. If more that one SBom issupplied then all Sbom Ids must be of the same project and release with in that project.=item B<-rootdir=xxx>This option specifies the root directory where the packages will be copied to.It is the top level directory in which the project/release directories will be createdand the packages copied to.This option is mandatory and must specify a directory that exists.=item B<-projectdir=xxx>This option specifies the project directory that will be used in the rootdir for this release.The project directory by default comes from the project name in Deployment Manager under which theSBom sbomid is under.This option is provided to allow this script to be used against releases that are alreadypopulated and whose project directory does not match the Deployment Manager project name.If the option is specified it must be used with the -releasedir option and must be a directoryname that exists in the rootdir.Additionally if used to populate a release then it must always be used to update the release.=item B<-releasedir=xxx>This option specifies the release directory that will be used in the projectdir for this release.The release directory by default comes from the release name in Deployment Manager underwhich the SBom sbomid is under.This option is provided to allow this script to be used against releases that are alreadypopulated and whose release directory does not match the Deployment Manager release name.If the option is specified it must be used with the -projectdir option and must be a directoryname that exists in the projectdir.Additionally if used to populate a release then it must always be used to update the release.=item B<-filter=xxx[,yyy]>This option specifies a comma separated list of shell wildcard filter rule thatis used to determine which files are copied from package version directory inGBE_DPKG to the release directory. This can be supplied multiple times tospecify rules for copying.This must be specified on the command line the first time this command is run against a releaseand packages are copied to the project/release directory. These values are then written to aconfig file in the project/release directory so the same values can be used on subsequent runs.In these subsequent runs this option need not be specified as the config items will be used, howeverthey can be changed by specifying them again on the command line and the config will be re-written.The values of these will depend on what builds are required for each project. Some examples are--filter='*-SOLARIS10_SPARC64-[DP].pkg.gz'--filter='*-SOLARIS10_SPARC86-[DP].pkg.gz'--filter='*-SOLARIS10_X64-[DP].pkg.gz'--filter='*-SOLARIS10_X86-[DP].pkg.gz'--filter='*-WIN32.exe,*.deb'--filter='scripts/*.sh'=item B<-addFilter=xxx[,yyy]>This option allows new filters to be added to an existing set of filters. Thisoption can be specified multiple times.=item B<-delFilter=xxx[,yyy]>This option deletes one or more filter rules from an existing set of filters. Thisoption can be specified multiple times.=item B<-test>This option will display what would be copied without actually copying anything=item B<-verbose>This option will display progress information as the program executes.=back=head1 DESCRIPTIONThis program is used to update a projects release directory with the versions ofpackages as indicated by the specified Deployment Manager SBoms.Every invocation of this tool requires the -sbomid and -rootdir options specified.The sbomid is used to get all the required information from Deployment Manager aboutwhich package version are required, as well as the project name and release name underwhich the Sboms are under.The sbomid option can be specified multiple times to copy packages from multiple SBomsto the Project Release directory. All Sboms that are specified must be under thesame Release under the same Project otherwise the script will abort.The rootdir is used to specify the root location of the global releases directory.This will be used as the root location from where the project and release specificdirectories will be created to copy the files to.By default the project and release names come from the project and release tree underwhich the Sboms appear in Deployment Manager. These values are used to populate the releasedirand projectdir respectively.Alternatively you may override the projectdir and releasedir from Deployment Managerby specifying the -projectdir=xxx and -releasedir=xxx options on the command line.If they are used they must both be specified and must be valid directories ofrootdir/projectdir/releasedir. These options are provided to allow existing releasedirectory structures to be used that don't match Deployment Managers project and release names.The final release specific directory will be B<rootdir/projectdir/releasedir> andall packages will be copied into this directory under a PkgName/PkgVersion directory tree.For example running the command=over 8=item jats updateRelease -sbomid=51904 -rootdir=/export/devl/releases=backwill retrieve "BANGKOK (BKK)" as the project and "R1" as the release and willcopy packages into the "/export/devl/releases/BANGKOK (BKK)/R1" directory.In addition to using Deployment Manager SBoms to determine which Package/Versions arerequired to be copied this script also uses a set of shell wildcard filters that areused to determine which files are actually copied when invoked.The filter rules can be supplied on the command line if available read from aconfiguration file saved in the project/releasedir the last time the script was runon this release directory.One or more filter rules must be specified on the command line the first time this commandis run against a project release directory. These filter values are then written to a configfile in the project/release directory so the same values can be used on subsequent runs.In subsequent runs the filter rules will be loaded from the config file and need not be specifiedon the command line, however the filter rules in the config file can be changed by specifyingthem again on the command line and the config will be re-written.=cut