Rev 7383 | Blame | Compare with Previous | Last modification | View Log | RSS feed
######################################################################### Copyright (c) VIX TECHNOLOGY (AUST) LTD## Module name : process_release_notes.pl# Module type : JATS Utility# Compiler(s) : Perl# Environment(s): jats## Description : Poll RM and generate Release Notes as required# Must run on the same machine as the package archive# It directly manipulates the target package# Requires access to the machines sudo mechism## Usage : See POD at the end of this file##......................................................................#require 5.008_002;use strict;use warnings;use JatsError;use JatsSystem;use Getopt::Long;use Pod::Usage; # required for help supportuse File::Temp qw/ tempfile tempdir /;use LWP::UserAgent;use HTTP::Request::Common 'POST';use Fcntl ':flock';use FindBin; # Determine the current directoryuse JatsEnv;use JatsRmApi;use FileUtils;use DBI;my $VERSION = "1.0.0"; # Update thismy $GBE_DPKG = $ENV{GBE_DPKG}; # Sanitised by Jatsmy $GBE_RM_URL = $ENV{GBE_RM_URL};my $RM_DB;my $tempdir;my $sudoUtilsDir = '/home/releasem/sbin';my $lockFile = '/tmp/JATSRN_LOCK';my $lockFile1 = '/tmp/JATSRN_LOCK1';my $scriptDir = "$FindBin::Bin";my $maxRunTime = 5 * 60; # Seconds. Time to first warningmy $maxRunTime1 = 5 * 60; # Seconds. Time to following warnings## Options#my $opt_verbose = 0;my $opt_help = 0;my $opt_age = 2;my $opt_keepTemp = 0;my $opt_status = 0;my $opt_pvid;## Package information#my @Packages;my @packageItems = qw( PV_ID -LINK_PV_ID RTAG_ID NAME VERSION PKG_ID PROJ_ID RNINFO BUILD_TYPE BS_ID);my %rmConfig;#-------------------------------------------------------------------------------# Function : Main Entry## Description :## Inputs :## Returns :#my $result = GetOptions ('help+' => \$opt_help, # flag, multiple use allowed'manual:3' => \$opt_help, # flag'verbose:+' => \$opt_verbose, # flag'age:i' => \$opt_age, # Number'keeptemp!' => \$opt_keepTemp, # flag'status+' => \$opt_status, # flag'pvid:i' => \$opt_pvid, # Number);## Process help and manual options#pod2usage(-verbose => 0, -message => "Version: $VERSION") if ($opt_help == 1 || ! $result);pod2usage(-verbose => 1) if ($opt_help == 2);pod2usage(-verbose => 2) if ($opt_help > 2);ErrorConfig( 'name' =>'RelNotes','verbose' => $opt_verbose );$opt_verbose-- if ($opt_verbose > 0);## Sanity Test#Error("Not running on a Unix System") unless ($ENV{GBE_UNIX});Error("Sudo Utils not found: $sudoUtilsDir") unless ( -d $sudoUtilsDir);Warning("Not running as a suitable user" ) unless ($ENV{USER} eq 'buildadm' || $ENV{USER} eq 'releasem' || $ENV{USER} eq 'pkgadm');EnvImport ('GBE_PERL');## Ensure only one instance is running# PerlMonks say to use $0 as the lock file, but that did not work.# Perhaps the file was open in an editor# Solution: Create my own file# Generate an error if the script has been running for too long#makeFile($lockFile);makeFile($lockFile1) unless -f $lockFile1;open (my $self, '<', $lockFile) || Error("Couldn't open self: $!");unless ( flock ($self, (LOCK_EX | LOCK_NB)) ){my $runTime = getFileTime($lockFile);my $runTime1 = getFileTime($lockFile1);if ( $runTime > $maxRunTime && $runTime1 >= $maxRunTime1) {unlink $lockFile1;makeFile($lockFile1);Error("Script has been running for too long.: $runTime(s)");}exit(1);}## Init parameters#$tempdir = tempdir('/tmp/JATSRN_XXXXXX',CLEANUP => !$opt_keepTemp);Warning("Need to remove tempdir: " . DisplayPath($tempdir)) if $opt_keepTemp;Verbose("Tempdir:" , DisplayPath($tempdir));Verbose("scriptDir:" , DisplayPath($scriptDir));# Force GBE_ABT. Currently the Release Manager proxy password only works# When used in this mode$ENV{GBE_ABT} = 1 unless defined $ENV{GBE_ABT};## User RW password, if provided#$ENV{GBE_RM_USERNAME} = $ENV{GBE_RM_USERNAME_RW} if (exists $ENV{GBE_RM_USERNAME_RW});$ENV{GBE_RM_PASSWORD} = $ENV{GBE_RM_PASSWORD_RW} if (exists $ENV{GBE_RM_PASSWORD_RW});if ($ENV{GBE_ABT} && $ENV{GBE_RM_USERNAME} && $ENV{GBE_RM_USERNAME} !~ m~]$~ ){Verbose("Access RM database as proxy user");$ENV{GBE_RM_USERNAME} = $ENV{GBE_RM_USERNAME} . '[release_manager]';}## Interogate the RM database and locate packages that need to be processed#$opt_pvid ? LocatePackagesByPvid() : LocatePackages();Verbose("Package to process:" . scalar @Packages);#DebugDumpData("Packages", \@Packages);## Process each found entry#foreach my $entry ( @Packages){GenerateReleaseNote( $entry);}unlink $lockFile;unlink $lockFile1;exit 0;#-------------------------------------------------------------------------------# Function : LocatePackages## Description : Locate packages in need of Release Notes# The hard part is handling patches# The sql will# 1) Locate packages that may need a release note# 2) Trace back to the most likly Release that the pakage is in# For patches - it may be in many released# Use the latest## Inputs : None## Returns : Nothing# Populates the @Packages array#sub LocatePackages{my $m_sqlstr ="select * from (select aa.*,rc.RTAG_ID,pkg.PKG_NAME,pv.PKG_VERSION,pv.pkg_id,rt.PROJ_ID,pv.release_notes_info,pv.build_type,pv.BS_ID,TRUNC(sysdate - pv.modified_stamp ) AS age,row_number() OVER (PARTITION BY aa.pv_id, pkg.pkg_name, pv.pkg_version ORDER BY rc.rtag_id desc) rnfrom (SELECT pv.pv_id, NVL( pp.PV_ID, pv.pv_id) as LINK_PV_IDFROM release_manager.package_versions pv, release_manager.PACKAGE_PATCHES ppWHERE (release_notes_info IS NULLOR (release_notes_info LIKE 'MSG:%'AND release_notes_info != 'MSG:5' ))AND pv.dlocked = 'Y'AND pv.modified_stamp > (sysdate - $opt_age)AND pv.pv_id = pp.PATCH_ID(+)) aa,release_manager.package_versions pv,release_manager.packages pkg,release_manager.RELEASE_CONTENT rc,release_manager.RELEASE_TAGS rtwhere aa.PV_ID = pv.PV_IDAND pv.PKG_ID = pkg.PKG_IDand RC.pv_id = aa.LINK_PV_IDAND rt.RTAG_ID = rc.RTAG_ID) where RN = 1";populateArrayFromSql('LocatePackages', \@Packages, $m_sqlstr, \@packageItems);# DebugDumpData("Packages", \@Packages);}#-------------------------------------------------------------------------------# Function : LocatePackagesByPvid## Description : Locate one package, as specified by its PVID# This mode is only used in testing## Inputs : Global: $opt_pvid## Returns : Populate the @Packages array#sub LocatePackagesByPvid{# Extract: PV_ID LINK_PV_ID RTAG_ID NAME VERSION PKG_ID PROJ_ID RNINFO AGEmy $m_sqlstr = "SELECT pv.pv_id,pv.pv_id,rc.rtag_id,pkg.pkg_name,pv.pkg_version,pv.pkg_id,rt.proj_id,pv.release_notes_info,pv.build_type,pv.BS_IDFROM release_manager.package_versions pv,release_manager.release_content rc,release_manager.packages pkg,release_manager.release_tags rtWHERE pv.pv_id = rc.pv_idAND pkg.pkg_id = pv.pkg_idAND rc.rtag_id = rt.rtag_idAND pv.pv_id = $opt_pvid";populateArrayFromSql('LocatePackagesByPvid', \@Packages, $m_sqlstr, \@packageItems);#DebugDumpData("Packages", \@Packages);}#-------------------------------------------------------------------------------# Function : GenerateReleaseNote## Description : Generate One Release Note# Invoke several JATS utilities to do the hard work## Inputs : $entry - Hash of useful infomation## Returns : Nothing# Will exit on error#sub GenerateReleaseNote{my ($entry) = @_;my $outfile;my $rv;my @args;#DebugDumpData("Entry", $entry);Message("-------------------- $entry->{NAME}, $entry->{VERSION}") if $opt_status || $opt_verbose;my $pkgBase = catdir($GBE_DPKG, $entry->{NAME}, $entry->{VERSION});Verbose("Processing: $pkgBase");## Handle special class of packages# Manually Built# No Build Standard# Not in dpkg_archive# Example: unit_test_br_kcm_ct_saftp# This type of package appears to be a place holder# Still needs some level of processing#if ($entry->{BUILD_TYPE} eq 'M' && $entry->{BS_ID} == 3 && ! -d $pkgBase){Verbose ("Placeholder package detected");#DebugDumpData("Entry", $entry);updatePlaceHolderRmNoteInfo($entry);$rv = 0;}else{##unless (-d $pkgBase) {Warning("Package not in archive: $pkgBase");return;}## Make the target Package Version Writable and Ensure that the doc directory# exists. This is done via a script that is invoked with SUDO, partially to# interwork with the existing system#runSudoCommand('make_writable', 'dpkg_archive', $entry->{NAME},$entry->{VERSION});runSudoCommand('make_docFolder', 'dpkg_archive', $entry->{NAME},$entry->{VERSION}, 'doc');## Get the basic data required for the Release Note# jats jats_get_releasenote_data.pl -pvid=1002782#$outfile = catdir($tempdir, join('_',$entry->{NAME},$entry->{VERSION})) . '.xml';$rv = localTool('jats_get_releasenote_data', '-verbose', $opt_verbose, '-pvid', $entry->{PV_ID}, '-outfile', $outfile );## Generate the actual release note and update the Release Manager# jats jats_gen_releasenote.pl -releasenote=mcrdemo_1.2.3157.cr.xml -UpdateRmFilesunless ($rv){$rv = localTool('jats_gen_releasenote.pl', '-verbose', $opt_verbose, '-UpdateRmFiles', '-releasenote', $outfile);}## Make the target PackageVersion ReadOnly#runSudoCommand('make_readonly', 'dpkg_archive', $entry->{NAME},$entry->{VERSION});}# Release note complete - or failed# Signal End of Package Processing - which will trigger blat# Do this even if we have a Release Note Failure## make_release_changed# archive=archive-path# pkg_name="package-name"# pkg_version="package-version"# rtag_id=release-tag-id# pkg_id=package-id# pv_id=package-version-id# proj_id=project-id# mode_id=change-mode-id (1 pkg added, 2 pkg removed, 3 pkg released)# 3 = enumRELEASE_CHANGE_MODE_PKG_RELEASED## Note: The ReleasedPackagereport tool parses the make_released_changed entries# It requires quotes on the pkg_name and pkg_version for correct parsing#push @args, 'archive=dpkg_archive';push @args, 'mode_id=3';push @args, 'pkg_name="' . $entry->{NAME} . '"';push @args, 'pkg_version="' . $entry->{VERSION} . '"';push @args, 'rtag_id=' . $entry->{RTAG_ID};push @args, 'pkg_id=' . $entry->{PKG_ID};push @args, 'pv_id=' . $entry->{PV_ID};push @args, 'proj_id=' . $entry->{PROJ_ID};runSudoCommand('make_release_changed', @args);## Email interested users#notifyInterestedUsers($entry);Error ("Did not generate Release Note: $entry->{NAME}, $entry->{VERSION}") if ($rv);}#-------------------------------------------------------------------------------# Function : updatePlaceHolderRmNoteInfo## Description : Insert Release Note Info into the Release Manager Database# This has the side effect that RM will see the Release Note# as being fully generated.## Used only by special PlaceHolder packages## Inputs : $entry - Hash of useful infomation## Returns :#sub updatePlaceHolderRmNoteInfo{my ($entry) = @_;executeRmQuery ('updateRmNoteInfo','update release_manager.package_versions set release_notes_info=\'MSG:5\' where pv_id=' . $entry->{PV_ID});}#-------------------------------------------------------------------------------# Function : localTool## Description : Run a jats tool from the same directory as this script## Inputs : $name - Name of the script# ... - Arguments for the command## Returns :#sub localTool{my $cmd = shift;$cmd .= '.pl' unless ( $cmd =~ m~\.pl$~i );$cmd = catdir($scriptDir, $cmd);Error ("Command not found: $cmd") unless -f $cmd;return System ( '--NoShell', $::GBE_PERL, $cmd, @_ );}#-------------------------------------------------------------------------------# Function : notifyInterestedUsers## Description : A user can register interest in a package (name) being built in a# specific Project# Perhaps this should be done by the build tool, but at the moment# this release note generation process is really a part of the build## NOTE: Similar code exists in the RM web server to handle manually# released packages.## Inputs : $entry - Hash of useful infomation## Returns :#sub notifyInterestedUsers{my ($entry) = @_;my @userList;## Determine interested users#my $m_sqlstr = "SELECT U.user_email, prj.proj_name, rt.rtag_name, pkg.pkg_name, pv.pkg_version ,pv.pv_id, rt.rtag_idFROM RELEASE_MANAGER.PACKAGE_INTEREST PI,RELEASE_MANAGER.PACKAGES PKG,RELEASE_MANAGER.PACKAGE_VERSIONS PV,RELEASE_MANAGER.USERS U,RELEASE_MANAGER.RELEASE_TAGS RT,RELEASE_MANAGER.PROJECTS PRJWHERE PKG.PKG_ID = PI.PKG_IDAND RT.RTAG_ID = $entry->{RTAG_ID}AND PV.PV_ID = $entry->{PV_ID}AND PV.PKG_ID = PKG.PKG_IDAND PRJ.PROJ_ID = RT.PROJ_IDAND PRJ.PROJ_ID = PI.PROJ_IDAND U.USER_ID = PI.USER_ID";my @items = qw( USER_EMAIL PROJ_NAME RTAG_NAME PKG_NAME PKG_VERSION PV_ID RTAG_ID );populateArrayFromSql('notifyInterestedUsers', \@userList, $m_sqlstr, \@items);## Do we have something to do#return unless (@userList);#DebugDumpData("userList", \@userList);# Ensure we have basic emailing information#getRmConfig();## Send Email to all the required users# Note: This emailer requires a sendmail utility#foreach my $entry ( @userList){#$entry->{USER_EMAIL} = 'dpurdie@vixtechnology.com';#Debug0("Sending email to David Purdie indead of real user");my $rnUrl = $GBE_RM_URL . "/fixed_issues.asp?pv_id=$entry->{PV_ID}&rtag_id=$entry->{RTAG_ID}";my $subject = "Package Release Notification: $entry->{PKG_NAME} $entry->{PKG_VERSION}";my $content = "Version <a href='$rnUrl'>$entry->{PKG_VERSION}</a> of Package $entry->{PKG_NAME} in Project $entry->{PROJ_NAME} on Release Branch $entry->{RTAG_NAME} has been released.";$content .= "<p>Package Link: $rnUrl";$content .= "<p>You have received this email as a result of your interest in this package. If you do not wish to receive further emails then remove your package interest from the notifications area in Release Manager.";my $req = POST( 'mailto:' . $entry->{USER_EMAIL},From => $rmConfig{'BUILD FAILURE MAIL SENDER'},CC => 'buildadm@vixtechnology.com',Date => scalar localtime,Subject => $subject,Content_Type => qq(text/html),Content => $content,);my $response = LWP::UserAgent->new->request($req);if ($response->code != 202){Warning("Email Send Error: $response->code");#DebugDumpData("Response", \$response);}}}#-------------------------------------------------------------------------------# Function : getRmConfig## Description : Get Basic config from Release Manager# Only do it once## Just need:# 'BUILD FAILURE MAIL SENDER'# 'MAIL SERVER'## Inputs : None## Returns : Populates a global hash#sub getRmConfig{return if keys(%rmConfig) > 0;my $m_sqlstr = "SELECT * from BUILD_SERVICE_CONFIG";performSqlQueryCallback('getRmConfig',$m_sqlstr,sub {my ($pRow) = @_;$rmConfig{ $pRow->[0] } = $pRow->[1];});#DebugDumpData("rmConfig", \%rmConfig);}#-------------------------------------------------------------------------------# Function : runSudoCommand## Description : Run a Unix command as root via the sodo system# Requires that target commands be available# Requires sudo to be configured to allow this user to run them## Notes : The sudoers file on the archive server needs to be configured to allow:# This user to run programs from /home/releasem/sbin without a password# The users sudo credentials must not timeout (or be used)## Inputs : prog - Command to run# arguments - Command arguments## Returns : Nothing#sub runSudoCommand{my ($prog, @arguments) = @_;my $cmd;my $rv;## Construct command#$cmd = catdir($sudoUtilsDir, $prog);$rv = System('--NoShell','sudo','-n', $cmd, @arguments);Warning("SudoCmd Result: $prog: $rv") if ($rv);}#-------------------------------------------------------------------------------# Function : performSqlQueryCallback## Description : Perform a general Sql query and invoke a user function for# each row of results## Inputs : $fname - Name of query for error reporting# $m_sqlstr - Query string# $f_process - Function called for each row in the result# Use closure to have callback modify other data## Returns : Number of rows found#sub performSqlQueryCallback{my ($fname, $m_sqlstr, $f_process ) = @_;my $found = 0;## Connect to the database - once#connectRM(\$RM_DB, $opt_verbose) unless $RM_DB;$m_sqlstr =~ s~\s+~ ~g;Verbose3("SQL:", $m_sqlstr);my $sth = $RM_DB->prepare($m_sqlstr);if ( defined($sth) ){if ( $sth->execute( ) ){if ( $sth->rows ){while ( my @row = $sth->fetchrow_array ){$found++;&$f_process(\@row);}}$sth->finish();}else{Error("$fname:Execute failure: $m_sqlstr", $sth->errstr() );}}else{Error("$fname:Prepare failure" );}unless ( $found ){Verbose("$fname:No data found");}return $found;}#-------------------------------------------------------------------------------# Function : populateArrayFromSql## Description : Issue an SQL query and push the results into an array of hashes# where each row from the query is a hash and the entire result is an# array## Inputs : name - For error reporting# pArray - Ref to the output array# sql - Sql to process# pItems - Array of items to extract# Must match the SQL SELECT arguments# Item names starting with '-' do not end up in the# generated XML# Returns :#sub populateArrayFromSql{my ($fname, $pArray, $m_sqlstr, $pItems) = @_;performSqlQueryCallback($fname,$m_sqlstr,sub {my ($pRow) = @_;my %entry;push @{$pArray}, populateHash( \%entry, $pRow, $pItems);});#DebugDumpData("populateArrayFromSql", $pArray);}#-------------------------------------------------------------------------------# Function : populateHash## Description : Put an array of data items into a hash## Inputs : pHash - ref to output hash# pRow - Ref to the row data# pItems - Ref to an hash array of entry names## Returns : pHash#sub populateHash{my ($pHash, $pRow, $pItems) = @_;foreach my $item ( @{$pItems} ) {my $data = shift @{$pRow};if (defined $data){$data =~ s~^\s+~~;$data =~ s~\s+$~~;## Store in hash#$pHash->{$item} = $data;}}return $pHash;}#-------------------------------------------------------------------------------# Function : executeRmQuery## Description : Execute a simple RM query. One that does not expect any return data## Inputs : $fname - OprName, for error reporting# $m_sqlstr - SQL String## Returns : Will exit on error#sub executeRmQuery{my ($fname, $m_sqlstr) = @_;## Connect to the Database - once#connectRM(\$RM_DB, 0) unless $RM_DB;Verbose2('ExecuteQuery:', $fname);## Create the full SQL statement#my $sth = $RM_DB->prepare($m_sqlstr);if ( defined($sth) ){if ( $sth->execute() ){$sth->finish();}else{Error("$fname: Execute failure: $m_sqlstr", $sth->errstr() );}}else{Error("$fname: Prepare failure");}}#-------------------------------------------------------------------------------# Function : getFileTime## Description : Get the timestamp from a file## Inputs : Path to the file## Returns : Time since the file was created in seconds#sub getFileTime{my ($fname, $idx) = @_;my $stamp = (stat($fname))[8];if (defined $stamp) {$stamp = time() - $stamp;} else {$stamp = 0;}return $stamp;}#-------------------------------------------------------------------------------# Function : makeFile## Description : Create a file## Inputs : Path to the file## Returns :#sub makeFile{my ($name) = @_;my $fh;open( $fh, '>',$name);print $fh '';close $fh;}#-------------------------------------------------------------------------------# Documentation#=pod=for htmltoc SYSUTIL::=head1 NAMEprocess_release_notes - Create Release Notes for newly created packages=head1 SYNOPSISjats process_release_notes [options]Options:-help - Brief help message-help -help - Detailed help message-man - Full documentation-verbose - Display additional progress messages-pvid=nn - PVID of package to process(test mode)-age=nn - Examine packages created in last n days-keeptemp - Keep the temp workspace-status - Display status=head1 OPTIONS=over 8=item B<-help>Print a brief help message and exits.=item B<-help -help>Print a detailed help message with an explanation for each option.=item B<-man>Prints the manual page and exits.=item B<-pvid=nn>This option bypasses the normal mechanism of determining packages to be processed andwill process just the specified package. This is normally only used to test the operationof the subsystem.=item B<-age=nn>This option control the period of time period used in determining which packages to process.The number is in days. The default value is 2 days.=item B<-keeptemp>If this option is specified, then the temporary directory created in the processing will beretained. The user is responsible for deleting the directory.This option is normally only used in testing.=item B<-status>When set, this option will cause the program to display the package name and version of eachpackage being processed.=back=head1 DESCRIPTIONThis utility program is apart of the Package Release process. It is an interim solution.The script is designed to be run as a cron job on the dpkg_archive server.It will scan the Release Manager database for packages that have been release, but donot yet have release notes. For each package it will then generate a Release Note by:=over 4=item *Invoke jats_get_releasenote_data to:Extract relevant information from=over 4=item *The Release Manager database=item *the ClearQuest database (now contained within the Release Manager database)=item *the Jira Issue server=backand create and XML file.=item *Invoke jats_gen_releasenote to:=over 4=item *Process the XML file created above, and package list info found within the new package.If no package information is found then the utility will create a package list, but this ismuch slower than having it created by the package builder.=item *Create an XML file containing all the package information=item *Create an HTML based Release Note, from the XML file. This file could also be used to create a Release note in another format if required.=item *Transfer these two files into the package in dpkg_archive=item *Insert the packages file list into the Release Manager database.=back=item *Make the package read-only. This will trigger any post-install processing of thepackage. This is only required for packages that form a part of the build system.=item *Create a tar-zip of the package. This is stored within dpkg_archive and is available to othertools to transfer the package as a single entity. The tarZip is only retained as long as itis needed by the other tools.=item *Flag that the package has been released. This may trigger the BLAT package file transfer processand the package will be transferred to remote sites.=item *Determine the users that have registered interest in being informed when the package is released.It will then email these users.=back=head1 EXAMPLE=head2 process_release_notesThis will perform a single scan of the Release Manager database and generate Release Notes as required.=cut