Rev 4688 | Blame | Compare with Previous | Last modification | View Log | RSS feed
######################################################################### Copyright (c) VIX TECHNOLOGY (AUST) LTD## Module name : process_release_notes.pl# Module type : JATS Utility# Compiler(s) : Perl# Environment(s): jats## Description : Poll RM and generate Release Notes as required# Must run on the same machine as the package archive# It directly manipulates the target package# Requires access to the machines sudo mechism## Usage : See POD at the end of this file##......................................................................#require 5.008_002;use strict;use warnings;use JatsError;use JatsSystem;use Getopt::Long;use Pod::Usage; # required for help supportuse File::Temp qw/ tempfile tempdir /;use LWP::UserAgent;use HTTP::Request::Common 'POST';use Fcntl ':flock';use JatsRmApi;use FileUtils;use DBI;my $VERSION = "1.0.0"; # Update thismy $GBE_DPKG = $ENV{GBE_DPKG}; # Sanitised by Jatsmy $RM_DB;my $tempdir;my $sudoUtilsDir = '/home/releasem/sbin';my $lockFile = '/tmp/JATSRN_LOCK';## Options#my $opt_verbose = 0;my $opt_help = 0;my $opt_age = 2;my $opt_keepTemp = 0;my $opt_status = 0;my $opt_pvid;## Package information#my @Packages;my @packageItems = qw( PV_ID RTAG_ID NAME VERSION PKG_ID PROJ_ID RNINFO);my %rmConfig;#-------------------------------------------------------------------------------# Function : Main Entry## Description :## Inputs :## Returns :#my $result = GetOptions ('help+' => \$opt_help, # flag, multiple use allowed'manual:3' => \$opt_help, # flag'verbose:+' => \$opt_verbose, # flag'age:i' => \$opt_age, # Number'keeptemp!' => \$opt_keepTemp, # flag'status+' => \$opt_status, # flag'pvid:i' => \$opt_pvid, # Number);## Process help and manual options#pod2usage(-verbose => 0, -message => "Version: $VERSION") if ($opt_help == 1 || ! $result);pod2usage(-verbose => 1) if ($opt_help == 2);pod2usage(-verbose => 2) if ($opt_help > 2);ErrorConfig( 'name' =>'RelNotes','verbose' => $opt_verbose );$opt_verbose-- if ($opt_verbose > 0);## Sanity Test#Error("Not running on a Unix System") unless ($ENV{GBE_UNIX});Error("Sudo Utils not found: $sudoUtilsDir") unless ( -d $sudoUtilsDir);Warning("Not running as a sutable user" ) unless ($ENV{USER} eq 'buildadm' || $ENV{USER} eq 'releasem');## Ensure only one instance is running# PerlMonks say to use $0 as the lock file, but that did not work.# Perhaps the file was open in an editor# Solution: Create my own file#open( FW, '>',$lockFile);print FW '';close FW;open (my $self, '<', $lockFile) || Error("Couldn't open self: $!");flock ($self, (LOCK_EX | LOCK_NB)) || Error("This script is already running");## Init parameters#$tempdir = tempdir('/tmp/JATSRN_XXXXXX',CLEANUP => !$opt_keepTemp);Warning("Need to remove tempdir: " . DisplayPath($tempdir)) if $opt_keepTemp;Verbose("Tempdir:" , DisplayPath($tempdir));# Force GBE_ABT. Currently the Release Manager proxy password only works# When used in this mode$ENV{GBE_ABT} = 1 unless defined $ENV{GBE_ABT};## User RW password, if provided#$ENV{GBE_RM_USERNAME} = $ENV{GBE_RM_USERNAME_RW} if (exists $ENV{GBE_RM_USERNAME_RW});$ENV{GBE_RM_PASSWORD} = $ENV{GBE_RM_PASSWORD_RW} if (exists $ENV{GBE_RM_PASSWORD_RW});if ($ENV{GBE_ABT} && $ENV{GBE_RM_USERNAME} && $ENV{GBE_RM_USERNAME} !~ m~]$~ ){Verbose("Access RM database as proxy user");$ENV{GBE_RM_USERNAME} = $ENV{GBE_RM_USERNAME} . '[release_manager]';}## Interogate the RM database and locate packages that need to be processed#$opt_pvid ? LocatePackagesByPvid() : LocatePackages();Verbose("Package to process:" . scalar @Packages);#DebugDumpData("Packages", \@Packages);## Process each found entry#foreach my $entry ( @Packages){GenerateReleaseNote( $entry);}unlink $lockFile;exit 0;#-------------------------------------------------------------------------------# Function : LocatePackages## Description : Locate packages in need of Release Notes## Inputs : None## Returns : Nothing# Populates the @Packages array#sub LocatePackages{my $m_sqlstr ="SELECT * from(SELECT pv.pv_id,rc.rtag_id,pkg.pkg_name,pv.pkg_version,pv.pkg_id,rt.proj_id ,pv.release_notes_info,TRUNC(sysdate - pv.modified_stamp ) AS age ,row_number() OVER (PARTITION BY pv.pv_id, pkg.pkg_name, pv.pkg_version ORDER BY rc.rtag_id) rnFROM release_manager.package_versions pv,release_manager.release_content rc,release_manager.packages pkg,release_manager.release_tags rtWHERE pv.pv_id = rc.pv_idAND pkg.pkg_id = pv.pkg_idAND rc.rtag_id = rt.rtag_idAND pv.pv_id IN (SELECT pv_idFROM release_manager.package_versionsWHERE (release_notes_info IS nullOR release_notes_info like 'MSG:%' )AND pv.dlocked = 'Y'AND pv.modified_stamp > (sysdate - $opt_age)))where rn = 1";populateArrayFromSql('LocatePackages', \@Packages, $m_sqlstr, \@packageItems);# DebugDumpData("Packages", \@Packages);}#-------------------------------------------------------------------------------# Function : LocatePackagesByPvid## Description : Locate one package, as specified by its PVID# This mode is only used in testing## Inputs : Global: $opt_pvid## Returns : Populate the @Packages array#sub LocatePackagesByPvid{# Extract: PV_ID RTAG_ID NAME VERSION PKG_ID PROJ_ID RNINFO AGEmy $m_sqlstr = "SELECT pv.pv_id,rc.rtag_id,pkg.pkg_name,pv.pkg_version,pv.pkg_id,rt.proj_id,pv.release_notes_infoFROM release_manager.package_versions pv,release_manager.release_content rc,release_manager.packages pkg,release_manager.release_tags rtWHERE pv.pv_id = rc.pv_idAND pkg.pkg_id = pv.pkg_idAND rc.rtag_id = rt.rtag_idAND pv.pv_id = $opt_pvid";populateArrayFromSql('LocatePackagesByPvid', \@Packages, $m_sqlstr, \@packageItems);#DebugDumpData("Packages", \@Packages);}#-------------------------------------------------------------------------------# Function : GenerateReleaseNote## Description : Generate One Release Note# Invoke several JATS utilities to do the hard work## Inputs : $entry - Hash of useful infomation## Returns : Nothing# Will exit on error#sub GenerateReleaseNote{my ($entry) = @_;my $outfile;my $rv;my @args;#DebugDumpData("Entry", $entry);Message("-------------------- $entry->{NAME}, $entry->{VERSION}") if $opt_status || $opt_verbose;my $pkgBase = catdir($GBE_DPKG, $entry->{NAME}, $entry->{VERSION});Verbose("Processing: $pkgBase");unless (-d $pkgBase) {Warning("Package not in archive: $pkgBase");return;}## Make the target Package Version Writable and Ensure that the doc directory# exists. This is done via a script that is invoked with SUDO, partially to# interwork with the existing system#runSudoCommand('make_writable', 'dpkg_archive', $entry->{NAME},$entry->{VERSION});runSudoCommand('make_docFolder', 'dpkg_archive', $entry->{NAME},$entry->{VERSION}, 'doc');## Get the basic data required for the Release Note# jats jats_get_releasenote_data.pl -pvid=1002782#$outfile = catdir($tempdir, join('_',$entry->{NAME},$entry->{VERSION})) . '.xml';$rv = JatsTool('jats_get_releasenote_data', '-verbose', $opt_verbose, '-pvid', $entry->{PV_ID}, '-outfile', $outfile );## Generate the actual release note and update the Release Manager# jats jats_gen_releasenote.pl -releasenote=mcrdemo_1.2.3157.cr.xml -UpdateRmFilesunless ($rv){$rv = JatsTool('jats_gen_releasenote.pl', '-verbose', $opt_verbose, '-UpdateRmFiles', '-releasenote', $outfile);}## Release note complete - or failed# Make the target PackageVersion ReadOnly# Signal End of Package Processing - which will trigger blat## Do this even if we have a Release Note Failure#runSudoCommand('make_readonly', 'dpkg_archive', $entry->{NAME},$entry->{VERSION});# make_release_changed# archive=archive-path# pkg_name=package-name# pkg_version=package-version# rtag_id=release-tag-id# pkg_id=package-id# pv_id=package-version-id# proj_id=project-id# mode_id=change-mode-id (1 pkg added, 2 pkg removed, 3 pkg released)#push @args, 'archive=dpkg_archive';push @args, 'mode_id=3';push @args, 'pkg_name=' . $entry->{NAME};push @args, 'pkg_version=' . $entry->{VERSION};push @args, 'rtag_id=' . $entry->{RTAG_ID};push @args, 'pkg_id=' . $entry->{PKG_ID};push @args, 'pv_id=' . $entry->{PV_ID};push @args, 'proj_id=' . $entry->{PROJ_ID};runSudoCommand('make_release_changed', @args);## Email interested users#notifyInterestedUsers($entry);Error ("Did not generate Release Note: $entry->{NAME}, $entry->{VERSION}") if ($rv);}#-------------------------------------------------------------------------------# Function : notifyInterestedUsers## Description : A user can register interest in a package (name) being built in a# specific Project# Perhaps this should be done by the build tool, but at the moment# this release note generation process is really a part of the build## NOTE: Similar code exists in the RM web server to handle manually# released packages.## Inputs : $entry - Hash of useful infomation## Returns :#sub notifyInterestedUsers{my ($entry) = @_;my @userList;## Determine interested users#my $m_sqlstr = "SELECT U.user_email, prj.proj_name, rt.rtag_name, pkg.pkg_name, pv.pkg_version ,pv.pv_id, rt.rtag_idFROM PACKAGE_INTEREST PI,PACKAGES PKG,PACKAGE_VERSIONS PV,USERS U,RELEASE_TAGS RT,PROJECTS PRJWHERE PKG.PKG_ID = PI.PKG_IDAND RT.RTAG_ID = $entry->{RTAG_ID}AND PV.PV_ID = $entry->{PV_ID}AND PV.PKG_ID = PKG.PKG_IDAND PRJ.PROJ_ID = RT.PROJ_IDAND PRJ.PROJ_ID = PI.PROJ_IDAND U.USER_ID = PI.USER_ID";my @items = qw( USER_EMAIL PROJ_NAME RTAG_NAME PKG_NAME PKG_VERSION PV_ID RTAG_ID );populateArrayFromSql('notifyInterestedUsers', \@userList, $m_sqlstr, \@items);## Do we have something to do#return unless (@userList);#DebugDumpData("userList", \@userList);# Ensure we have basic emailing information#getRmConfig();## Send Email to all the required users# Note: This emailer requires a sendmail utility#foreach my $entry ( @userList){#$entry->{USER_EMAIL} = 'dpurdie@vixtechnology.com';#Debug0("Sending email to David Purdie indead of real user");my $subject = "Package Release Notification: $entry->{PKG_NAME} $entry->{PKG_VERSION}";my $content = "Version <a href='http://bms:8002/ManagerSuite/Release_Manager/fixed_issues.asp?pv_id=$entry->{PV_ID}&rtag_id=$entry->{RTAG_ID}'>$entry->{PKG_VERSION}</a> of Package $entry->{PKG_NAME} in Project $entry->{PROJ_NAME} on Release Branch $entry->{RTAG_NAME} has been released.";$content .= "<p>Package Link: http://bms:8002/ManagerSuite/Release_Manager/fixed_issues.asp?pv_id=$entry->{PV_ID}&rtag_id=$entry->{RTAG_ID}";$content .= "<p>You have received this email as a result of your interest in this package. If you do not wish to receive further emails then remove your package interest from the notifications area in Release Manager.";my $req = POST( 'mailto:' . $entry->{USER_EMAIL},From => $rmConfig{'BUILD FAILURE MAIL SENDER'},CC => 'buildadm@vixtechnology.com',Date => scalar localtime,Subject => $subject,Content_Type => qq(text/html),Content => $content,);my $response = LWP::UserAgent->new->request($req);if ($response->code != 202){Warning("Email Send Error: $response->code");#DebugDumpData("Response", \$response);}}}#-------------------------------------------------------------------------------# Function : getRmConfig## Description : Get Basic config from Release Manager# Only do it once## Just need:# 'BUILD FAILURE MAIL SENDER'# 'MAIL SERVER'## Inputs : None## Returns : Populates a global hash#sub getRmConfig{return if keys(%rmConfig) > 0;my $m_sqlstr = "SELECT * from BUILD_SERVICE_CONFIG";performSqlQueryCallback('getRmConfig',$m_sqlstr,sub {my ($pRow) = @_;$rmConfig{ $pRow->[0] } = $pRow->[1];});#DebugDumpData("rmConfig", \%rmConfig);}#-------------------------------------------------------------------------------# Function : runSudoCommand## Description : Run a Unix command as root via the sodo system# Requires that target commands be available# Requires sudo to be configured to allow this user to run them## Notes : The sudoers file on the archive server needs to be configured to allow:# This user to run programs from /home/releasem/sbin without a password# The users sudo credentials must not timeout (or be used)## Inputs : prog - Command to run# arguments - Command arguments## Returns : Nothing#sub runSudoCommand{my ($prog, @arguments) = @_;my $cmd;my $rv;## Construct command#$cmd = catdir($sudoUtilsDir, $prog);$rv = System('--NoShell','sudo','-n', $cmd, @arguments);Warning("SudoCmd Result: $prog: $rv") if ($rv);}#-------------------------------------------------------------------------------# Function : performSqlQueryCallback## Description : Perform a general Sql query and invoke a user function for# each row of results## Inputs : $fname - Name of query for error reporting# $m_sqlstr - Query string# $f_process - Function called for each row in the result# Use closure to have callback modify other data## Returns : Number of rows found#sub performSqlQueryCallback{my ($fname, $m_sqlstr, $f_process ) = @_;my $found = 0;## Connect to the database - once#connectRM(\$RM_DB, $opt_verbose) unless $RM_DB;$m_sqlstr =~ s~\s+~ ~g;Verbose3("SQL:", $m_sqlstr);my $sth = $RM_DB->prepare($m_sqlstr);if ( defined($sth) ){if ( $sth->execute( ) ){if ( $sth->rows ){while ( my @row = $sth->fetchrow_array ){$found++;&$f_process(\@row);}}$sth->finish();}else{Error("$fname:Execute failure: $m_sqlstr", $sth->errstr() );}}else{Error("$fname:Prepare failure" );}unless ( $found ){Verbose("$fname:No data found");}return $found;}#-------------------------------------------------------------------------------# Function : populateArrayFromSql## Description : Issue an SQL query and push the results into an array of hashes# where each row from the query is a hash and the entire result is an# array## Inputs : name - For error reporting# pArray - Ref to the output array# sql - Sql to process# pItems - Array of tems to extract# Must match the SQL SELECT arguments# Item names starting with '-' do not end up in the# generated XML# Returns :#sub populateArrayFromSql{my ($fname, $pArray, $m_sqlstr, $pItems) = @_;performSqlQueryCallback($fname,$m_sqlstr,sub {my ($pRow) = @_;my %entry;push @{$pArray}, populateHash( \%entry, $pRow, $pItems);});#DebugDumpData("populateArrayFromSql", $pArray);}#-------------------------------------------------------------------------------# Function : populateHash## Description : Put an array of data items into a hash## Inputs : pHash - ref to output hash# pRow - Ref to the row data# pItems - Ref to an hash array of entry names## Returns : pHash#sub populateHash{my ($pHash, $pRow, $pItems) = @_;foreach my $item ( @{$pItems} ) {my $data = shift @{$pRow};if (defined $data){$data =~ s~^\s+~~;$data =~ s~\s+$~~;## Store in hash#$pHash->{$item} = $data;}}return $pHash;}#-------------------------------------------------------------------------------# Documentation#=pod=for htmltoc SYSUTIL::=head1 NAMEprocess_release_notes - Create Release Notes for newly created packages=head1 SYNOPSISjats process_release_notes [options]Options:-help - Brief help message-help -help - Detailed help message-man - Full documentation-verbose - Display additional progress messages-pvid=nn - PVID of package to process(test mode)-age=nn - Examine packages created in last n days-keeptemp - Keep the temp workspace-status - Display status=head1 OPTIONS=over 8=item B<-help>Print a brief help message and exits.=item B<-help -help>Print a detailed help message with an explanation for each option.=item B<-man>Prints the manual page and exits.=item B<-pvid=nn>This option bypasses the normal mechanism of determining packages to be processed andwill process just the specified package. This is normally only used to test the operationof the subsystem.=item B<-age=nn>This option control the period of time period used in determing which packages to process.The number is in days. The default value is 2 days.=item B<-keeptemp>If this option is specified, then the temporary directory created in the processing will beretained. The user is responsible for deleting the directory.This option is normally only used in testing.=item B<-status>When set, this option will cause the program to display the package name and version of eachpackage being processed.=back=head1 DESCRIPTIONThis utility program is apart of the Package Release process. It is an interim solution.The script is designed to be run as a cron jon on the dpkg_archive server.It will scan the Release Manager database for packages that have been release, but donot yet have release notes. For each package it will then generate a Release Note by:=over 4=item *Invoke jats_get_releasenote_data to:Extract relevent information from=over 4=item *The Release Manager database=item *the ClearQuest database (now conatined within the Release Manager database)=item *the Jira Issue server=backand create and XML file.=item *Invoke jats_gen_releasenote to:=over 4=item *Process the the XML file created above, and package list info found within the new package.If no package informationis found then the utility will create a package list, but this ismuch slower than having it created by the package builder.=item *Create an XML file containing all the package information=item *Create an HTML based Release Note, from the XML file. This file could also be used to create a Release note in another format if required.=item *Transfer these two files into the package in dpkg_archive=item *Insert the packages file list into the Release Manager database.=back=item *Make the package read-only. This will trigger any post-install processing of thepackage. This is only required for packages that form a part of the build system.=item *Flag that the package has been released. This may tigger the BLAT package file transfer processand the package will be transferred to remote sites.=item *Determine the users that have registered interest in being informed when the package is released.It will then email these users.=back=head1 EXAMPLE=head2 process_release_notesThis will perform a single scan of the Release Manager database and generate Release Notes as required.=cut