Subversion Repositories DevTools

########################################################################
# Copyright (c) VIX TECHNOLOGY (AUST) LTD
#
# Module name   : jats_gen_releasenote.pl
# Module type   : Makefile system
# Compiler(s)   : Perl
# Environment(s): jats
#
# Description   : Generate the package Release Note
#                 Requires
#                   1) Release Note data generated by jats_get_releasenote_data.pl
#                   2) Target package in dpkg_archive
#                   3) One or more built.files.xxx.xml (optional, but desirable)
#
#                 Will:
#                   Merge built.files.xxx.xml files into the release note data
#                   Insert build meta data
#                   Save the Release Note MetaData within the package
#                   Generate the HTML release note - save within the package
#                   Insert Release_Contents into RM Database
#
# Usage:        See POD
#
#......................................................................#

require 5.008_002;
use strict;
use warnings;

use Pod::Usage;
use Getopt::Long;
use XML::Simple;
use File::Path;
use File::Copy;
use XML::LibXML;
use Digest::MD5;
use File::Find;
use FindBin;                                # Determine the current directory
use Encode qw(decode encode);

use JatsError;
use JatsSystem;
use Getopt::Long;
use Pod::Usage;
use FileUtils;
use JatsEnv;
use JatsRmApi;

use DBI;

my $VERSION = "1.0.0";                                      # Update this. Inserted into meta data field

# User options
my $opt_verbose = 0;
my $opt_help = 0;
my $opt_archive;
my $archive_tag;
my $opt_pname;
my $opt_pversion;
my $opt_release_note;
my $opt_pvid;
my $opt_outname;
my $opt_outputXml;
my $opt_updateRmFiles;
my $opt_workingDir;
my $opt_noRM;

#   Global vars
our $GBE_HOSTNAME;
my $DPKG_ROOT;
my $DPKG_DOC;
my $JATS_RN;
my $OUT_ROOT;


#
#   Structure to translate -archive=xxx option to archive variable
#   These are the various dpkg_archives known to JATS
#
my %Archive2Var =( 'main'       =>  'GBE_DPKG',
                   'store'      =>  'GBE_DPKG_STORE',
                   'cache'      =>  'GBE_DPKG_CACHE',
                   'local'      =>  'GBE_DPKG_LOCAL',
                   'sandbox'    =>  'GBE_DPKG_SBOX',
                   'deploy'     =>  'GBE_DPLY',
                   );

#-------------------------------------------------------------------------------
# Function        : Main Entry
#
# Description     :
#
# Inputs          :
#
# Returns         :
#
{
    my $result = GetOptions (
                    "help+"         => \$opt_help,              # flag, multiple use allowed
                    "manual:3"      => \$opt_help,              # flag, set value
                    "verbose:+"     => \$opt_verbose,           # flag, increment
                    "archive=s"     => \$opt_archive,           # string
                    "releasenote=s" => \$opt_release_note,      # string
                    "UpdateRmFiles" => \$opt_updateRmFiles,     # flag
                    "WorkDir=s"     => \$opt_workingDir,        # String
                    "noRM"          => \$opt_noRM,              # Flag
                    );

    #
    #   Process help and manual options
    #
    pod2usage(-verbose => 0, -message => "Version: $VERSION")  if ($opt_help == 1  || ! $result);
    pod2usage(-verbose => 1)  if ($opt_help == 2 );
    pod2usage(-verbose => 2)  if ($opt_help > 2);

    ErrorConfig( 'name'    =>'GenRn', 'verbose' => $opt_verbose );
    InitFileUtils();

    #
    #   Needed EnvVars
    #
    EnvImport ('GBE_HOSTNAME');
    $JATS_RN = $FindBin::Bin;
    Error("Release Note tools not found", $JATS_RN)
        unless (-d $JATS_RN);


    #
    #   Sanity Check
    #
    Error("Release Note data not provided") unless $opt_release_note;
    Error("Release Note data not found") unless -f $opt_release_note;
    if (defined $opt_workingDir)
    {
        $opt_workingDir = FullPath($opt_workingDir);
        Error("Working directory is not a directory") unless -d $opt_workingDir;
        Error("Working directory is writable") unless -w $opt_workingDir;
    }
    
    #
    #   Determine the target archive
    #   The default archive is GBE_DPKG, but this may be changed
    #
    $opt_archive = 'main' unless ( $opt_archive );
    my $archive_tag = $Archive2Var{$opt_archive};
    Error("Unknown archive specified: $opt_archive")
        unless ( $archive_tag );
    $DPKG_ROOT = $ENV{$archive_tag} || '';
    Verbose ("Archive Variable: $archive_tag" );
    Verbose2 ("Archive Path: $DPKG_ROOT" );

    #
    #   Process the release note and extract essential information
    #
    processReleaseNote();

    #
    #   Locate the target package
    #
    $DPKG_ROOT = catdir($DPKG_ROOT, $opt_pname, $opt_pversion);
    Verbose ("Package Path: ", DisplayPath($DPKG_ROOT) );
    Error ("Package not found in archive", $DPKG_ROOT) unless -d $DPKG_ROOT;

    #   OUT_ROOT is really only used for testing
    #   Needs to mimic $DPKG_ROOT
    $OUT_ROOT = defined($opt_workingDir) ? $opt_workingDir : $DPKG_ROOT; 
    Verbose2 ("Output Path: $OUT_ROOT" );

    #
    #   Calculate output filenames
    #   Html Release Note:
    #       RELEASE_NOTES_PVID_PKGNAME_CLEANV.html
    #
    $DPKG_DOC = catdir($OUT_ROOT, 'doc');
    $opt_outputXml = catdir($DPKG_DOC, 'release_note.xml');

    my $cleanv = $opt_pversion;
    $cleanv =~ s~\.~_~g;
    $opt_outname = join('_',
                        'RELEASE_NOTES',
                        $opt_pvid,
                        $opt_pname,
                        $cleanv,    
                        ) . '.html';
    Verbose("Note Name: $opt_outname");

    #
    #   Locate the release note data file
    #   This was created as a prerequisite to the build
    #   Note: windows requires '/' in the file list
    #
    my @filesList;
    my $appendGenFiles = 1;
    foreach my $item ( glob(catdir($DPKG_ROOT, 'built.files.*.xml'))) {
        $item =~ s~\\~/~g;
        push @filesList, $item;
    }
    unless (scalar @filesList > 0)
    {
        Warning("No file list found within the package: $opt_pname/$opt_pversion");
        $appendGenFiles = 0;
        #
        #   No filelist found in the package
        #   This may occur for packages that did not go though the build system
        #   Create a package-list
        #
        my $item = generateFileList();
        $item =~ s~\\~/~g;
        push @filesList, $item;
    }

    #
    #   Create output directory within the package
    #
    mkpath($DPKG_DOC, 0, 0775);

    #
    #   Generate name of temp file
    #   
    my $builtFiles = 'built.files.releasenote.xml';
    if (defined $opt_workingDir) {
        $builtFiles = join('/', $opt_workingDir, $builtFiles);
    }
    Verbose("TempFile: $builtFiles");

    #
    #   Merge the Release Note and the various file lists into a single
    #   XML file. Keep the file local
    #
    System('--NoShell', '--Exit', 'java', '-jar', 
           catdir($JATS_RN, 'saxon9he.jar'), 
           '-xsl:' . catdir($JATS_RN, 'merge.xslt'),
           "-s:$opt_release_note",
           'fileList=' . join(',', @filesList),
           '-o:' . $builtFiles
            );

    #
    #   Generate the HTML Release Note
    #   Output directly to the body of the package
    #
    System('--NoShell', '--Exit', 'java', '-jar', 
           catdir($JATS_RN, 'saxon9he.jar'), 
           '-xsl:' . catdir($JATS_RN, 'releaseNote2html.xslt'),
           '-s:' . $builtFiles,
           "-o:" . catdir($DPKG_DOC, $opt_outname)
            );

    #
    #   Transfer the XML database directly into the package
    #
    unless( File::Copy::copy($builtFiles, $opt_outputXml) ) {
        Error("Cannot transfer XML release note", $!, $opt_outputXml);
    }
    unlink $builtFiles; 

    #
    #   Delete the built.files.xxx.xml that may be present in the root of the package
    #   These are no longer needed - they have been incorporated into the release_note.xml file
    #
    foreach my $item ( glob(catdir($DPKG_ROOT, 'built.files.*.xml'))) {
        unlink $item;
    }

    #
    #   Update the Release Manager entry - File Data
    #
    if ($opt_updateRmFiles)
    {
        my $genFileData;
        $genFileData = genFileData($opt_outname, 'release_note.xml') if ($appendGenFiles);

        updateRmFiles($genFileData);
        updateRmNoteInfo();
    }

    #
    #   All done
    #
    Verbose ("Release Note:", DisplayPath(catdir($DPKG_DOC, $opt_outname)));
    exit 0;
}

#-------------------------------------------------------------------------------
# Function        : processReleaseNote 
#
# Description     : Extract essential information from the Release Note Data
#
# Inputs          : 
#
# Returns         : Populates global variables 
#
sub processReleaseNote
{

    #
    #   Extract essential information
    #   
    my $xml;
    my $parser = XML::LibXML->new();
    my $doc    = $parser->parse_file($opt_release_note);

    #
    #   Extract the 'package' data
    #   Only expect one of them
    #   
    my @nodes = $doc->findnodes('/package_data/package');
    my $node = $nodes[0];
    Error ("Package section not found in $opt_release_note") unless defined $node;

    $opt_pname    = $node->getAttribute( 'name' );
    $opt_pversion = $node->getAttribute( 'version' );
    $opt_pvid     = $node->getAttribute( 'pvid' );
    
    Verbose("Package Name: $opt_pname") ;
    Verbose("Package Version: $opt_pversion") ;
    Verbose("Package Pvid: $opt_pvid") ;
}

#-------------------------------------------------------------------------------
# Function        : genFileData  
#
# Description     : Generate File Data for some locally generated files so that they
#                   will appear in the Release Manager database 
#
# Inputs          : List of files to process
#                   Assumed they are in the $DPKG_DOC directory 
#
# Returns         : An array of data items
#                   Need:
#                       {path}
#                       {name}
#                       {size}
#                       {md5sum}
#                       {fullname}
#                       {type}
#                       {machtype}
#                       {host}
#                   
sub genFileData
{
    my (@fileList) = @_;
    my $retData;
    Verbose('genFileData');

    #
    #   Create entry for the 'doc' folder
    #
    {
        my %data;
        $data{fullname} = 'doc';
        $data{path} = 'doc';
        $data{type} = 'dir';
        $data{machtype} = 'unknown';
        $data{host} = $GBE_HOSTNAME;
        $data{noWarn} = 1;
        push @{$retData},\%data;
    }

    #
    #   Process each file
    #
    foreach my $file ( @fileList) {
        my %data;

        my $source = catfile($DPKG_DOC, $file);
        $data{md5sum} = genMd5Sum($source);
        $data{size} = (stat($source))[7];

        $data{name} = $file;
        $data{fullname} = 'doc/' . $file;
        $data{path} = 'doc/';
        $data{type} = 'file';
        $data{machtype} = 'unknown';
        $data{host} = $GBE_HOSTNAME;
        push @{$retData},\%data;
    }

    return $retData;
}

#-------------------------------------------------------------------------------
# Function        : generateFileList
#                   generateFileListProc 
#
# Description     : A fileList has not been found in the target package
#                   Perhaps the package was not created by the build system
#                   
#                   Generate a fileList        
#
# Inputs          : None
#
# Returns         : Path to the generated file list
#
my @generateFileListData;
my $baseLength;
sub generateFileList
{
    my $outXmlFile;
    Verbose("Generate internal filelist - none found in package");

    #
    #   Scan the package and process each file
    #
    $baseLength = length($DPKG_ROOT);
    File::Find::find( \&generateFileListProc, $DPKG_ROOT );

    #
    #   Write out XML of the Generated file list
    #
    my $data;
    $data->{file} = \@generateFileListData;

    #
    #   Write out sections of XML
    #       Want control over the output order
    #       Use lots of attributes and only elements for arrays
    #       Save as one attribute per line - for readability
    #
    $outXmlFile = catdir($OUT_ROOT,"built.files.$GBE_HOSTNAME.xml");

    Verbose2('Meta File', $outXmlFile);
    my $xs = XML::Simple->new( NoAttr =>0, AttrIndent => 1 );

    open (my $XML, '>', $outXmlFile) || Error ("Cannot create output file: $outXmlFile", $!);
    $xs->XMLout($data, 
                'RootName' => 'files', 
                'XMLDecl'  => '<?xml version="1.0" encoding="UTF-8"?>',
                'OutputFile' => $XML);
    close $XML;

    return $outXmlFile;
}

sub generateFileListProc
{
    my %data;
    my $md5sum;
    my $source = $File::Find::name;
    my $type = 'dir';

    my $target = substr($source, $baseLength);
    $target =~ s~^/~~;
    $target =~ s~/$~~;
    Verbose2("GenFileList: $source, $target");
    return if (length ($target) == 0);

    if (-l $source)
    {
        $type = 'link';
    }
    elsif ( -f $source)
    {
        $type = 'file';
        $md5sum = genMd5Sum($source);
    }

    #
    #   Convert from iso-8859-1 into utf-8
    #
    $target = decode( 'iso-8859-1', $target );
    $target = encode( 'utf-8', $target );

    if (-d $source)
    {
        $data{path} = $target;
    }
    else
    {
        $data{path} = StripFileExt($target);
        $data{name} = StripDir($target);
        if (defined $md5sum)
        {
            $data{size} = (stat($source))[7];
            $data{md5sum} = $md5sum;
        }
    }

    $data{fullname} = $target;
    $data{type} = $type;
    $data{machtype} = 'unknown';
    $data{host} = $GBE_HOSTNAME;

    # Put a nice '/' on the end of the path elements
    $data{path} .= '/'
        if ( exists ($data{path}) && length($data{path}) > 0);

    push @generateFileListData, \%data;
}

#-------------------------------------------------------------------------------
# Function        : genMd5Sum 
#
# Description     : Generate the MD5SUM of a specified file
#
# Inputs          : $source - Full path to file 
#
# Returns         : md5sum of the file
#                   Will not return on error 
#
sub genMd5Sum
{
    my ($source) = @_;
    my $md5sum;

    Verbose2("Calculate MD5 Digest: $source");
    open(my $fh , $source) or Error ("Can't open '$source': $!");
    binmode $fh, ':crlf';
    $md5sum = Digest::MD5->new->addfile($fh)->hexdigest;
    close $fh;

    return $md5sum;
}

#-------------------------------------------------------------------------------
# Function        : updateRmFiles 
#
# Description     : 
#
# Inputs          : $genFileData            - Data for generated files [Optional] 
#
# Returns         : 
#
my $RM_DB;
my  $updateRmFilesData;
sub updateRmFiles
{
    my ($genFileData) = @_;
    my $eCount = 0;
    #
    #   If in use the the autobuilder - which it should be, then
    #   modify the user name to access RM with a proxy user. This is the
    #   same method used in the 'buildtool'
    #
    if ($ENV{GBE_ABT} && $ENV{GBE_RM_USERNAME} && $ENV{GBE_RM_USERNAME} !~ m~]$~ )
    {
        Verbose("Access RM database as proxy user");
        $ENV{GBE_RM_USERNAME} = $ENV{GBE_RM_USERNAME} . '[release_manager]'; 
    }

    #
    #   Read in the release note data base
    #
    my $rnDataBase = catdir($DPKG_DOC, 'release_note.xml');
    if (! -f $rnDataBase )
    {
        Error("Release Note XML database not found: $!", $rnDataBase);
    }

    #
    #   Extract essential information
    #   Don't use a simple parser - its gets it wrong
    #   
    my $xml;
    my $parser = XML::LibXML->new();
    my $doc    = $parser->parse_file($rnDataBase);

    #
    #   Extract the 'package' data
    #   Only expect one of them

    my @nodes = $doc->findnodes('/package_data/package');
    my $node = $nodes[0];
    Error ("Package section not found in $rnDataBase") unless defined $node;

    my %eldata;
    foreach my $atr ($node->findnodes( "./@*")) {     
        Debug("Package Data:" .  $atr->nodeName() . ":" . $atr->value);
        $xml->{package}{$atr->nodeName} = $atr->value;
    }

    #
    #   Extract the file data
    #   Tricky - needs to be an array
    #
    foreach my $element ($doc->findnodes('/package_data/files/file')) {
       my %eldata;
       foreach my $node ($element->findnodes( "./@*")) {     
           Debug("File:" .  $node->nodeName() . ":" . $node->value);
           $eldata{$node->nodeName} = $node->value;
       }
       push @{$xml->{file}}, \%eldata;
    }
    #DebugDumpData("XML DATA", $xml);

    #
    #   Sanity Test the data
    #
    Error("Sanity Check Failure. PVID") unless($opt_pvid eq $xml->{package}{pvid});
    Error("Sanity Check Failure. Name") unless($opt_pname eq $xml->{package}{name});
    Error("Sanity Check Failure. Version") unless($opt_pversion eq $xml->{package}{version});
    Error("Sanity Check Failure. File Section") unless(exists $xml->{file});

    #
    #   Append any additional file data
    #       This will be the doc/ doc/release_note.xml and doc/RELEASE_NOTES.....html
    #
    if ($genFileData) {
        foreach ( @{$genFileData}) {
            push @{$xml->{file}}, $_;
        }
    }

    #
    #   Delete any existing entry(s)
    #
    updateRmFilesDelete();

    #
    #   Release Manager Database will barf if there are duplicate entries
    #   Maintain a hash of items processed, and only process each once
    #
    my %fullNameSeen;

    $updateRmFilesData = "";
    foreach my $entry (@{$xml->{file}})
    {
        #
        #   Ignore 'merge' entries
        #
        next if ((exists $entry->{type}) && ($entry->{type} eq 'merge'));
        #
        #   Clean up the data
        #
        my $fname = $entry->{name} || '';
        my $fpath  = $entry->{path} || '';
        my $fsize  = $entry->{size} || 0;
        my $fmd5  = $entry->{md5sum} || '';
        $fpath =~ s~/$~~ unless $fname;

        unless( $fullNameSeen{$fpath}{$fname} )
        {
            $fullNameSeen{$fpath}{$fname} = 1;

            #
            #   Create SQL fragment for the insert
            #
            $eCount++;
            my $quoteFname = $fname;
            $quoteFname =~ s~'~''~g;
            my $entry = " INTO release_manager.release_components ( pv_id, file_name, file_path, byte_size, crc_cksum, crc_modcrc ) " .
                        " VALUES ( $opt_pvid, '$quoteFname', '$fpath',$fsize , '$fmd5', '')";
            $updateRmFilesData .= $entry;

            # The size of the aggregation is key to performance
            # Too big is as bad as too small
            if (length($updateRmFilesData) > 10000)
            {
                updateRmFilesInsert();
            }
        }
        else
        {
            unless ($entry->{noWarn}) {
                Warning("Multiple file entries for: $opt_pname, $opt_pversion : $fpath $fname");
            }
        }
    }
    updateRmFilesInsert();
    Verbose ("Inserted $eCount entries into Release_Components");
}

#-------------------------------------------------------------------------------
# Function        : updateRmFilesInsert 
#
# Description     : Insert entries using the partial SQL statement 
#                   Must be called when the partial SQL buffer get large
#                   as well as at the end to fluch any outstanding inserts
#
# Inputs          : 
#
# Returns         : 
#
sub updateRmFilesInsert
{
    if (length($updateRmFilesData) > 0)
    {
        executeRmQuery(
            'updateRmFilesInsert',
            'INSERT ALL' . $updateRmFilesData . ' SELECT 1 FROM DUAL'
            );

        $updateRmFilesData = ""; 
    }
}
#-------------------------------------------------------------------------------
# Function        : updateRmFilesDelete 
#
# Description     : Delete ALL entires in the 'release_components' table associated
#                   with the current pvid
#                   
#                   Needs to be done before new entries are added.    
#
# Inputs          : 
#
# Returns         : 
#
sub updateRmFilesDelete
{
    executeRmQuery(
        'updateRmFilesDelete',
        'delete from release_manager.release_components where pv_id=' . $opt_pvid
        );
}

#-------------------------------------------------------------------------------
# Function        : updateRmNoteInfo 
#
# Description     : Insert Release Note Info into the Release Manager Database
#                   This has the side effect that RM will see the Release Note
#                   as being fully generated.
#
# Inputs          : 
#
# Returns         : 
#
sub updateRmNoteInfo
{
    #
    #   Determine the path to the Release Note in a form that is suitable for the Release Manager
    #   Database. This expects: /dpkg_archive/PkgName/PkgVer/doc/pkgFileName
    #
    my $rnPath = join('/', '', 'dpkg_archive', $opt_pname, $opt_pversion, 'doc', $opt_outname);
    Verbose("RN_INFO:", $rnPath);

    # Into the database
    executeRmQuery (
        'updateRmNoteInfo', 
        'update release_manager.package_versions set release_notes_info=\'' . $rnPath . '\' where pv_id=' . $opt_pvid
        );
}

#-------------------------------------------------------------------------------
# Function        : executeRmQuery 
#
# Description     : Execute a simple RM query. One that does not expect any return data
#
# Inputs          : $fname           - OprName, for error reporting
#                   $m_sqlstr        - SQL String
#
# Returns         : Will exit on error
#
sub executeRmQuery
{
    my ($fname, $m_sqlstr) = @_;

    #
    #   Testing only
    #   Display the Database access commands - don't execute them
    #
    if ($opt_noRM) {
        Debug0("$fname", $m_sqlstr);
        return;
    }

    #
    #   Connect to the Database - once
    #
    connectRM(\$RM_DB, 0) unless $RM_DB;

    Verbose2('ExecuteQuery:', $fname);
    #
    #   Create the full SQL statement
    #
    my $sth = $RM_DB->prepare($m_sqlstr);
    if ( defined($sth) )
    {
        if ( $sth->execute() )
        {
            $sth->finish();
        }
        else
        {
            Error("$fname: Execute failure: $m_sqlstr", $sth->errstr() );
        }
    }
    else
    {
        Error("$fname: Prepare failure");
    }
}


#-------------------------------------------------------------------------------
#   Documentation
#

=pod

=for htmltoc    SYSUTIL::

=head1 NAME

jats_gen_releasenote - Generate a Release Note

=head1 SYNOPSIS

 jats jats_gen_releasenote [options]

 Options:
    -help              - Brief help message
    -help -help        - Detailed help message
    -man               - Full documentation
    -verbose           - Display additional progress messages
    -outfile=name      - [Optional] Name of the output XML file
    -archive=name      - [Optional] Name package archive
    -releasenote=path  - Path to the Release Note Data
    -UpdateRmFiles     - Update the Files list in Release Manager
    -WorkDir=path      - [Test] Path to work area
    -NoRM              - [Test] No RM present
    

=head1 OPTIONS

=over 8

=item B<-help>

Print a brief help message and exits.

=item B<-help -help>

Print a detailed help message with an explanation for each option.

=item B<-man>

Prints the manual page and exits.

=item B<-pvid=nn>

This option provides identifies the PackageVersion to be processed.

This option is mandatory.

=item B<-outfile=name>

This option specifies the output file name.

If not provided by the user the output filename will be created in the current directory
and it will be named after the package name and package version.

If the filename does not end in .xml, then .xml will be appended to the file name.

=item B<-UpdateRmFiles>

This option will case the utility to ppdate the Files list in Release Manager.

The existing file list will be replaced by the one within the package.

Note: Write Access to Release Manager is required.

=item B<-WorkDir=Path>

This option is provided to aid in testing of this tool. 

If provided the tool will use the specified path when creating files.

=item B<-NoRM>

This option is provided to aid in testing of this tool. 

When provided, this option will supress access to the Release Manager database. Instead the
SQL commands will be displayed.

=back

=head1 DESCRIPTION

This utility program is used to extract sufficient information from Release Manager and other
associated databases so that a Release Note can be created.

The extracted data is stored in an XML format. The intent is that XSLT will be used to create
an HTML based release note.


=head1 EXAMPLE

=head2 jats jats_gen_releasenote -releasenote=/tmp/myreleasenote.xml

This will process release note data in the specified XML file.

=cut