Subversion Repositories DevTools

#! perl
########################################################################
# COPYRIGHT - VIX IP PTY LTD ("VIX"). ALL RIGHTS RESERVED.
#
# Module name   : jats.sh
# Module type   : Makefile system
# Compiler(s)   : n/a
# Environment(s): jats
#
# Description   : Generate and test build manifest
#                 This tool is intended to be used by the automated build system
#
# Usage:
#
#
#......................................................................#

require 5.008_002;

use strict;
use warnings;
use JatsError;
use FileUtils;

use Pod::Usage;                             # required for help support
use Getopt::Long;
use Cwd;
use ExtUtils::Manifest qw(manifind);        # Needs perl later than 5.6.1

my $VERSION = "1.0.0";                      # Update this

#
#   Options
#
my $opt_debug   = $ENV{'GBE_DEBUG'};        # Allow global debug
my $opt_verbose = $ENV{'GBE_VERBOSE'};      # Allow global verbose
my $opt_help = 0;
my $opt_manual = 0;
my $opt_buildsystem;
my $opt_manifest_file;
my $opt_rootdir;
my $opt_check;
my $opt_generate;
my @opt_exclude;
my @opt_rewrite;
my %opt_rewrite;
my $opt_quiet;

my @num_excluded;
my @num_rewrite;

#-------------------------------------------------------------------------------
# Function        : Mainline Entry Point
#
# Description     :
#
# Inputs          :
#
my $result = GetOptions (
                "help+"         => \$opt_help,              # flag, multiple use allowed
                "manual|man"    => \$opt_manual,            # flag
                "verbose:+"     => \$opt_verbose,           # flag, multiple use allowed
                "quiet:+"       => \$opt_quiet,             # flag, multiple use allowed
                "buildsystem"   => \$opt_buildsystem,       # Flag,
                "rootdir=s"     => \$opt_rootdir,           # String
                "manifest=s"    => \$opt_manifest_file,     # String
                "check"         => \$opt_check,             # Flag
                "generate"      => \$opt_generate,          # Flag
                "exclude=s"     => \@opt_exclude,           # Multiple strings
                "rewrite=s"     => \@opt_rewrite,           # Multiple strings
                );

                #
                #   UPDATE THE DOCUMENTATION AT THE END OF THIS FILE !!!
                #

#
#   Process help and manual options
#
pod2usage(-verbose => 0, -message => "Version: $VERSION") if ($opt_help == 1 || ! $result);
pod2usage(-verbose => 1) if ($opt_help == 2 );
pod2usage(-verbose => 2) if ($opt_manual || ($opt_help > 2));
pod2usage(-verbose => 0, -message => "Version: $VERSION") if ( $#ARGV >= 0 );

#
#   Configure the error reporting process now that we have the user options
#
ErrorConfig( 'name'    =>'MANIFEST',
             'verbose' => $opt_verbose,
            );


#
#   Validate user options
#
Error ("No Manifest file provided") if ( ! $opt_manifest_file);
#Error ("No Rootdir provided") if ( !$opt_rootdir);

#
#   Ensure the md5sum utility can be found
#
Verbose ("Check existance of utility: md5sum");
$result = qx( md5sum --version 2>&1 );
Error( "Cannot locate utility md5sum" )if ( $? );

#
#   Init the file utilities
#
InitFileUtils();

#-------------------------------------------------------------------------------
#   Generate the manifest file
#   This is done using the MD5SUM utility
#
#   Note: The MD5SUM program has a significant startup time.
#         The script will process up to 10 files at a time to improve speed
#
if ( ! $opt_check || $opt_generate )
{
    #
    #   Process the rewrite array and create a hash of entries
    #   Any file that is rewritten will be excluded
    #
    foreach my $entry ( @opt_rewrite )
    {
        my( $old, $new, @rest ) = split('=', $entry );
        Error ("Bad rewite entry: $entry") if ( @rest || !$new || !$old );
        $opt_rewrite{$old} = $new;
        $opt_rewrite{$new} = "";
    }

    #
    #   Add in the excluded entries
    #
    foreach my $entry ( @opt_exclude )
    {
        $opt_rewrite{$entry} = "";
    }
#    DebugDumpData("RewriteData", \%opt_rewrite);


    #
    #   Delete the existing file, if it exists
    #   Done in the users current directory
    #
    Verbose ("Delete any existing manifest file");
    if ( -f $opt_manifest_file )
    {
        unlink ($opt_manifest_file) || Error( "Cannot delete existing manifest file: $opt_manifest_file");
    }

    #
    #   Open a handle to the manifest file
    #
    open (MANIFEST, ">", $opt_manifest_file) || Error("Cannot create manifest file: $opt_manifest_file");

    #
    #   Change to the specified subdirectory
    #
    if ( $opt_rootdir )
    {
        Verbose ("chdir to specified root directory");
        chdir( $opt_rootdir ) || Error("Specified Root Diretory not found: $opt_rootdir");
    }

    #
    #   Generate a list of files to be included in the manifest
    #   Directories are not included
    #
    Verbose ("Generate file list");
    my $found = manifind();
    #DebugDumpData( "Data", $found );
    my @files = sort keys %$found;
    $found = "";
    Verbose("Number of files in manifest: " . (1 + $#files));
    Error ("No files in manifest") unless ( $#files > 0 );

    #
    #   Generate Md5sum info for each file
    #
    Verbose ("Generate manifest file");
    my @list;
    foreach my $file ( @files )
    {
        #
        #   Detect files that require special handling
        #   Some files are rewritten ( auto.pl -> build.pl )
        #   Some file are excluded ( build.pl )
        #
        my $rewrite;
        my $fileonly = $file;
        $fileonly =~ s~.*/~~;
        Verbose2("Processing: $file");
        if ( exists $opt_rewrite{$fileonly} )
        {
            $rewrite = $opt_rewrite{$fileonly};
            unless ( $rewrite )
            {
                Verbose("Excluding: $file");
                push @num_excluded, $file;
                next;
            }
        }

        #
        #   Speed up MD5 processing by passing multiple files to the command
        #   When we hit a "rewritten" file, then we will first process and files
        #   in the speed up list, then the one special file
        #
        if ( ! $rewrite )
        {
            push @list, '"' . $file . '"';
        }

        if ( $#list >= 10 || $rewrite )
        {
            print '.' unless ( ($opt_verbose > 1) || $opt_quiet);
            process_list( @list );
            @list = "";
        }

        #
        #   Rewrite
        #   process a single entry. There will not be many of these
        #
        if ( $rewrite )
        {
            #
            #   Generate the MD5SUM info
            #
            print '.' unless ( ($opt_verbose > 1) || $opt_quiet);
            my $data = qx(md5sum "$file");

            #
            #   Rewrite any required file name
            #
            Verbose ("Rewrite: $fileonly to $rewrite");
            $data =~ s~$fileonly$~$rewrite~;
            push @num_rewrite, $file;

            #   Append to the output file
            #
            print MANIFEST "$data";
            chomp($data);
            Verbose2( "REWRITE: $data" );
        }
    }
    process_list( @list );
    close MANIFEST;

    #
    #   Informational warnings
    #
    Message("Number of files excluded : " . (1 + $#num_excluded), @num_excluded) if ( $#num_excluded >= 0);
    Message("Number of files rewritten: " . (1 + $#num_rewrite), @num_rewrite) if ( $#num_rewrite >= 0 );
    Message("Files scanned: " . (1 + $#files) );

}

sub process_list
{
    my @list = @_;

    #
    #   Anything to do
    #
    Verbose3 ("process_list", @list, $#list);
    return if ( $#list < 1 );

#    print ".";
    open(CMD, "md5sum @list |") || Error "Can't run command: $!";
    while (<CMD>)
    {
        print MANIFEST $_;
        chomp;
        Verbose2("CMD: $_");
    }
    close CMD;
}

#-------------------------------------------------------------------------------
#   Validate the manifest
#
if ( $opt_check )
{
    my @failed;
    my $match_count = 0;
    #
    #   Determine the absolute path to the manifest file
    #   Required as the file may be a relative path
    #
    my $abs_manifest = AbsPath( $opt_manifest_file );
    Verbose("AbsPath manifest path: $abs_manifest");

    #
    #   Ensure the file exists
    #
    Error("Cannot locate manifest file: $opt_manifest_file") unless -f $opt_manifest_file;
    Error("Cannot locate abs manifest file: $opt_manifest_file") unless -f $abs_manifest;

    #
    #   Change to the specified subdirectory
    #
    if ( $opt_rootdir )
    {
        Verbose ("chdir to specified root directory");
        chdir( $opt_rootdir ) || Error("Specified Root Diretory not found: $opt_rootdir");
    }

    #
    #   Test Md5sum info for each file
    #   Process each line to allow success to be suppressed
    #
    Verbose ("Validate manifest file");
    open(CMD, "md5sum --check $abs_manifest  2>&1 |") || Error "Can't run command: $!";
    while (<CMD>)
    {
        chomp;
        Verbose2("CMD: $_");
        next if ( $_ =~ /^md5sum/ );
        if ( $_ =~ /: OK$/ )
        {
            $match_count++;
            next;
        };
        push @failed, $_;
    }
    close CMD;

    #
    #   Report failed files
    #
    Message ("Files validated: $match_count");
    if ( @failed )
    {
        Error("The following lines failed the Md5SUM:", @failed);
    }
}

exit 0;


#-------------------------------------------------------------------------------
#   Documentation
#

=pod

=for htmltoc    SYSUTIL::

=head1 NAME

jats_manifest - Generate and Test a manifest

=head1 SYNOPSIS

jats etool jats_manifest [options]

 Options:
    -help              - brief help message
    -help -help        - Detailed help message
    -man               - Full documentation
    -verbose[=n]       - Verbose operation
    -quiet             - Supress progress messages
    -generate          - Generate the manifest(default)
    -check             - Check the provided manifest
    -manifest=file     - Path of the manifest file to use
    -rootdir=path      - Root of the file tree to process (default=.)
    -exclude=file      - File to excluded during generation
    -rewrite=old=new   - Filename to rewrite during generation

=head1 OPTIONS

=over 8

=item B<-help>

Print a brief help message and exits.

=item B<-help -help>

Print a detailed help message with an explanation for each option.

=item B<-man>

Prints the manual page and exits.

=item B<-verbose[=n]>

This option will increase the level of verbosity of the command.

If an argument is provided, then it will be used to set the level, otherwise the
existing level will be incremented. This option may be specified multiple times.

=item B<-quiet>

This option will supress the '.' progress messages that are created as the manifest
is generated.

=item B<-generate>

Generate the  manifest file. This is the default operation.

This option may be used in conjunction with B<-check> to generate and check the
manifest in the one operation.

=item B<-check>

Check the provided manifest against the provide directory tree.

=item B<-manifest>

This option specifies the path to the manifest file to generate or check. If the
file is within the rootdir, then it may cause an error on checking.

This option is mandatory.

=item B<-rootdir=path>

This option specifies the root of the directory tree to process. If it is not
provided then then current directory will be used.

=item B<-exclude=file>

This option specifies the name of a file to be excluded from the generated
manifest. It has no effect during checking. The named file does not include
any path information. All files matching the name will be excluded.

This option may be used multiple times to exclude multiple files.

=item B<-rewrite=old=new>

This option allows a file to be renamed during the manifest generation
process. This may be used within the build environment to allow the auto.pl file
to be known as build.pl within the manifest.

This option may be used multiple times to rename multiple files.

=back

=head1 DESCRIPTION

This program will generate and check a "manifest" file.

The file consists of a line of information for each file within the
specified directory tress. The file is created and processed by the MD5SUM
utility.

=head1 EXAMPLE

=head2 Generation

 jats etool jats_manifest -manifest=MANIFEST
                          -rootdir=c:/clearcase/view/DPG_SWBase
                          -rewrite=auto.pl=build.pl

This command will generate a file called MANIFEST in the current directory based
on all files in the directory tree rooted at c:/clearcase/view/DPG_SWBase. The
file 'auto.pl' will be stored in the manifest as build.pl.

=head2 Checking

 jats etool jats_manifest -manifest=MANIFEST
                          -rootdir=c:/my_static_view/DPG_SWBase
                          -check

This command will check the directory tree rooted at
c:/my_static_view/DPG_SWBase against the MANIFEST file found in the local directory.

=cut