Subversion Repositories DevTools

#!/usr/bin/perl -w

########################################################################

# Copyright (C) 2008 ERG Limited, All rights reserved

#

# Module name   : ddlfile.pl

# Module type   : Standalone utility

# Compiler(s)   : n/a

# Environment(s): windows (dos), unix (solaris and linux)

#

# Description   : Recreates a DDL file from the directory structure 

#                 created by schemadump.pl.  This is useful for checking

#                 that schemadump.pl works, and also is intended

#                 to assist in creating a database from a set of

#                 clearcase files.

#

# History       : Created by Haydon Knight May 2008

#

# Usage         : ddlfile.pl [options] -src=<srcDir> -dest=<destFile>

#

########################################################################

#######################################################################

# Use lines

#######################################################################

require 5.6.1;

use strict;

use warnings;

use Pod::Usage;                             # required for help support

use Getopt::Long;

use commonExports; # see commonExports.pm in this directory

#######################################################################

# Function prototypes

#######################################################################

sub startupChecks();

sub parseCommandLine();

sub main();

sub getFilesWithExtras();

sub insertExtraSlash(\@);

sub getOrderedSqlFiles();

sub sortSqlFiles(\@);

sub getSqlFiles($);

sub directoryListing($);

sub getDDLHeader();

sub getSearchDirs();

#######################################################################

# Constant global variables

#######################################################################

my $VERSION = "1.0.1";

#######################################################################

# Hash tables - these are specific to the release manager database

#               and need to be adjusted if running this script on other databases

#######################################################################

#######################################################################

# Other global variables

#######################################################################

my $srcDir;

my $destFile;

#######################################################################

# Main code

#######################################################################

parseCommandLine();

startupChecks();

main();

finish();

#######################################################################

# Function definitions

#######################################################################

#-------------------------------------------------------------------------------

# Function  : main

#

# Purpose   : Main function of this script

#

# Arguments : none

#

# Returns   : none

#

# Notes     : Does everything but setting up at start and tearing down at end.

#

sub main()

{

    my @outputLines = getDDLHeader(); # The header for the whole output file

    # Get the headers for each DDL-data-type 

    # i.e. a header for the sequences, a header for the procedures etc.

    my @headers = getHeaders();

    # Get a list of all directories to find .sql files in

    my @searchDirs = getSearchDirs();

    # These need extra slashes inserted

    my @filesWithExtras = getFilesWithExtras();

    for( my $i = 0; $i < scalar(@searchDirs); $i++)

    {

        logprint "Working with searchDir $i: '$searchDirs[$i]'";

        push @outputLines, "$headers[$i]\n";

        my @sqlFiles = getSqlFiles( $searchDirs[$i] );

        logprint "Initially got " . scalar(@sqlFiles) . " .sql files";

        sortSqlFiles( @sqlFiles );

        logprint "Now got " . scalar(@sqlFiles) . " .sql files";

        my $searchDirRelativePath = $searchDirs[$i];

        $searchDirRelativePath =~ s~/+$~~;           # get rid of trailing slashes

        $searchDirRelativePath =~ s~.*/(.*?/.*)~$1~; # make it into a relative path

        for( my $j = 0; $j < scalar(@sqlFiles); $j++)

        {

            my @sqlFileLines = readArray( $sqlFiles[$j] );

            foreach my $sqlFileLine (@sqlFileLines )

            {

                $sqlFileLine =~ s~\\~/~g;

            }

            push @outputLines, @sqlFileLines;

        }

        push @outputLines, "\n";

        logprint "Now have " . scalar(@outputLines) . " output lines";

    }

    writeArray( $destFile, @outputLines);

}

#-------------------------------------------------------------------------------

# Function  : getFilesWithExtras

#

# Purpose   : Reads in from a file a list of files that need to have extra slashes

#             inserted into them before consolidation into the recreated datapump dump.

#

# Arguments : none

#

# Returns   : @filesWithExtras - list of such files

#

# Notes     : The input file is written out by schemadump.pl - each line corresponds

#             to a file in package_body that had the ALTER PACKAGE part tacked on to the end.

#             This whole extras featureset is specific to the release manager database.

#

sub getFilesWithExtras()

{

    my @filesWithExtras;

    open( O, "$srcDir/bookKeeping/filesWithExtras") or 

        die "Could not open '$srcDir/bookKeeping/filesWithExtras'\n";

    while( <O> )

    {

        s~[\n\r]+$~~;

        s~\\~/~g;

        s~.*/(.*?/.*?/.*)~$1~;

        push @filesWithExtras, "$srcDir/$_";

    }

    close( O );

    return @filesWithExtras;

}

#-------------------------------------------------------------------------------

# Function  : insertExtraSlash

#

# Purpose   : Adds in a single line containing a slash to an array

#

# Arguments : $refSqlFileLines (io) - reference to @sqlFileLines - a list of file lines

#                                     corresponding to one particular .sql file.

#

# Returns   : none

#

# Notes     : This function is only needed for 'PACKAGE BODY' in the release manager database,

#             and is quite specific to that.

#

sub insertExtraSlash(\@)

{

    my ($refSqlFileLines) = @_;

    # Stores line number containing 'ALTER PACKAGE'

    my $alterPackageLine = -1;

    for( my $i = $#$refSqlFileLines; $i >= 0; $i--) # iterate backwards through the array

    {

        if( $$refSqlFileLines[$i] =~ m~ALTER PACKAGE~ )

        {

            $alterPackageLine = $i;

            last;

        }

    }

    return if $alterPackageLine <= 0;

    my $needToAddASlash = 0;

    for( my $i = $alterPackageLine; $i >= 0; $i--) # iterate backwards through the array

    {

        return if $$refSqlFileLines[$i] =~ m~^\s*/\s*$~; # already has a slash

        next if $$refSqlFileLines[$i] =~ m~^\s*$~ ; # blank line - keep looking

        $needToAddASlash = 1;

        last; # line has some other text on it - we need to add a slash

    }

    return unless $needToAddASlash;

    splice @$refSqlFileLines, $alterPackageLine, 0, ("/\n");

}

#-------------------------------------------------------------------------------

# Function  : getOrderedSqlFiles

#

# Purpose   : Reads in a list of .sql files - this list has the same ordering as the

#             objects had in the original datapump dump file.

#

# Arguments : none

#

# Returns   : @orderedSqlFiles - list of .sql files (full paths)

#

# Notes     :

#

sub getOrderedSqlFiles()

{

    my @orderedSqlFiles;

    open( INDEXFILE, "$srcDir/bookKeeping/orderedFiles" ) or 

        die "Could not open file '$srcDir/bookKeeping/orderedFiles' for reading\n";

    while( <INDEXFILE> )

    {

        s~[\n\r]+$~~;

        s~\\~/~g;

        push @orderedSqlFiles, $_;

    }

    close(INDEXFILE);

    return @orderedSqlFiles;

}

#-------------------------------------------------------------------------------

# Function  : sortSqlFiles

#

# Purpose   : Takes a list of .sql files and sorts them so that they have the same

#             order as their objects in the original datapump dump file.

#

# Arguments : $refSqlFiles (io) - reference to @sqlFiles - list of .sql files to be sorted

#

# Returns   : none

#

# Notes     : Algorithm has the following steps:

#               + get relative paths to .sql files in @sqlFiles

#               + get list of all .sql files that were originally written out.  Store their

#                    relative paths in @orderedSqlFiles

#               + get a list of .sql files that are in both @sqlFiles and @orderedSqlFiles

#                    These are ordered since @orderedSqlFiles is ordered and are stored

#                    in @goodOrderedSqlFiles.

#               + get a list of .sql files that are in @sqlFiles but not in @orderedSqlFiles.

#                    Store in @newSqlFiles

#               + Set passed in list as (@goodOrderedSqlFiles, @newSqlFiles)

#                    It is now sorted.

#

#             I have not tested the @newSqlFiles functionality - HSK May 2008.

#

sub sortSqlFiles(\@)

{

    # @sqlFiles is a list of files we need to sort

    # Files not in original datapump dump need to come last

    # Files in original datapump dump need to be sorted in the same order

    # that they were in in the enormous datapump dump file

    my ($refSqlFiles) = @_;

    my @sqlFiles = @$refSqlFiles;

    foreach my $sqlFile (@sqlFiles)

    {

        # Turn into relative path

        $sqlFile =~ s~.*/(.*?/.*?/.*)~$1~;

    }

    # A list of all files written out in the original datapump dump

    # These are in order of the order datapump put them into its enormous file

    my @orderedSqlFiles = getOrderedSqlFiles();

    foreach my $sqlFile (@orderedSqlFiles)

    {

        # Turn into relative path

        $sqlFile =~ s~.*/(.*?/.*?/.*)~$1~;

    }

    logprint "Got " . scalar(@orderedSqlFiles) . " orderedSqlFiles; the first is '$orderedSqlFiles[0]'";

    # These are those files in @sqlFiles that were also in the

    # original datapump dump

    my @goodOrderedSqlFiles;

    foreach my $orderedSqlFile (@orderedSqlFiles)

    {

        push @goodOrderedSqlFiles, $orderedSqlFile if isOneOf( $orderedSqlFile, @sqlFiles);

    }

    logprint "Got " . scalar(@goodOrderedSqlFiles) . " goodOrderedSqlFiles";

    # In theory, now @goodOrderedSqlFiles should be the same files

    # as @sqlFiles, but in order (of course, files added since the

    # original datapump dump are excluded)

    my @newSqlFiles;

    foreach my $sqlFile (@sqlFiles)

    {

        push @newSqlFiles, $sqlFile if !isOneOf( $sqlFile, @goodOrderedSqlFiles);       

    }

    logprint "Got " . scalar(@newSqlFiles) . " newSqlFiles";

    # Get @$refSqlFiles to contain the sorted files

    # Here, we make no effort to sort files added since the datapump dump

    @$refSqlFiles = (@goodOrderedSqlFiles, @newSqlFiles);

    # Change back to a full path    

    foreach my $sqlFile (@$refSqlFiles)

    {

        $sqlFile = "$srcDir/$sqlFile";

    }

}

#-------------------------------------------------------------------------------

# Function  : getHeaders

#

# Purpose   : Construct a list of headers for each DDL-Data-Object

#

# Arguments : none

#

# Returns   : @headers - a list of the headers

#

# Notes     : Each header is just a one-line string.  (At least for the release manager database).

#             Both this function and getSearchDirs() read from $srcDir/bookKeeping/objectTypes

#

sub getHeaders()

{

#   Get headers that look like this:

#    -- new object type path is: SCHEMA_EXPORT/SYNONYM/SYNONYM

    my @headers;

    open( ORDERFILE, "$srcDir/bookKeeping/objectTypes");

    push @headers, "${subSectionSplitter}$_" while( <ORDERFILE> );

    close( ORDERFILE );

    foreach my $header (@headers)

    {

        $header =~ s~[\n\r]+$~~;

        $header =~ s~\\~/~g;

        logprint "Got a header: '$header'\n";

    }

    return @headers;

}

#-------------------------------------------------------------------------------

# Function  : getSearchDirs

#

# Purpose   : Returns a list of directories to search for files in

#

# Arguments : none

#

# Returns   : @searchDirs - the list (has a full path)

#

# Notes     : Both this function and getHeaders() read from $srcDir/bookKeeping/objectTypes

#

sub getSearchDirs()

{

    my @searchDirs;

    open( ORDERFILE, "$srcDir/bookKeeping/objectTypes");

    while( <ORDERFILE> )

    {

        tr~A-Z~a-z~;

        s~[\n\r]+$~~;

        s~\\~/~g;

        my ($topDir, $botDir) = getTopBotDirs( $_ );

        push @searchDirs, "$srcDir/$topDir/$botDir";

    }

    close( ORDERFILE );

    return @searchDirs;

}

#-------------------------------------------------------------------------------

# Function  : getDDLHeader

#

# Purpose   : Reads the DDL header from a file

#

# Arguments : none

#

# Returns   : @ddlheaderLines - array of lines corresponding to the header

#

# Notes     : For the release manager database only a single line is returned.

#             This is '-- CONNECT RELEASE_MANAGER'.

#

sub getDDLHeader()

{

    my @ddlHeaderLines;

    open( DDLHEADER, "$srcDir/bookKeeping/DDLHeader") 

        or die "Could not open '$srcDir/bookKeeping/DDLHeader'\n";

    while( <DDLHEADER> )

    {

        s~[\n\r]+$~~;

        s~\\~/~g;

        push @ddlHeaderLines, $_;

    }

    close( DDLHEADER );

    return @ddlHeaderLines;

}

#-------------------------------------------------------------------------------

# Function  : directoryListing

#

# Purpose   : Lists the contents of a directory

#

# Arguments : $diry (i) - directory to be listed

#

# Returns   : @lines - files in the directory

#

# Notes     : Has separate unix/windows implementations.

#

sub directoryListing($)

{

    my ($diry) = @_;

    return runCommand("ls -1d $diry/*") if( $UNIX );

    (my $windowsDiry = $diry) =~ s~/~\\~g;

    my @lines = runCommand("dir $windowsDiry /B");

    my @goodLines;

    foreach my $line (@lines)

    {

        push @goodLines, "$diry/$line" unless $line =~ m~^\s*$~;

    }

    return @goodLines;

}

#-------------------------------------------------------------------------------

# Function  : getSqlFiles

#

# Purpose   : Gets a list of the .sql files for a particular directory

#

# Arguments : $searchDir - directory to search

#

# Returns   : @sqlFiles - list (full file paths)

#

# Notes     :

#

sub getSqlFiles($)

{

    my ($searchDir) = @_;

    my @files = directoryListing( $searchDir );

    my @sqlFiles;

    foreach my $file (@files)

    {

        push @sqlFiles, $file if $file =~ /\.sql$/;

    }

    return @sqlFiles;

}

#-------------------------------------------------------------------------------

# Function  : parseCommandLine

#

# Purpose   : Parses command line; invokes help if necessary

#

# Arguments : nothing

#

# Returns   : nothing

#

# Notes     : Sets up various global variables; these are not checked here - they should be

#             checked in startupChecks()

#

sub parseCommandLine()

{

    my $opt_help = 0;

    my $opt_manual = 0;

    my $result = GetOptions (

    "help"          => \$opt_help,              # flag

    "manual"        => \$opt_manual,            # flag

    "verbose+"      => \$verbose,               # flag

    "src=s"         => \$srcDir,                # String

    "dest=s"        => \$destFile,              # String

    "log=s"         => \$logFile,               # String

    );

    pod2usage(-verbose => 2) if( $opt_manual );

    pod2usage(-verbose => 0, -message => "Version: $VERSION") if ($opt_help > 0 || ! $result );

}

#-------------------------------------------------------------------------------

# Function  : startupChecks

#

# Purpose   : Checks that important variables are set sanely

#

# Arguments : nothing

#

# Returns   : nothing

#

# Notes     : Calls die() if things aren't set sanely.

#             This function opens the logfile.

#

sub startupChecks()

{

    die "You need to specify a source directory using '-src=<srcDir>'\n" unless $srcDir;

    die "You need to specify a destination file using '-dest=<destFile>'\n"

        unless $destFile;

    $srcDir   =~ s~\\~/~g;

    $destFile =~ s~\\~/~g;

    die "Source directory '$srcDir' does not exist\n" unless -d $srcDir;

    openLog();

}

#######################################################################

# Documentation

#######################################################################

=pod

=head1 NAME

ddlfile.pl - creates a DDL file from a set of directories - basically

does the opposite of schemadump.pl

=head1 SYNOPSIS

ddlfile.pl [options] -src=<srcDir> -dest=<destFile>

Options:

  -help              - brief help message

  -man               - Full documentation

  -src=srcDir        - Directory to extract files from (mandatory)

  -dest=destFile     - File to pack to (mandatory)

  -log=logFile       - Log messages to this file

=head1 OPTIONS

=over 8

=item B<-help>

Print a brief help message and exits.

=item B<-man>

Prints the manual page and exits.

=item B<-src=srcDir>

Specify what file to extract out the clearcase files from.

=item B<-dest=destFile>

Specify where to pack the clearcase files to.

=item B<-log=logFile>

Specify a file to write log messages to.  Default is to just write

to the terminal.

=back

=head1 DESCRIPTION

This script is used to recreate the output of datapump.  It is

designed for use with the release manager database.

=cut