Subversion Repositories DevTools

#!/bin/bash
# Copyright (c) VIX TECHNOLOGY (AUST) LTD
#set -x
################################################################################
#   Script to assist in managing dpkg_archive
#   It may do too much. It does:
#   * Reports free space on drives used
#   * Create a list of files in dpkg_archive - for users to grep
#   * Create a list of packages in dpkg_archive
#   * Generate a report of packages added and removed
#   * Scan dpkg_archive for broen links - at the top level only
#     May be historic. When dpkgarchive was spread over several disks
#   * Scan the 'releases' area for
#     * Broken links
#     * Absolute links
#
#
################################################################################
#
#   Config
#
MAILTO=David.Purdie@vixtechnology.com,John.Rowling@vixtechnology.com
FROM=buildadm@vixtechnology.com
CCTO=
PATH=$HOME/bin:$HOME/cron:$PATH
DPKG=/export/devl/dpkg_archive
RELEASES=/export/devl/releases
#QUARANTINE=/export/devl/quarantine
BASEDIR=$DPKG
METADIR=.dpkg_archive/pkg_cron
OUTDIR=$BASEDIR/$METADIR
GFIND=/usr/bin/find
GDF=/bin/df
#MOUNTS="$DPKG $RELEASES $QUARANTINE"
MOUNTS="$DPKG $RELEASES"


# Enable null globbing
#
shopt -s nullglob
#
#   Debug Control
#
DO_VERBOSE=
DO_FIND=1
DO_MAIL=1
DO_NEW=1
DO_REMOVED=1
DO_FILES=1
DO_SCAN=1
DATE=
YDATE=

#
#   Parse user (debug) options
#
for arg in $* ; do
    case $arg in
        nomail) DO_MAIL= ;;
        nofind) DO_FIND= ;;
        nofiles) DO_FILES= ;;
        noremoved) DO_REMOVED= ;;
        nonew)  DO_NEW= ;;
        noscan)  DO_SCAN= ;;
        today=*) DATE=${arg##*=} ;;
        yesterday=*) YDATE=${arg##*=} ;;
        mail=*) MAILTO=${arg##*=} ;;
        verbose) DO_VERBOSE=1 ;;
        *help)  echo "Options: nomail, nofind, nofiles, nonew, noremoved, noscan, today=xxx, yesterday=yyy, mail=zzz verbose"; exit 0 ;;
        * ) echo "Unknown option $arg"; exit 1 ;;
    esac
done

if [ $DO_VERBOSE ] ; then
    echo "DATE        : $DATE";
    echo "YDATE       : $YDATE";
    echo "MAILTO      : $MAILTO";
    echo "DO_FIND     : $DO_FIND";
    echo "DO_FILES    : $DO_FILES";
    echo "DO_NEW      : $DO_NEW";
    echo "DO_REMOVED  : $DO_REMOVED";
    echo "DO_SCAN     : $DO_SCAN";
    echo "DO_MAIL     : $DO_MAIL";
fi

if [ ! "$DATE" ] ; then
    DATE=$(date +%Y-%m-%d)
fi

#
# Determine Yesterdays date to locate yesterdays logfile
#
if [ ! "$YDATE" ] ; then
    DAYS=( Sat Sun Mon Tue Wed Thu Fri Sat)
    MONTHS=( Dec Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)

    YESTERDAY=$((`date +%e` -1))
    MONTH=`date +%m`
    YEAR=`date +%Y`
    NDAY=`date +%u`
    WEEKDAY=${DAYS[`date +%u`]}
    MONTH=${MONTH#0}
    if [ $YESTERDAY -eq "0" ]
    then
        MONTH=$((MONTH-1))

        if [ $MONTH -eq "0" ]
        then
            MONTH=12
            YEAR=$((YEAR-1))
        fi

        set `cal $MONTH $YEAR`
        shift $(($# - 1))
        YESTERDAY=$1
    fi

    TMONTH=${MONTHS[MONTH]}
    # echo $WEEKDAY $YESTERDAY $MONTH $TMONTH $YEAR
    #echo $YESTERDAY

    YDATE=$( printf "%4.4d-%2.2d-%2.2d" $YEAR $MONTH $YESTERDAY )
fi


# Files for the package-file-list
LFILE=$OUTDIR/dpkg_archive_list.$DATE.txt
LLFILE=$METADIR/dpkg_archive_list.$DATE.txt
L2FILE=dpkg_archive_list.$DATE.txt
CFILE=$BASEDIR/dpkg_archive_list.txt
C2FILE=$OUTDIR/dpkg_archive_list.txt

# Files for the package-list
PFILE=$OUTDIR/dpkg_archive_pkg.$DATE.txt            # Todays File
YFILE=$OUTDIR/dpkg_archive_pkg.$YDATE.txt           # Yesterdays File

P1FILE=$METADIR/dpkg_archive_pkg.$DATE.txt          # Symlink source for file in Archive directory
P2FILE=dpkg_archive_pkg.$DATE.txt                   # Symlink source for file in Meta directory

PL1FILE=$BASEDIR/dpkg_archive_pkg.txt               # Symlink target for file in Archive directory
PL2FILE=$OUTDIR/dpkg_archive_pkg.txt                # Symlink target for file in Meta directory

if [ "$DO_VERBOSE" ] ; then
    echo "PFILE: $PFILE"
    echo "YFILE: $YFILE"
    echo "LFILE: $LFILE"
fi
mkdir -p $OUTDIR

#
#   Find files
#   Find packages
#
if [ "$DO_FIND" ] ; then
    [ $DO_VERBOSE ] && echo "Finding all packages in the archive"
    #
    # Determine packages in archive
    #
    $GFIND -L $BASEDIR -maxdepth 2 -type d  > $PFILE

    # Create symlinks to the package-list
    #
    rm -f $PL1FILE
    ln -s $P1FILE $PL1FILE

    rm -f $PL2FILE
    ln -s $P2FILE $PL2FILE

    #
    #   Determine files in the archive
    #   Keep a link to simplify user access to the list
    #
    if [ $DO_FILES ] ; then
        [ $DO_VERBOSE ] && echo "Finding all Files in the archive"
        $GFIND -L $BASEDIR | sort > $LFILE

        #
        #   Create a link to the current list of all files
        #
        rm -f $CFILE
        ln -s $LLFILE $CFILE

        rm -f $C2FILE
        ln -s $L2FILE $C2FILE
    fi
    
fi

f1=/tmp/$$.file1
f2=/tmp/$$.file2
f3=/tmp/$$.file3

echo >> $f3 '<pre style="font-size: medium">'
echo >> $f3 "dpkg_archive Report"
echo >> $f3 "Yesterday:  $YFILE"
echo >> $f3 "Today    :  $PFILE"
echo >> $f3 ""
$GDF -h $MOUNTS >> $f3

#
#   Determine new packages
#
#echo "Yesterday:  $YFILE"
#echo "Today    :  $PFILE"
if [ -f $YFILE ] ; then

    if [ "$DO_NEW" ] ; then
        [ $DO_VERBOSE ] && echo "Finding new packages in the archive"

        echo >> $f3 ""
        echo >> $f3 "New packages added to dpkg_archive"
        totalsize=0
        sort $YFILE > $f1
        sort $PFILE > $f2
        newp=$( comm -13 $f1 $f2)
        for file in $newp ; do

            pname="${file##$BASEDIR/}"
            pver="${pname##*/}"
            pname="${pname%%/*}"

            if [ "$pname" = ".dpkg_archive" ] ; then
                continue
            fi

            if [ "$pname" = "$pver" ] ; then
                printf " D%10s %8s %10s %s\n" "-" "NewDir" "-" $file
                continue
            fi

            built='-'
            if [ -f $file/descpkg ] ; then
               built=$( sed -n '-e/^Released By:/s~.*: *\([-_.0-9a-zA-Z]*\).*~\1~p' '-e/^Built By:/s~.*: *\([-_.0-9a-zA-Z]*\).*~\1~p' < $file/descpkg)
            fi

            flag='*'
            [ "$built" = "buildadm" ] && flag=' '

            tag='ManCpy'
            for ii in $file/built.* ; do
                tag='Auto'
            done

            note='-'
            ls 1>/dev/null 2>&1 $file/doc/RELEASE_NOTES_* || note='N'

            result=$(du -ks $file)
            psize=$(echo $result | { read first rest; echo $first;})
            let "totalsize+=$psize"

            printf "%c%c%10s %8s %10s %s\n" "$note" "$flag" "$built" "$tag" $result
        done >> $f3

        let "sizeGb=$totalsize/1024/1024"
        echo >> $f3 "Total Blocks added: $totalsize, $sizeGb GB"
    fi

    if [ "$DO_REMOVED" ] ; then
        [ $DO_VERBOSE ] && echo "Finding removed packages in the archive"

        echo >> $f3 ""
        echo >> $f3 "Removed Package Versions"
        sort $YFILE > $f1
        sort $PFILE > $f2
        remp=$( comm -23 $f1 $f2 )
        for file in $remp ; do

            pname="${file##$BASEDIR/}"
            pver="${pname##*/}"
            pname="${pname%%/*}"

            if [ "$pname" != "$pver" ] ; then
                printf "  %10s %8s %10s %s\n" "-" "Deleted" "-" $file
                continue
            fi
        done >> $f3
    fi
    

    #
    #   Broken link report
    #
    if [ "$DO_SCAN" ] ; then
        #
        #   Scan dpkg_archive for broken and funny links
        #
        echo "" >> $f3
        echo "Dpkg_archive Link Report" >> $f3
        scan_release.pl $DPKG >> $f3

        #
        #   Scan releases area for broken and funny links
        #
        echo "" >> $f3
        echo "Releases Link Report" >> $f3
        scan_release.pl $RELEASES >> $f3

        #
        #   Scan unexpected facl state
        #
        echo "" >> $f3
        echo "dpkg_archive ACL Report" >> $f3
        faclscan.pl >> $f3

    fi
echo >> $f3 "</pre>"



    #
    #   Mail results
    #
    if [ "$DO_MAIL" ] ; then
        [ $DO_VERBOSE ] && echo "Mailing Results"
        sendEmail.pl -q -f ${FROM} -t ${MAILTO} ${CCTO} -u "[dpkg_archive] $DATE Archive Report from $HOSTNAME" '-o' 'message-content-type=html' -o "message-file=$f3"
    else
        cat $f3
    fi

else
    echo "Yesterdays package file not found"
    echo "Yesterday:  $YFILE"
    echo "Today    :  $PFILE"
fi

#
#   Clean up
#
rm -f /tmp/$$.file*

#
#   Remove old copies of files from the metadata directory
#   Compress old versions ( older than 10 days )
#
$GFIND $OUTDIR -type f -mtime +100 -exec rm -f  {} \;
$GFIND $OUTDIR -type f -mtime +10 -name '*.txt' -exec gzip -q {} \;