Subversion Repositories DevTools

#!/bin/bash

# Copyright (c) VIX TECHNOLOGY (AUST) LTD

#set -x

################################################################################

#   Script to assist in managing dpkg_archive

#   It may do too much. It does:

#   * Reports free space on drives used

#   * Create a list of files in dpkg_archive - for users to grep

#   * Create a list of packages in dpkg_archive

#   * Generate a report of packages added and removed

#   * Scan dpkg_archive for broen links - at the top level only

#     May be historic. When dpkgarchive was spread over several disks

#   * Scan the 'releases' area for

#     * Broken links

#     * Absolute links

#

#

################################################################################

#

#   Config

#

MAILTO=David.Purdie@vixtechnology.com,John.Rowling@vixtechnology.com

FROM=buildadm@vixtechnology.com

CCTO=

PATH=$HOME/bin:$HOME/cron:$PATH

DPKG=/export/devl/dpkg_archive

RELEASES=/export/devl/releases

#QUARANTINE=/export/devl/quarantine

BASEDIR=$DPKG

METADIR=.dpkg_archive/pkg_cron

OUTDIR=$BASEDIR/$METADIR

GFIND=/usr/bin/find

GDF=/bin/df

#MOUNTS="$DPKG $RELEASES $QUARANTINE"

MOUNTS="$DPKG $RELEASES"

# Enable null globbing

#

shopt -s nullglob

#

#   Debug Control

#

DO_VERBOSE=

DO_FIND=1

DO_MAIL=1

DO_NEW=1

DO_REMOVED=1

DO_FILES=1

DO_SCAN=1

DATE=

YDATE=

#

#   Parse user (debug) options

#

for arg in $* ; do

    case $arg in

        nomail) DO_MAIL= ;;

        nofind) DO_FIND= ;;

        nofiles) DO_FILES= ;;

        noremoved) DO_REMOVED= ;;

        nonew)  DO_NEW= ;;

        noscan)  DO_SCAN= ;;

        today=*) DATE=${arg##*=} ;;

        yesterday=*) YDATE=${arg##*=} ;;

        mail=*) MAILTO=${arg##*=} ;;

        verbose) DO_VERBOSE=1 ;;

        *help)  echo "Options: nomail, nofind, nofiles, nonew, noremoved, noscan, today=xxx, yesterday=yyy, mail=zzz verbose"; exit 0 ;;

        * ) echo "Unknown option $arg"; exit 1 ;;

    esac

done

if [ $DO_VERBOSE ] ; then

    echo "DATE        : $DATE";

    echo "YDATE       : $YDATE";

    echo "MAILTO      : $MAILTO";

    echo "DO_FIND     : $DO_FIND";

    echo "DO_FILES    : $DO_FILES";

    echo "DO_NEW      : $DO_NEW";

    echo "DO_REMOVED  : $DO_REMOVED";

    echo "DO_SCAN     : $DO_SCAN";

    echo "DO_MAIL     : $DO_MAIL";

fi

if [ ! "$DATE" ] ; then

    DATE=$(date +%Y-%m-%d)

fi

#

# Determine Yesterdays date to locate yesterdays logfile

#

if [ ! "$YDATE" ] ; then

    DAYS=( Sat Sun Mon Tue Wed Thu Fri Sat)

    MONTHS=( Dec Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)

    YESTERDAY=$((`date +%e` -1))

    MONTH=`date +%m`

    YEAR=`date +%Y`

    NDAY=`date +%u`

    WEEKDAY=${DAYS[`date +%u`]}

    MONTH=${MONTH#0}

    if [ $YESTERDAY -eq "0" ]

    then

        MONTH=$((MONTH-1))

        if [ $MONTH -eq "0" ]

        then

            MONTH=12

            YEAR=$((YEAR-1))

        fi

        set `cal $MONTH $YEAR`

        shift $(($# - 1))

        YESTERDAY=$1

    fi

    TMONTH=${MONTHS[MONTH]}

    # echo $WEEKDAY $YESTERDAY $MONTH $TMONTH $YEAR

    #echo $YESTERDAY

    YDATE=$( printf "%4.4d-%2.2d-%2.2d" $YEAR $MONTH $YESTERDAY )

fi

# Files for the package-file-list

LFILE=$OUTDIR/dpkg_archive_list.$DATE.txt

LLFILE=$METADIR/dpkg_archive_list.$DATE.txt

L2FILE=dpkg_archive_list.$DATE.txt

CFILE=$BASEDIR/dpkg_archive_list.txt

C2FILE=$OUTDIR/dpkg_archive_list.txt

# Files for the package-list

PFILE=$OUTDIR/dpkg_archive_pkg.$DATE.txt            # Todays File

YFILE=$OUTDIR/dpkg_archive_pkg.$YDATE.txt           # Yesterdays File

P1FILE=$METADIR/dpkg_archive_pkg.$DATE.txt          # Symlink source for file in Archive directory

P2FILE=dpkg_archive_pkg.$DATE.txt                   # Symlink source for file in Meta directory

PL1FILE=$BASEDIR/dpkg_archive_pkg.txt               # Symlink target for file in Archive directory

PL2FILE=$OUTDIR/dpkg_archive_pkg.txt                # Symlink target for file in Meta directory

if [ "$DO_VERBOSE" ] ; then

    echo "PFILE: $PFILE"

    echo "YFILE: $YFILE"

    echo "LFILE: $LFILE"

fi

mkdir -p $OUTDIR

#

#   Find files

#   Find packages

#

if [ "$DO_FIND" ] ; then

    [ $DO_VERBOSE ] && echo "Finding all packages in the archive"

    #

    # Determine packages in archive

    #

    $GFIND -L $BASEDIR -maxdepth 2 -type d  > $PFILE

    # Create symlinks to the package-list

    #

    rm -f $PL1FILE

    ln -s $P1FILE $PL1FILE

    rm -f $PL2FILE

    ln -s $P2FILE $PL2FILE

    #

    #   Determine files in the archive

    #   Keep a link to simplify user access to the list

    #

    if [ $DO_FILES ] ; then

        [ $DO_VERBOSE ] && echo "Finding all Files in the archive"

        $GFIND -L $BASEDIR | sort > $LFILE

        #

        #   Create a link to the current list of all files

        #

        rm -f $CFILE

        ln -s $LLFILE $CFILE

        rm -f $C2FILE

        ln -s $L2FILE $C2FILE

    fi

fi

f1=/tmp/$$.file1

f2=/tmp/$$.file2

f3=/tmp/$$.file3

echo >> $f3 '<pre style="font-size: medium">'

echo >> $f3 "dpkg_archive Report"

echo >> $f3 "Yesterday:  $YFILE"

echo >> $f3 "Today    :  $PFILE"

echo >> $f3 ""

$GDF -h $MOUNTS >> $f3

#

#   Determine new packages

#

#echo "Yesterday:  $YFILE"

#echo "Today    :  $PFILE"

if [ -f $YFILE ] ; then

    if [ "$DO_NEW" ] ; then

        [ $DO_VERBOSE ] && echo "Finding new packages in the archive"

        echo >> $f3 ""

        echo >> $f3 "New packages added to dpkg_archive"

        totalsize=0

        sort $YFILE > $f1

        sort $PFILE > $f2

        newp=$( comm -13 $f1 $f2)

        for file in $newp ; do

            pname="${file##$BASEDIR/}"

            pver="${pname##*/}"

            pname="${pname%%/*}"

            if [ "$pname" = ".dpkg_archive" ] ; then

                continue

            fi

            if [ "$pname" = "$pver" ] ; then

                printf " D%10s %8s %10s %s\n" "-" "NewDir" "-" $file

                continue

            fi

            built='-'

            if [ -f $file/descpkg ] ; then

               built=$( sed -n '-e/^Released By:/s~.*: *\([-_.0-9a-zA-Z]*\).*~\1~p' '-e/^Built By:/s~.*: *\([-_.0-9a-zA-Z]*\).*~\1~p' < $file/descpkg)

            fi

            flag='*'

            [ "$built" = "buildadm" ] && flag=' '

            tag='ManCpy'

            for ii in $file/built.* ; do

                tag='Auto'

            done

            note='-'

            ls 1>/dev/null 2>&1 $file/doc/RELEASE_NOTES_* || note='N'

            result=$(du -ks $file)

            psize=$(echo $result | { read first rest; echo $first;})

            let "totalsize+=$psize"

            printf "%c%c%10s %8s %10s %s\n" "$note" "$flag" "$built" "$tag" $result

        done >> $f3

        let "sizeGb=$totalsize/1024/1024"

        echo >> $f3 "Total Blocks added: $totalsize, $sizeGb GB"

    fi

    if [ "$DO_REMOVED" ] ; then

        [ $DO_VERBOSE ] && echo "Finding removed packages in the archive"

        echo >> $f3 ""

        echo >> $f3 "Removed Package Versions"

        sort $YFILE > $f1

        sort $PFILE > $f2

        remp=$( comm -23 $f1 $f2 )

        for file in $remp ; do

            pname="${file##$BASEDIR/}"

            pver="${pname##*/}"

            pname="${pname%%/*}"

            if [ "$pname" != "$pver" ] ; then

                printf "  %10s %8s %10s %s\n" "-" "Deleted" "-" $file

                continue

            fi

        done >> $f3

    fi

    #

    #   Broken link report

    #

    if [ "$DO_SCAN" ] ; then

        #

        #   Scan dpkg_archive for broken and funny links

        #

        echo "" >> $f3

        echo "Dpkg_archive Link Report" >> $f3

        scan_release.pl $DPKG >> $f3

        #

        #   Scan releases area for broken and funny links

        #

        echo "" >> $f3

        echo "Releases Link Report" >> $f3

        scan_release.pl $RELEASES >> $f3

        #

        #   Scan unexpected facl state

        #

        echo "" >> $f3

        echo "dpkg_archive ACL Report" >> $f3

        faclscan.pl >> $f3

    fi

echo >> $f3 "</pre>"

    #

    #   Mail results

    #

    if [ "$DO_MAIL" ] ; then

        [ $DO_VERBOSE ] && echo "Mailing Results"

        sendEmail.pl -q -f ${FROM} -t ${MAILTO} ${CCTO} -u "[dpkg_archive] $DATE Archive Report from $HOSTNAME" '-o' 'message-content-type=html' -o "message-file=$f3"

    else

        cat $f3

    fi

else

    echo "Yesterdays package file not found"

    echo "Yesterday:  $YFILE"

    echo "Today    :  $PFILE"

fi

#

#   Clean up

#

rm -f /tmp/$$.file*

#

#   Remove old copies of files from the metadata directory

#   Compress old versions ( older than 10 days )

#

$GFIND $OUTDIR -type f -mtime +100 -exec rm -f  {} \;

$GFIND $OUTDIR -type f -mtime +10 -name '*.txt' -exec gzip -q {} \;