Subversion Repositories DevTools

Rev

Rev 7396 | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 7396 Rev 7397
Line 61... Line 61...
61 
my $conf;
 61 
my $conf;
62 
my $yday = -1;
 62 
my $yday = -1;
63 
my $linkUp = 1;
 63 
my $linkUp = 1;
64 
my $RM_DB;
 64 
my $RM_DB;
65 
my $activeReleases;
 65 
my $activeReleases;
- 
 
 66 
my $wedgedCount = 0;
66 
 
 67 
 
67 
#
 68 
#
68 
#   Contain statisics maintained while operating
 69 
#   Contain statisics maintained while operating
69 
#       Can be dumped with a kill -USR2
 70 
#       Can be dumped with a kill -USR2
70 
#       List here for documentation
 71 
#       List here for documentation
Line 75... Line 76...
75 
    timeStamp => 0,                     # DateTime when statistics are dumped
 76 
    timeStamp => 0,                     # DateTime when statistics are dumped
76 
    upTime => 0,                        # Seconds since program start
 77 
    upTime => 0,                        # Seconds since program start
77 
    Cycle => 0,                         # Major process loop counter
 78 
    Cycle => 0,                         # Major process loop counter
78 
    phase => 'Init',                    # Current phase of operation
 79 
    phase => 'Init',                    # Current phase of operation
79 
    state => 'OK',                      # Nagios state
 80 
    state => 'OK',                      # Nagios state
- 
 
 81 
    wedged => 0,                        # Wedge indication - main loop not cycling
80 
                                        #
82 
                                        #
81 
                                        # The following are reset each day
 83 
                                        # The following are reset each day
82 
    dayStart => 0,                      # DateTime when daily data was reset
 84 
    dayStart => 0,                      # DateTime when daily data was reset
83 
    txCount => 0,                       # Packages Transferred
 85 
    txCount => 0,                       # Packages Transferred
- 
 
 86 
    txBytes => 0,                       # Bytes Transferred
84 
    delCount => 0,                      # Packages marked for deletion
 87 
    delCount => 0,                      # Packages marked for deletion
85 
    linkErrors => 0,                    # Transfer (S3) errors
 88 
    linkErrors => 0,                    # Transfer (S3) errors
86 
                                        #
89 
                                        #
87 
                                        # Per Cycle Data - Calculated each processing Cycle
 90 
                                        # Per Cycle Data - Calculated each processing Cycle
88 
);
 91 
);
Line 102... Line 105...
102 
    'verbose'         => {'default'   => 0      , 'fmt' => 'int'},                  # Debug ...
 105 
    'verbose'         => {'default'   => 0      , 'fmt' => 'int'},                  # Debug ...
103 
    'active'          => {'default'   => 1      , 'fmt' => 'bool'},                 # Disable alltogether
 106 
    'active'          => {'default'   => 1      , 'fmt' => 'bool'},                 # Disable alltogether
104 
    'debug'           => {'default'   => 0      , 'fmt' => 'bool'},                 # Log to screen
 107 
    'debug'           => {'default'   => 0      , 'fmt' => 'bool'},                 # Log to screen
105 
    'txdetail'        => {'default'   => 0      , 'fmt' => 'bool'},                 # Show transfer times
 108 
    'txdetail'        => {'default'   => 0      , 'fmt' => 'bool'},                 # Show transfer times
106 
    'noTransfers'     => {'default'   => 0      , 'fmt' => 'bool'},                 # Debugging option to prevent transfers
 109 
    'noTransfers'     => {'default'   => 0      , 'fmt' => 'bool'},                 # Debugging option to prevent transfers
- 
 
 110 
    'transferDir'     => {'default'   => 'pkg/S3TRANSFER' , 'fmt' => 'text'},
107 
 
 111 
 
108 
    'tagdir'          => {'mandatory' => 1      , 'fmt' => 'dir'},
 112 
    'tagdir'          => {'mandatory' => 1      , 'fmt' => 'mkdir'},
109 
    'workdir'         => {'mandatory' => 1      , 'fmt' => 'dir'},
 113 
    'workdir'         => {'mandatory' => 1      , 'fmt' => 'mkdir'},
110 
    'forcedirscan'    => {'default'   => 100    , 'fmt' => 'period'},
 114 
    'forcedirscan'    => {'default'   => 100    , 'fmt' => 'period'},
111 
    'forces3update'   => {'default'   => '30m'  , 'fmt' => 'period'},
 115 
    'forces3update'   => {'default'   => '30m'  , 'fmt' => 'period'},
112 
    'tagListUpdate'   => {'default'   => '1h'   , 'fmt' => 'period'},
 116 
    'tagListUpdate'   => {'default'   => '1h'   , 'fmt' => 'period'},
113 
    'S3Bucket'        => {'mandatory' => 1      , 'fmt' => 'text'},
 117 
    'S3Bucket'        => {'mandatory' => 1      , 'fmt' => 'text'},
114 
    'S3Profile'       => {'mandatory' => 1      , 'fmt' => 'text'},
 118 
    'S3Profile'       => {'mandatory' => 1      , 'fmt' => 'text'},
Line 133... Line 137...
133 
#
 137 
#
134 
while (1)
 138 
while (1)
135 
{
 139 
{
136 
    $logger->verbose3("Processing");
 140 
    $logger->verbose3("Processing");
137 
    $statistics{Cycle}++;
 141 
    $statistics{Cycle}++;
- 
 
 142 
    $wedgedCount = 0;
138 
    $now = time();
 143 
    $now = time();
139 
 
 144 
 
140 
    $statistics{phase} = 'ReadConfig';
 145 
    $statistics{phase} = 'ReadConfig';
141 
    readConfig();
 146 
    readConfig();
142 
    if ( $conf->{'active'} )
 147 
    if ( $conf->{'active'} )
Line 275... Line 280...
275 
 
 280 
 
276 
        #
 281 
        #
277 
        #   Examine the s3 bucket and extract useful information
 282 
        #   Examine the s3 bucket and extract useful information
278 
        #
 283 
        #
279 
        my $startTime = time;
 284 
        my $startTime = time;
280 
        $linkUp = 1;
 - 
 
281 
        $rv =  examineS3Bucket();
 285 
        $rv =  examineS3Bucket();
282 
         unless ($rv) {
 286 
         unless ($rv) {
283 
            $statistics{linkErrors}++;
 287 
            $statistics{linkErrors}++;
284 
            $linkUp = 0;
 288 
            $linkUp = 0;
- 
 
 289 
         } else {
- 
 
 290 
             $linkUp = 1;
285 
         }
 291 
         }
286 
 
 292 
 
287 
         #
 293 
         #
288 
         #   Display the duration of the refresh
 294 
         #   Display the duration of the refresh
289 
         #       Diagnostic use
 295 
         #       Diagnostic use
Line 651... Line 657...
651 
        return;
 657 
        return;
652 
    }
 658 
    }
653 
    $logger->verbose("Dependency Test: $reason");
 659 
    $logger->verbose("Dependency Test: $reason");
654 
 
 660 
 
655 
    #
 661 
    #
- 
 
 662 
    #   Create a monifest to go into the zip
- 
 
 663 
    #
- 
 
 664 
    my $manifest;
- 
 
 665 
    $manifest->{Packages} = [];
- 
 
 666 
    $manifest->{rtag_id} = $rtag_id;
- 
 
 667 
 
- 
 
 668 
    #
656 
    #   Generate the zip of the obejcets to be pushed to S3
 669 
    #   Generate the zip of the objects to be pushed to S3
- 
 
 670 
    #       Add directories
- 
 
 671 
    #       Update the manifest entries
657 
    #
 672 
    #
658 
    my $startTime = time;
 673 
    my $startTime = time;
659 
    my $zip = Archive::Zip->new();
 674 
    my $zip = Archive::Zip->new();
660 
    foreach my $entry (@{$curData}) {
 675 
    foreach my $entry (@{$curData}) {
661 
        my $src = getPackageBase($entry->[0], $entry->[1]);
 676 
        my $src = getPackageBase($entry->[0], $entry->[1]);
662 
        if (defined $src) {
 677 
        if (defined $src) {
663 
            $logger->verbose("Zip addTree Src: $src");
 678 
            $logger->verbose("Zip addTree Src: $src");
664 
 
 679 
 
- 
 
 680 
            my %data;
- 
 
 681 
            $data{name} = $entry->[0];
- 
 
 682 
            $data{version} = $entry->[1];
- 
 
 683 
            $data{pvid} = $entry->[2];
- 
 
 684 
            push @{$manifest->{Packages}}, \%data;
- 
 
 685 
 
665 
            if ( $zip->addTree( $src, '' ) != AZ_OK ) {
 686 
            if ( $zip->addTree( $src, '' ) != AZ_OK ) {
666 
                $logger->warn("Zip addTree Error: $rtag_id");
 687 
                $logger->warn("Zip addTree Error: $rtag_id");
667 
                return;
 688 
                return;
668 
            }
 689 
            }
669 
        }
 690 
        }
670 
    }
 691 
    }
671 
 
 692 
 
- 
 
 693 
    #   Add the manifest into the zip
- 
 
 694 
    my $jsonText = to_json( $manifest, { ascii => 1, pretty => 1 });
- 
 
 695 
    $zip->addString( $jsonText, 'ReleaseManifest.json' );
- 
 
 696 
    $logger->verbose("ManfestJson: $jsonText");
- 
 
 697 
 
- 
 
 698 
    #   Generate the zip file
672 
    my $zipFile = catdir( $conf->{'workdir'} , 'Images-' . $rtag_id . '.zip');
 699 
    my $zipFile = catdir( $conf->{'workdir'} , 'Images-' . $rtag_id . '.zip');
673 
    if ( $zip->writeToFileNamed($zipFile) != AZ_OK ) {
 700 
    if ( $zip->writeToFileNamed($zipFile) != AZ_OK ) {
674 
        $logger->warn("Zip write Error: $rtag_id");
 701 
        $logger->warn("Zip write Error: $rtag_id");
675 
        return;
 702 
        return;
676 
    }
 703 
    }
Line 748... Line 775...
748 
        $logger->logmsg("S3 Copy: Stats: $rtag_id, $size Gb, $duration Secs");
 775 
        $logger->logmsg("S3 Copy: Stats: $rtag_id, $size Gb, $duration Secs");
749 
    }
 776 
    }
750 
 
 777 
 
751 
    if ($cmdRv == 0) {
 778 
    if ($cmdRv == 0) {
752 
        $statistics{txCount}++;
 779 
        $statistics{txCount}++;
- 
 
 780 
        $statistics{txBytes} += -s $zipFile;
753 
 
 781 
 
754 
        #
 782 
        #
755 
        #   Mark the current entry as having been processed
 783 
        #   Mark the current entry as having been processed
756 
        #
 784 
        #
757 
        $activeReleases->{$rtag_id}{depsig} = $depsig;
 785 
        $activeReleases->{$rtag_id}{depsig} = $depsig;
Line 788... Line 816...
788 
    unless ( -d $src ) {
 816 
    unless ( -d $src ) {
789 
        $logger->warn("addPartsToImage: Package not found: $pname, $pver");
 817 
        $logger->warn("addPartsToImage: Package not found: $pname, $pver");
790 
        return undef;
 818 
        return undef;
791 
    }
 819 
    }
792 
 
 820 
 
793 
    #$src = catdir( $src, 'pkg', 'S3TRANSFER');
 821 
    $src = catdir( $src, $conf->{'transferDir'});
794 
    $src = catdir( $src, 'bin');
 - 
 
795 
    unless ( -d $src ) {
 822 
    unless ( -d $src ) {
796 
        $logger->verbose("addPartsToImage: Package has no pkg/S3TRANSFER: $pname, $pver");
 823 
        $logger->verbose("addPartsToImage: Package has no $conf->{'transferDir'}: $pname, $pver");
797 
        return undef;
 824 
        return undef;
798 
    }
 825 
    }
799 
    return $src;
 826 
    return $src;
800 
}
 827 
}
801 
 
 828 
 
Line 954... Line 981...
954 
        $logger->logmsg('Resetting daily statistics' );
 981 
        $logger->logmsg('Resetting daily statistics' );
955 
 
 982 
 
956 
        # Note: Must match @recoverTags in readStatistics
 983 
        # Note: Must match @recoverTags in readStatistics
957 
        $statistics{dayStart} = $time;
 984 
        $statistics{dayStart} = $time;
958 
        $statistics{txCount} = 0;
 985 
        $statistics{txCount} = 0;
- 
 
 986 
        $statistics{txBytes} = 0;
959 
        $statistics{delCount} = 0;
 987 
        $statistics{delCount} = 0;
960 
        $statistics{linkErrors} = 0;
 988 
        $statistics{linkErrors} = 0;
961 
    }
 989 
    }
962 
}
 990 
}
963 
 
 991 
 
Line 971... Line 999...
971 
#
 999 
#
972 
# Returns         :
1000 
# Returns         :
973 
#
 1001 
#
974 
sub readStatistics
 1002 
sub readStatistics
975 
{
 1003 
{
976 
    my @recoverTags = qw(dayStart txCount delCount linkErrors);
 1004 
    my @recoverTags = qw(dayStart txCount txBytes delCount linkErrors);
977 
 
 1005 
 
978 
    if ($conf->{'statsfile'} and -f $conf->{'statsfile'})
 1006 
    if ($conf->{'statsfile'} and -f $conf->{'statsfile'})
979 
    {
 1007 
    {
980 
        if (open my $fh, $conf->{'statsfile'})
 1008 
        if (open my $fh, $conf->{'statsfile'})
981 
        {
 1009 
        {
Line 1014... Line 1042...
1014 
    #   A few local stats
 1042 
    #   A few local stats
1015 
    #
 1043 
    #
1016 
    $statistics{SeqNum}++;
 1044 
    $statistics{SeqNum}++;
1017 
    $statistics{timeStamp} = time();
 1045 
    $statistics{timeStamp} = time();
1018 
    $statistics{upTime} = $statistics{timeStamp} - $startTime;
 1046 
    $statistics{upTime} = $statistics{timeStamp} - $startTime;
- 
 
 1047 
    $statistics{wedged} = $wedgedCount++ > 30  ? 1 : 0;
- 
 
 1048 
    $statistics{state} = $statistics{wedged} ? 'Wedged' : $statistics{state};
- 
 
 1049 
    $statistics{state} = $linkUp ? $statistics{state} : 'S3 Link Errors' ;
1019 
 
 1050 
 
1020 
    #   Reset daily accumulations - on first use each day
 1051 
    #   Reset daily accumulations - on first use each day
1021 
    resetDailyStatistics($statistics{timeStamp});
 1052 
    resetDailyStatistics($statistics{timeStamp});
1022
1053
1023 
    #
 1054 
    #