Subversion Repositories DevTools

Rev

Rev 5272 | Rev 5535 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 5272 Rev 5534
Line 95... Line 95...
95
# May be supplemented by xxx.cnf file
95
# May be supplemented by xxx.cnf file
96
my %retainPkgs = (
96
my %retainPkgs = (
97
    'core_devl' => 1,
97
    'core_devl' => 1,
98
);
98
);
99
 
99
 
-
 
100
#
-
 
101
#   Statistics
-
 
102
#   Listed here to ensure that they exist in the stats file
-
 
103
#
-
 
104
my %statistics = (
-
 
105
    timeStamp               => 0,               # Age of the stats file
-
 
106
    statsName               => 'Quarantine',    # Name of the stats file
-
 
107
    state                   => 'OK',            # Overall reported state
-
 
108
 
-
 
109
    # Error counters
-
 
110
    QuarantineError         => 0,
-
 
111
    S3TransferError         => 0,
-
 
112
 
-
 
113
    # Major Statistics
-
 
114
    Quarantine              => 0,
-
 
115
 
-
 
116
    # Minor Statistics
-
 
117
    fileNotInReleaseManager => 0,
-
 
118
    inDeploymentManager     => 0,
-
 
119
    inSdk                   => 0,
-
 
120
    isPatch                 => 0,
-
 
121
    ManualBuild             => 0,
-
 
122
    RetainTime              => 0,
-
 
123
    NoBuildStandard         => 0,
-
 
124
    NoPackageEntry          => 0,
-
 
125
    NoPVid                  => 0,
-
 
126
    NotInArchive            => 0,
-
 
127
    NotInReleaseManager     => 0,
-
 
128
    NotLocked               => 0,
-
 
129
    SecondLevelPackage      => 0,
-
 
130
    TopLevelPackage         => 0,
-
 
131
    TotalPackages           => 0,
-
 
132
    ZeroTime                => 0,
-
 
133
);
-
 
134
 
100
#-------------------------------------------------------------------------------
135
#-------------------------------------------------------------------------------
101
# Function        : Main Entry
136
# Function        : Main Entry
102
#
137
#
103
# Description     :
138
# Description     :
104
#
139
#
Line 187... Line 222...
187
    prepQdir();
222
    prepQdir();
188
    unless ($opt_purge) {
223
    unless ($opt_purge) {
189
        readInputData();
224
        readInputData();
190
        processDpkgArchive();
225
        processDpkgArchive();
191
        reportMissingPkgs();
226
        reportMissingPkgs();
-
 
227
        reportStats();
192
    }
228
    }
193
 
229
 
194
    Verbose ("Quarantine to: $quarantineInstance");
230
    Verbose ("Quarantine to: $quarantineInstance");
195
    Verbose ("Log to: $logPath");
231
    Verbose ("Log to: $logPath");
196
}
232
}
Line 955... Line 991...
955
                    {
991
                    {
956
                        #
992
                        #
957
                        #   Package is in dpkg-archive, but not in Release
993
                        #   Package is in dpkg-archive, but not in Release
958
                        #   Manager. Allow for a short while
994
                        #   Manager. Allow for a short while
959
                        #
995
                        #
-
 
996
                        $statistics{TotalPackages}++;
-
 
997
                        $statistics{'NotInReleaseManager'}++;
960
                        if ( $mtime > $config{retainNoRm} )
998
                        if ( $mtime > $config{retainNoRm} )
961
                        {
999
                        {
962
                            #Log("Package not in RM: $pkgName, $pkgVersion, Age: $mtime");
1000
                            #Log("Package not in RM: $pkgName, $pkgVersion, Age: $mtime");
963
                            quarantineItem( 'X', $mtime, $pkgPath );
1001
                            quarantineItem( 'X', $mtime, $pkgPath );
-
 
1002
                            $statistics{'Quarantine'}++;
-
 
1003
                        }
-
 
1004
 
-
 
1005
                        if ($opt_explain)
-
 
1006
                        {
-
 
1007
                            Information("Reason:-, $pkgName, $pkgVersion, Reason:NotInReleaseManager");
964
                        }
1008
                        }
965
                    }
1009
                    }
966
 
1010
 
967
#Message("$pkgName, $pkgVersion, $pkgPvid{$pkgName}{$pkgVersion}{mtime}");
1011
#Message("$pkgName, $pkgVersion, $pkgPvid{$pkgName}{$pkgVersion}{mtime}");
968
                }
1012
                }
Line 972... Line 1016...
972
        elsif ( -f $pkgDir )
1016
        elsif ( -f $pkgDir )
973
        {
1017
        {
974
            Warning("Unexpected file in dpkg_archive: $pkgName");
1018
            Warning("Unexpected file in dpkg_archive: $pkgName");
975
            Log("Unexpected file in dpkg_archive: $pkgName");
1019
            Log("Unexpected file in dpkg_archive: $pkgName");
976
            quarantineItem( 'F', -1, $pkgDir );
1020
            quarantineItem( 'F', -1, $pkgDir );
-
 
1021
            $statistics{'fileNotInReleaseManager'}++;
-
 
1022
            $statistics{'Quarantine'}++;
-
 
1023
            $statistics{'NotInReleaseManager'}++;
-
 
1024
 
-
 
1025
            if ($opt_explain)
-
 
1026
            {
-
 
1027
                Information("Reason:-, $pkgDir, -, Reason:fileNotInReleaseManager");
-
 
1028
            }
977
        }
1029
        }
978
        else
1030
        else
979
        {
1031
        {
980
            Warning("Unexpected entry in dpkg_archive: $pkgName");
1032
            Warning("Unexpected entry in dpkg_archive: $pkgName");
981
        }
1033
        }
Line 997... Line 1049...
997
            my $keepReason = '';
1049
            my $keepReason = '';
998
            my $entry = $Packages{$pvid};
1050
            my $entry = $Packages{$pvid};
999
 
1051
 
1000
            {
1052
            {
1001
                # Examine entry. Determine a reason to keep the package
1053
                # Examine entry. Determine a reason to keep the package
-
 
1054
                #   Some reasons to keep a package are no longer needed now that versions are pumped into S3
1002
 
1055
 
1003
                unless ($entry) { $keepReason ='NoPackageEntry'; last;}
1056
                unless ($entry) { $keepReason ='NoPackageEntry'; last;}
1004
                unless ($entry->{dpkg_archive}) { $keepReason ='NotInArchive'; last;}
1057
                unless ($entry->{dpkg_archive}) { $keepReason ='NotInArchive'; last;}
1005
                unless ($pvid) { $keepReason = 'NoPVid'; last;}
1058
                unless ($pvid) { $keepReason = 'NoPVid'; last;}
1006
                if (exists $entry->{tlp}) { $keepReason ='tlp'; last;}
1059
                if (exists $entry->{tlp}) { $keepReason = 'TopLevelPackage'; last;}
1007
                if (exists $entry->{slp}) { $keepReason ='slp'; last;}
1060
                if (exists $entry->{slp}) { $keepReason = 'SecondLevelPackage'; last;}
1008
                if (exists $entry->{sdk}) { $keepReason ='sdk'; last;}
1061
                if (exists $entry->{sdk}) { $keepReason ='inSdk'; last;}
1009
                if (exists $entry->{dm}) { $keepReason ='dm'; last;}
1062
                if (exists $entry->{dm}) { $keepReason = 'inDeploymentManager'; last;}
1010
                if ($entry->{isPatch}) { $keepReason ='isPatch'; last;}
1063
                if ($entry->{isPatch}) { $keepReason = 'isPatch'; last;}
1011
                unless ($mtime) { $keepReason ='ZeroTime'; last;}
1064
                unless ($mtime) { $keepReason ='ZeroTime'; last;}
1012
                if ($mtime <= $config{retain}) { $keepReason ='mtime<Retain:' . ($config{retain} - $mtime); last;}
1065
                if ($mtime <= $config{retain}) { $keepReason ='RetainTime:' . ($config{retain} - $mtime); last;}
1013
                unless ($entry->{buildStandard}) { $keepReason ='NoBuildStandard' . $mtime; last;}
1066
                #unless ($entry->{buildStandard}) { $keepReason ='NoBuildStandard:' . $mtime; last;}
1014
                if ($entry->{locked} ne 'Y') { $keepReason ='NotLocked:' . $entry->{locked}; last;}
1067
                if ($entry->{locked} ne 'Y') { $keepReason ='NotLocked:' . $entry->{locked}; last;}
1015
                #if ($entry->{buildType} eq 'M') { $keepReason ='ManualBuild:' . $entry->{buildType}; last;}
1068
                #if ($entry->{buildType} eq 'M') { $keepReason ='ManualBuild:' . $entry->{buildType}; last;}
1016
 
1069
 
1017
                $pkgPvid{$pkgName}{$pkgVersion}{keepReason} = $keepReason;
1070
                $pkgPvid{$pkgName}{$pkgVersion}{keepReason} = $keepReason;
1018
            }
1071
            }
1019
 
1072
 
1020
            unless ( $keepReason )
1073
            unless ( $keepReason )
1021
            {
1074
            {
1022
                Verbose2("Quarantine:$pvid, $pkgName, $pkgVersion, Age:$mtime, Lock:$entry->{locked}, Patch:$entry->{isPatch}, BS:$entry->{buildStandard}, BT:$entry->{buildType}");
1075
                Verbose2("Quarantine:$pvid, $pkgName, $pkgVersion, Age:$mtime, Lock:$entry->{locked}, Patch:$entry->{isPatch}, BS:$entry->{buildStandard}, BT:$entry->{buildType}");
1023
                quarantineItem( 'Q', $mtime, join('/', $config{dpkg_archive}, $pkgName, $pkgVersion ) );
1076
                quarantineItem( 'Q', $mtime, join('/', $config{dpkg_archive}, $pkgName, $pkgVersion ) );
-
 
1077
                $keepReason = 'Quarantine';
1024
            }
1078
            }
1025
 
1079
 
1026
            if ($opt_explain)
1080
            if ($opt_explain)
1027
            {
1081
            {
1028
                Information("Reason:$pvid, $pkgName, $pkgVersion, Reason:$keepReason");
1082
                Information("Reason:$pvid, $pkgName, $pkgVersion, Reason:$keepReason");
1029
            }
1083
            }
-
 
1084
 
-
 
1085
            #
-
 
1086
            #   Maintain Stats
-
 
1087
            #       Only use the Base Reason - remove details after the ':' character
-
 
1088
            #
-
 
1089
            my $sReason = $keepReason;
-
 
1090
            $sReason =~ s~:.*$~~;
-
 
1091
            $statistics{$sReason}++;
-
 
1092
            $statistics{TotalPackages}++;
1030
        }
1093
        }
1031
    }
1094
    }
1032
 
1095
 
1033
    #
1096
    #
1034
    # Perform the quarantine
1097
    # Perform the quarantine
Line 1117... Line 1180...
1117
    }
1180
    }
1118
    return @releases;
1181
    return @releases;
1119
}
1182
}
1120
 
1183
 
1121
#-------------------------------------------------------------------------------
1184
#-------------------------------------------------------------------------------
-
 
1185
# Function        : reportStats 
-
 
1186
#
-
 
1187
# Description     : Report statistics
-
 
1188
#                   Write statistics to a file
-
 
1189
#                       Write to a tmp file, then rename.
-
 
1190
#                       Attempt to make the operation atomic - so that the file consumer
-
 
1191
#                       doesn't get a badly formed file.
-
 
1192
#   
-
 
1193
#
-
 
1194
# Inputs          : 
-
 
1195
#
-
 
1196
# Returns         : 
-
 
1197
#
-
 
1198
sub reportStats
-
 
1199
{
-
 
1200
    #
-
 
1201
    #   Time stamp the stats
-
 
1202
    #
-
 
1203
    $statistics{'timeStamp'} = time();
-
 
1204
 
-
 
1205
    #
-
 
1206
    #   Save stats to a known file for Nagios to use
-
 
1207
    #   
-
 
1208
    my $statsfiletmp = join('/', $config{logBase}, 'quarantine.stats.tmp' );
-
 
1209
    my $statsfile    = join('/', $config{logBase}, 'quarantine.stats');
-
 
1210
 
-
 
1211
    my $fh;
-
 
1212
    unless (open ($fh, '>', $statsfiletmp))
-
 
1213
    {
-
 
1214
        $fh = undef;
-
 
1215
        Warning("Cannot create temp stats file: $!");
-
 
1216
    }
-
 
1217
    else
-
 
1218
    {
-
 
1219
        foreach my $key ( sort { lc($a) cmp lc($b) } keys %statistics)
-
 
1220
        {
-
 
1221
            print $fh $key . ':' . $statistics{$key} . "\n";
-
 
1222
            Log('Statistics: '. $key . ':' . $statistics{$key});
-
 
1223
        }
-
 
1224
        close $fh;
-
 
1225
 
-
 
1226
        # Rename temp to real file
-
 
1227
        rename  $statsfiletmp,  $statsfile;
-
 
1228
    }
-
 
1229
}
-
 
1230
 
-
 
1231
 
-
 
1232
#-------------------------------------------------------------------------------
1122
# Function        : quarantineItem
1233
# Function        : quarantineItem
1123
#
1234
#
1124
# Description     : Add item to the list of stuff to be quarantined
1235
# Description     : Add item to the list of stuff to be quarantined
1125
#
1236
#
1126
# Inputs          : $reason         - Reason
1237
# Inputs          : $reason         - Reason
Line 1192... Line 1303...
1192
                {
1303
                {
1193
                    ReportError ("Move $path to S3");
1304
                    ReportError ("Move $path to S3");
1194
                    $s3msg = ' - with S3 error';
1305
                    $s3msg = ' - with S3 error';
1195
                    $s3error = 1;
1306
                    $s3error = 1;
1196
                    $emsg = ' - S3 Error prevented quarantine';
1307
                    $emsg = ' - S3 Error prevented quarantine';
-
 
1308
                    $statistics{'S3TransferError'}++;
-
 
1309
 
1197
                }
1310
                }
1198
                Log (sprintf("S3Quarantined:%s%s,%4.4s,%s%s", $testMsg, $entry->{reason}, $entry->{age}, $path, $s3msg));
1311
                Log (sprintf("S3Quarantined:%s%s,%4.4s,%s%s", $testMsg, $entry->{reason}, $entry->{age}, $path, $s3msg));
1199
            }
1312
            }
1200
 
1313
 
1201
            unless ($s3error)
1314
            unless ($s3error)
Line 1214... Line 1327...
1214
                {
1327
                {
1215
                    $rv = system ('mv', '-n', $path, $tdir);
1328
                    $rv = system ('mv', '-n', $path, $tdir);
1216
                    if ( $rv )
1329
                    if ( $rv )
1217
                    {
1330
                    {
1218
                        ReportError ("Move $path to $tdir");
1331
                        ReportError ("Move $path to $tdir");
-
 
1332
                        $statistics{'QuarantineError'}++;
-
 
1333
 
1219
                        #
1334
                        #
1220
                        # Clean up what may have been moved
1335
                        # Clean up what may have been moved
1221
                        # NOTE: deleted so that we don't loose stuff if it gets ugly
1336
                        # NOTE: deleted so that we don't loose stuff if it gets ugly
1222
        #                rmtree( $tpath);
1337
        #                rmtree( $tpath);
1223
        #                rmdir ($tdir);
1338
        #                rmdir ($tdir);
Line 1329... Line 1444...
1329
 Options:
1444
 Options:
1330
    -help              - brief help message
1445
    -help              - brief help message
1331
    -help -help        - Detailed help message
1446
    -help -help        - Detailed help message
1332
    -man               - Full documentation
1447
    -man               - Full documentation
1333
    -verbose[=n]       - Control output
1448
    -verbose[=n]       - Control output
-
 
1449
    -explain           - Display each package version disposition
1334
    -phase=nn          - Perform named phases
1450
    -phase=nn          - Perform named phases
1335
    -purge             - Just purge the old quarantined files
1451
    -purge             - Just purge the old quarantined files
1336
    -test              - Do not delete files
1452
    -test              - Do not delete files
1337
    -limit=n           - Limit packages processed. Test only
1453
    -limit=n           - Limit packages processed. Test only
1338
    -pcount=n          - Limit package count
1454
    -pcount=n          - Limit package count
Line 1361... Line 1477...
1361
 
1477
 
1362
A verbose level of 1. will display progress information
1478
A verbose level of 1. will display progress information
1363
 
1479
 
1364
A verbose level of 3. will display detailed tracing of all operations
1480
A verbose level of 3. will display detailed tracing of all operations
1365
 
1481
 
-
 
1482
=item B<-explain[=n]>
-
 
1483
 
-
 
1484
This option will output a line per package-version explaining the reason that
-
 
1485
packages are retained.
-
 
1486
 
-
 
1487
Only a level of 1 is supported.
-
 
1488
 
1366
=item B<-phase=list>
1489
=item B<-phase=list>
1367
 
1490
 
1368
This option will limit the work done by the program. There are two phases
1491
This option will limit the work done by the program. There are two phases
1369
called: 1 and 2.
1492
called: 1 and 2.
1370
 
1493