Subversion Repositories DevTools

Rev

Rev 5708 | Rev 6024 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 5708 Rev 5806
Line 37... Line 37...
37
my $opt_rtag_id;
37
my $opt_rtag_id;
38
my $opt_test = 0;
38
my $opt_test = 0;
39
my $opt_patch = 1;
39
my $opt_patch = 1;
40
my $opt_rootpkg;
40
my $opt_rootpkg;
41
my $opt_rootpkg_version;
41
my $opt_rootpkg_version;
-
 
42
my $opt_extract;
42
 
43
 
43
#
44
#
44
#   Data Base Interface
45
#   Data Base Interface
45
#
46
#
46
my $RM_DB;
47
my $RM_DB;
Line 64... Line 65...
64
my $sbom_branch;
65
my $sbom_branch;
65
my $sbom_project;
66
my $sbom_project;
66
my $sbom_version;
67
my $sbom_version;
67
my $rtag_release;
68
my $rtag_release;
68
my $rtag_project;
69
my $rtag_project;
-
 
70
my %config;
69
 
71
 
70
#
72
#
71
#   Packages to be ignored
73
#   Packages to be ignored
72
#
74
#
73
my %ignore;
75
my %ignore;
Line 93... Line 95...
93
                "rtagid|rtag_id=s"  => \$opt_rtag_id,           # string
95
                "rtagid|rtag_id=s"  => \$opt_rtag_id,           # string
94
                "rootpackage=s"     => \$opt_rootpkg,           # String
96
                "rootpackage=s"     => \$opt_rootpkg,           # String
95
                "ignore=s",         => sub{my ($a,$i) = @_; $ignore{$i} = 0 },
97
                "ignore=s",         => sub{my ($a,$i) = @_; $ignore{$i} = 0 },
96
                "test!"             => \$opt_test,              #[no]flag
98
                "test!"             => \$opt_test,              #[no]flag
97
                "patch!"            => \$opt_patch,             #[no]flag
99
                "patch!"            => \$opt_patch,             #[no]flag
-
 
100
                "extract=s"         => \$opt_extract,           # Name of file
98
                );
101
                );
99
 
102
 
100
#
103
#
101
#   Process help and manual options
104
#   Process help and manual options
102
#
105
#
103
pod2usage(-verbose => 0, -message => "Version: $VERSION")  if ($opt_help == 1  || ! $result);
106
pod2usage(-verbose => 0, -message => "Version: $VERSION")  if ($opt_help == 1  || ! $result);
104
pod2usage(-verbose => 1)  if ($opt_help == 2 );
107
pod2usage(-verbose => 1)  if ($opt_help == 2 );
105
pod2usage(-verbose => 2)  if ($opt_help > 2);
108
pod2usage(-verbose => 2)  if ($opt_help > 2);
106
 
109
 
107
ErrorConfig( 'name'    => 'ESCROW',
110
ErrorConfig( 'name'    => 'CHECK',
108
             'verbose' => $opt_verbose );
111
             'verbose' => $opt_verbose );
109
 
112
 
110
#
113
#
111
#   Sanity test
114
#   Sanity test
112
#
115
#
113
unless ( $opt_rtag_id || $opt_sbom_id || $#ARGV >= 1)
116
unless ( $opt_rtag_id || $opt_sbom_id || $opt_extract || $#ARGV >= 1)
114
{
117
{
115
    Error ("Need sbomid and/or rtagid",
118
    Error ("Need sbomid and/or rtagid",
116
           "Example: -sbomid=13543, for NZS Phase-1",
119
           "Example: -sbomid=13543, for NZS Phase-1",
117
           "Example: -sbomid=13543 -rtagid=xxxx, for NZS Phase-1, compared against given release",
120
           "Example: -sbomid=13543 -rtagid=xxxx, for NZS Phase-1, compared against given release",
118
           "Example: -rtagid=2362, for Sydney R1/R2",
121
           "Example: -rtagid=2362, for Sydney R1/R2",
119
           "Example: -rtagid=8843 -root=StockholmSBOM",
122
           "Example: -rtagid=8843 -root=StockholmSBOM",
120
           "Example: PackageName PackageVersion, for extracting a single package",
123
           "Example: PackageName PackageVersion, for extracting a single package",
121
    )
124
    )
122
}
125
}
123
 
126
 
124
 
-
 
125
Warning ("No sbomid provided. Output based an a Release") unless ( $opt_sbom_id );
-
 
126
$fpref = "release" unless ( $opt_sbom_id );
-
 
127
 
-
 
128
#
127
#
129
#   Import essential EnvVars
128
#   Import essential EnvVars
130
#
129
#
131
EnvImport('GBE_DPKG');
130
EnvImport('GBE_DPKG');
132
 
131
 
-
 
132
#
-
 
133
#   The extract option is special
-
 
134
#   It places the progam in a different mode
-
 
135
#
-
 
136
if ( $opt_extract )
-
 
137
{
-
 
138
    Error ("Cannot mix -extract with sbomid or rtagid" )
-
 
139
        if ( $opt_rtag_id || $opt_sbom_id || $#ARGV >= 0 );
-
 
140
 
-
 
141
    Error ("Cannot use -nopatch or -ignore with -extract")
-
 
142
        if ( ! $opt_patch || keys %ignore );
-
 
143
 
-
 
144
    Error ("S3 Recovery process must be run as buildadm")
-
 
145
        unless ($ENV{USER} eq 'buildadm');
-
 
146
 
-
 
147
    extract_files();
-
 
148
    exit (0);
-
 
149
}
-
 
150
 
-
 
151
 
-
 
152
Warning ("No sbomid provided. Output based an a Release") unless ( $opt_sbom_id );
-
 
153
$fpref = "release" unless ( $opt_sbom_id );
-
 
154
 
133
if ( $opt_sbom_id )
155
if ( $opt_sbom_id )
134
{
156
{
135
    #
157
    #
136
    #   Determines the OS_ID's for the bom
158
    #   Determines the OS_ID's for the bom
137
    #
159
    #
Line 239... Line 261...
239
#
261
#
240
#   Generate output files
262
#   Generate output files
241
#       1) Jats extract commands
263
#       1) Jats extract commands
242
#       2) Error list
264
#       2) Error list
243
my $file;
265
my $file;
244
$file = "${fpref}_extract.txt";
266
$file = "${fpref}_missing.txt";
245
push @create_list, $file;
267
push @create_list, $file;
246
open (JE, ">$file" ) || Error ("Cannot create $file");
268
open (JE, ">$file" ) || Error ("Cannot create $file");
247
 
269
 
248
$file = "${fpref}_status.txt";
270
#$file = "${fpref}_status.txt";
249
push @create_list, $file;
271
#push @create_list, $file;
250
 
272
#
251
open (ST, ">$file" ) || Error("Cannot create $file");
273
#open (ST, ">$file" ) || Error("Cannot create $file");
252
print ST "Cannot build:\n";
274
#print ST "Cannot build:\n";
253
 
275
 
254
my $missing_count = 0;
276
my $missing_count = 0;
255
my $pkg_count = 0;
277
my $pkg_count = 0;
256
foreach my $name ( sort keys %Package )
278
foreach my $name ( sort keys %Package )
257
{
279
{
Line 263... Line 285...
263
 
285
 
264
       if (! -d $pdir )
286
       if (! -d $pdir )
265
       {
287
       {
266
           Warning("Not in dpkg_archive: $name $ver");
288
           Warning("Not in dpkg_archive: $name $ver");
267
           $missing_count++;
289
           $missing_count++;
-
 
290
           print JE $name . '/' . $ver . "\n";
268
           next;
291
           next;
269
       }
292
       }
270
 
293
 
271
       if (! $tfile)
294
       if (! $tfile)
272
       {
295
       {
273
           Warning("Not in dpkg_archive: $name $ver - Bad Form");
296
           Warning("Not in dpkg_archive: $name $ver - Bad Form");
274
           $missing_count++;
297
           $missing_count++;
-
 
298
           print JE $name . '/' . $ver . "\n";
275
           next;
299
           next;
276
       }
300
       }
277
    }
301
    }
278
}
302
}
279
 
303
 
280
Message("Packages: $pkg_count");
304
Message("Packages: $pkg_count");
281
Message("Missing: $missing_count");
305
Message("Missing: $missing_count");
282
 
306
 
283
close (JE);
307
close (JE);
284
close (ST);
308
#close (ST);
285
 
309
 
286
 
310
 
287
#
311
#
288
#   Display names of files created
312
#   Display names of files created
289
#
313
#
Line 940... Line 964...
940
#print "Stray: $pv_id, $name, $ver\n";
964
#print "Stray: $pv_id, $name, $ver\n";
941
    }
965
    }
942
}
966
}
943
 
967
 
944
#-------------------------------------------------------------------------------
968
#-------------------------------------------------------------------------------
-
 
969
# Function        : extract_files
-
 
970
#
-
 
971
# Description     : Alternate mode of operation
-
 
972
#                   Extract files from the generated list. This is intended to
-
 
973
#                   be run as a seperate phase taking the 'extract' file
-
 
974
#
-
 
975
# Inputs          :
-
 
976
#
-
 
977
# Returns         : 
-
 
978
#
-
 
979
sub extract_files
-
 
980
{
-
 
981
    my @extract_order;
-
 
982
    my %extract;
-
 
983
    ErrorConfig( 'name'    => 'CHECK-EXTRACT' );
-
 
984
 
-
 
985
    #
-
 
986
    #   Open the file and read in data in one hit
-
 
987
    #   Each line is <PkgName>/<PkgVersion>
-
 
988
    #
-
 
989
    Error ("Cannot find specified file: $opt_extract")
-
 
990
        unless ( -f $opt_extract );
-
 
991
 
-
 
992
    #
-
 
993
    #   Sanity test
-
 
994
    #
-
 
995
    my $tfile = 'sbin/getPkgFromS3.sh';
-
 
996
    Error("Cannot find expected utility: $tfile") unless (-f $tfile);
-
 
997
 
-
 
998
    $tfile = 'sbin/jats_quarantine.cnf';
-
 
999
    Error("Cannot find expected config file: $tfile") unless (-f $tfile);
-
 
1000
    ReadConfig($tfile);
-
 
1001
 
-
 
1002
    open (FH, "<$opt_extract" ) || Error ("Cannot open file");
-
 
1003
    while ( <FH> )
-
 
1004
    {
-
 
1005
        s~[\r\n]+$~~;
-
 
1006
        Verbose2 ($_);
-
 
1007
        next unless ( $_ );
-
 
1008
 
-
 
1009
        push @extract_order, $_;
-
 
1010
    }
-
 
1011
    close FH;
-
 
1012
 
-
 
1013
    #
-
 
1014
    #   Log the file processing
-
 
1015
    #
-
 
1016
    my $lfile = "${opt_extract}.log";
-
 
1017
    Message ("Creating logfile: ${opt_extract}.log");
-
 
1018
    open (FH, ">$lfile" ) || Error ("Cannot open log file: $lfile");
-
 
1019
 
-
 
1020
    #
-
 
1021
    #   Process each entry
-
 
1022
    #
-
 
1023
    foreach my $pv ( @extract_order )
-
 
1024
    {
-
 
1025
        if ( $opt_test )
-
 
1026
        {
-
 
1027
            Verbose ("$pv");
-
 
1028
            print FH "$pv : TEST\n";
-
 
1029
        }
-
 
1030
        else
-
 
1031
        {
-
 
1032
            my $rv = System ('sbin/getPkgFromS3.sh', "--bucket=$config{S3Bucket}", '-p', $::GBE_DPKG . '/' . $pv);
-
 
1033
            print FH "$pv : SUCCESS\n" unless $rv;
-
 
1034
            print FH "$pv : ERROR\n" if $rv;
-
 
1035
        }
-
 
1036
    }
-
 
1037
    close FH;
-
 
1038
    Message ("Results in logfile: ${opt_extract}.log");
-
 
1039
}
-
 
1040
 
-
 
1041
 
-
 
1042
#-------------------------------------------------------------------------------
-
 
1043
# Function        : ReadConfig
-
 
1044
#
-
 
1045
# Description     : Read in quarantine config file
-
 
1046
#                   Only used during extraction processing
-
 
1047
#
-
 
1048
# Inputs          : $cfile   - Config file to read
-
 
1049
#
-
 
1050
# Returns         : 
-
 
1051
#
-
 
1052
 
-
 
1053
sub ReadConfig
-
 
1054
{
-
 
1055
    my ($cfile) = @_;
-
 
1056
    open (CF, '<', $cfile ) || Error ("Connot open: $cfile");
-
 
1057
    while ( <CF> )
-
 
1058
    {
-
 
1059
        s~\s+$~~;
-
 
1060
        s~^\s+~~;
-
 
1061
        next if ( m~\s*#~ );        # Comment
-
 
1062
        next unless $_;             # Empty
-
 
1063
        if ( m~(.*?)\s*=\s*(.*)~ ) {
-
 
1064
            $config{$1} = $2;
-
 
1065
        }
-
 
1066
    }
-
 
1067
    close CF;
-
 
1068
 
-
 
1069
    #
-
 
1070
    # Check that I've got what I need
-
 
1071
    # 
-
 
1072
    foreach ( qw(S3Bucket S3Key S3Secret))
-
 
1073
    {
-
 
1074
        ReportError("Config Item not found: $_") unless defined $config{$_};
-
 
1075
    }
-
 
1076
    ErrorDoExit();
-
 
1077
 
-
 
1078
    #
-
 
1079
    #   Export the Secrets in EnvVars
-
 
1080
    #   Use program defaults so that we don't need to specify them
-
 
1081
    #   on the command line - for all to see
-
 
1082
    #
-
 
1083
    $ENV{AWSKEY} = $config{S3Key};
-
 
1084
    $ENV{AWSSECRET} = $config{S3Secret};
-
 
1085
}
-
 
1086
 
-
 
1087
#-------------------------------------------------------------------------------
945
#   Documentation
1088
#   Documentation
946
#
1089
#
947
 
1090
 
948
=pod
1091
=pod
949
 
1092
 
Line 961... Line 1104...
961
    -man               - Full documentation
1104
    -man               - Full documentation
962
    -sbomid=xxx        - Specify the SBOM to process
1105
    -sbomid=xxx        - Specify the SBOM to process
963
    -rtagid=xxx        - Specify the Release to process (Optional)
1106
    -rtagid=xxx        - Specify the Release to process (Optional)
964
    -rootpackage=xxx   - Specifies a root package. In conjunction with -rtagid.
1107
    -rootpackage=xxx   - Specifies a root package. In conjunction with -rtagid.
965
    -ignore=name       - Ignore packages with the specified name
1108
    -ignore=name       - Ignore packages with the specified name
-
 
1109
    -extract=fname     - Extract files from a previous run
966
    -verbose           - Enable verbose output
1110
    -verbose           - Enable verbose output
967
    -[no]patch         - Ignore/Include patches. Default:Include
1111
    -[no]patch         - Ignore/Include patches. Default:Include
968
    -[no]test          - Reduced package scanning for test
1112
    -[no]test          - Reduced package scanning for test
969
 
1113
 
970
=head1 OPTIONS
1114
=head1 OPTIONS
Line 1012... Line 1156...
1012
=item B<-ignore=name>
1156
=item B<-ignore=name>
1013
 
1157
 
1014
All versions of the named package will be ignored. This parameter is options.
1158
All versions of the named package will be ignored. This parameter is options.
1015
It may be used multiple times.
1159
It may be used multiple times.
1016
 
1160
 
-
 
1161
=item B<-extract=name>
-
 
1162
 
-
 
1163
This option will process the 'extract' file created in a previous run of this
-
 
1164
program and restore missing package-versions as specified in the file.
-
 
1165
 
-
 
1166
The command will then create a log file recording packages that could ne be
-
 
1167
extracted.
-
 
1168
 
-
 
1169
This option does not not interwork with many of the other command line options.
-
 
1170
This option cannot be used in conjunction with the -rtagid, -sbomid, rootpackage
-
 
1171
and -nopatch.
-
 
1172
 
1017
=item B<-[no]patch>
1173
=item B<-[no]patch>
1018
 
1174
 
1019
This option is used ignore patches. If -nopatch is selected, then packages
1175
This option is used ignore patches. If -nopatch is selected, then packages
1020
versions that look like a patch will be added to the ignore list.
1176
versions that look like a patch will be added to the ignore list.
1021
 
1177