| 6133 |
dpurdie |
1 |
########################################################################
|
|
|
2 |
# Copyright (c) VIX TECHNOLOGY (AUST) LTD
|
|
|
3 |
#
|
|
|
4 |
# Module name : jats_generate_deployable.pl
|
|
|
5 |
# Module type : Makefile system
|
|
|
6 |
# Compiler(s) : Perl
|
|
|
7 |
# Environment(s): jats build system
|
|
|
8 |
#
|
|
|
9 |
# Description : Extracts current package version list from release manager
|
|
|
10 |
# based on the 'IS_DEPOLYABLE' flag in a given Release
|
|
|
11 |
# and copies resultant packages to release specific
|
|
|
12 |
# directory.
|
|
|
13 |
#
|
|
|
14 |
# Based on jats_update_release.pl and jats_gen_bom.pl but it is
|
|
|
15 |
# intended to be used by the PULSE digital distribution process
|
|
|
16 |
#......................................................................#
|
|
|
17 |
|
|
|
18 |
require 5.008_002;
|
|
|
19 |
use File::Basename;
|
|
|
20 |
use File::Copy;
|
|
|
21 |
use File::Path;
|
|
|
22 |
use strict;
|
|
|
23 |
use warnings;
|
|
|
24 |
use JatsEnv;
|
|
|
25 |
use JatsError;
|
|
|
26 |
use JatsRmApi;
|
|
|
27 |
use ArrayHashUtils;
|
|
|
28 |
use FileUtils;
|
|
|
29 |
use DBI;
|
|
|
30 |
use Getopt::Long;
|
|
|
31 |
use Pod::Usage; # required for help support
|
|
|
32 |
use JSON;
|
|
|
33 |
|
|
|
34 |
#
|
|
|
35 |
# Config Options
|
|
|
36 |
#
|
|
|
37 |
my $VERSION = "1.0.0"; # Update this
|
|
|
38 |
my $opt_help = 0;
|
|
|
39 |
my $opt_verbose = $ENV{'GBE_VERBOSE'}; # Allow global verbose
|
|
|
40 |
my $opt_rtagid;
|
|
|
41 |
my $opt_rootdir = '.';
|
|
|
42 |
my $opt_test;
|
|
|
43 |
my $opt_showFilters;
|
|
|
44 |
my @opt_addFilters;
|
|
|
45 |
my @opt_delFilters;
|
|
|
46 |
my $opt_showFiles;
|
|
|
47 |
my @opt_addFiles;
|
|
|
48 |
my @opt_delFiles;
|
|
|
49 |
|
|
|
50 |
|
|
|
51 |
#
|
|
|
52 |
# Constants
|
|
|
53 |
#
|
|
|
54 |
my $CONFFILE = '.bomGen';
|
|
|
55 |
my $MANIFEST = 'MANIFEST.json';
|
|
|
56 |
my $TFVARS = 'MANIFEST.tf';
|
|
|
57 |
|
|
|
58 |
#
|
|
|
59 |
# Globals
|
|
|
60 |
#
|
|
|
61 |
my $DM_DB; # Data Base Interface
|
|
|
62 |
my %bomList; # All files in the BOM
|
|
|
63 |
my $bomInfo; # Sbom meta data
|
|
|
64 |
|
|
|
65 |
#
|
|
|
66 |
# Configuration file vars
|
|
|
67 |
#
|
|
|
68 |
my @confFilters;
|
|
|
69 |
my @confFiles;
|
|
|
70 |
my %filtersUsed;
|
|
|
71 |
|
|
|
72 |
#-------------------------------------------------------------------------------
|
|
|
73 |
# Function : Main
|
|
|
74 |
#
|
|
|
75 |
# Description : Main entry point
|
|
|
76 |
# Parse user options
|
|
|
77 |
#
|
|
|
78 |
# Inputs :
|
|
|
79 |
#
|
|
|
80 |
# Returns :
|
|
|
81 |
#
|
|
|
82 |
|
|
|
83 |
my $result = GetOptions (
|
|
|
84 |
"help:+" => \$opt_help, # flag, multiple use allowed
|
|
|
85 |
"manual:3" => \$opt_help, # flag, multiple use allowed
|
|
|
86 |
"verbose:+" => \$opt_verbose, # flag
|
|
|
87 |
"rtagid|rtag_id=s" => \$opt_rtagid, # Number
|
|
|
88 |
"rootdir=s" => \$opt_rootdir, # string
|
|
|
89 |
|
|
|
90 |
"addfilter=s" => \@opt_addFilters, # multiple strings
|
|
|
91 |
"delfilter=s" => \@opt_delFilters, # multiple strings
|
|
|
92 |
"showfilters" => \$opt_showFilters, # flag
|
|
|
93 |
|
|
|
94 |
"addfiles=s" => \@opt_addFiles, # multiple strings
|
|
|
95 |
"delfiles=s" => \@opt_delFiles, # multiple strings
|
|
|
96 |
"showfiles" => \$opt_showFiles, # flag
|
|
|
97 |
|
|
|
98 |
"test" => \$opt_test, # flag
|
|
|
99 |
);
|
|
|
100 |
|
|
|
101 |
#
|
|
|
102 |
# Process help and manual options
|
|
|
103 |
#
|
|
|
104 |
pod2usage(-verbose => 0, -message => "Version: $VERSION") if ($opt_help == 1 || ! $result);
|
|
|
105 |
pod2usage(-verbose => 1) if ($opt_help == 2 );
|
|
|
106 |
pod2usage(-verbose => 2) if ($opt_help > 2);
|
|
|
107 |
|
|
|
108 |
ErrorConfig( 'name' => 'GenDeploy',
|
|
|
109 |
'verbose' => $opt_verbose );
|
|
|
110 |
|
|
|
111 |
#
|
|
|
112 |
# Sanity tests
|
|
|
113 |
#
|
|
|
114 |
|
|
|
115 |
# Supplied rootdir must exists as a directory
|
|
|
116 |
Error("Root dir not specified")
|
|
|
117 |
unless defined $opt_rootdir;
|
|
|
118 |
Error("Root dir not a valid directory: ", $opt_rootdir )
|
|
|
119 |
unless( -d $opt_rootdir );
|
|
|
120 |
|
|
|
121 |
# Environment var GBE_DPKG must exists as a directory
|
|
|
122 |
Error("GBE_DPKG Environment var is not a directory")
|
|
|
123 |
unless ( -d $ENV{GBE_DPKG} );
|
|
|
124 |
|
|
|
125 |
LoadFilterConfig();
|
|
|
126 |
ProcessFilterArgs();
|
|
|
127 |
|
|
|
128 |
# Non Filter operations
|
|
|
129 |
# Must supply an rtagid
|
|
|
130 |
Error("Need --rtagid", "Example: -rtagid=2362" )
|
|
|
131 |
unless ($opt_rtagid);
|
|
|
132 |
|
|
|
133 |
#
|
|
|
134 |
# This command is destined to be used in a directory where group permissions
|
|
|
135 |
# are important. Ensure that the user is not killing group access
|
|
|
136 |
#
|
|
|
137 |
umask 0002;
|
|
|
138 |
|
|
|
139 |
#
|
|
|
140 |
# Body of the processing
|
|
|
141 |
# Save generation time into the meta data
|
|
|
142 |
my $now = time;
|
|
|
143 |
$bomInfo->{version} = "2.0.0";
|
|
|
144 |
$bomInfo->{timestamp}{epoch} = $now;
|
|
|
145 |
$bomInfo->{timestamp}{utc} = gmtime($now);
|
|
|
146 |
|
|
|
147 |
Message("Copying packages from $ENV{GBE_DPKG} to $opt_rootdir");
|
|
|
148 |
|
|
|
149 |
#
|
|
|
150 |
# Processing
|
|
|
151 |
#
|
|
|
152 |
connectRM(\$DM_DB);
|
|
|
153 |
GetReleaseInfo(); # Get Release Metadata
|
|
|
154 |
GetPackageData(); # Get RM Data
|
|
|
155 |
RemoveDuplicates(); # Need P or D, but not both
|
|
|
156 |
CopyInNew(); # Copy new files
|
|
|
157 |
RemoveExcess(); # Remove files no longer required
|
|
|
158 |
GenFileData(); # Generate file metadata
|
|
|
159 |
WriteManifest(); # Save out meta data
|
|
|
160 |
exit 0;
|
|
|
161 |
|
|
|
162 |
#-------------------------------------------------------------------------------
|
|
|
163 |
# Function : GenFileData
|
|
|
164 |
#
|
|
|
165 |
# Description : Generate meta data on each file
|
|
|
166 |
# Much of this is a guess.
|
|
|
167 |
# Assume files look like:
|
|
|
168 |
#
|
|
|
169 |
# VIXcryptoKeyManager-1.0.2061.cr-WIN32.exe
|
|
|
170 |
# erg-pkgmnt_1.0.3010.cr_UBUNTU16_P.deb
|
|
|
171 |
# erg-pkgmnt_1.0.3010.cr_RHEL7_P.rpm
|
|
|
172 |
# xxxxxx.sh - bit trickier
|
|
|
173 |
#
|
|
|
174 |
# Inputs : None
|
|
|
175 |
#
|
|
|
176 |
# Returns : Populates $bomInfo
|
|
|
177 |
#
|
|
|
178 |
sub GenFileData
|
|
|
179 |
{
|
|
|
180 |
my @elist;
|
|
|
181 |
my @edup;
|
|
|
182 |
foreach my $file (sort keys %bomList)
|
|
|
183 |
{
|
|
|
184 |
my $data;
|
|
|
185 |
my $alias;
|
|
|
186 |
|
|
|
187 |
$bomList{$file}{version} =~ m~(.*)\.([a-z]+)$~;
|
|
|
188 |
my $pvfull = $1;
|
|
|
189 |
my $proj = $2;
|
|
|
190 |
my $pv = $pvfull;
|
|
|
191 |
$pv =~ s~\.\d+$~~;
|
|
|
192 |
|
|
|
193 |
if ($file =~ m~^(.*)-(.*)\.(.*)-(WIN.*)\.(exe)$~i)
|
|
|
194 |
{
|
|
|
195 |
$data->{name} = $1;
|
|
|
196 |
$data->{version} = $2;
|
|
|
197 |
$data->{prj} = $3;
|
|
|
198 |
$data->{arch} = $4;
|
|
|
199 |
$data->{type} = $5;
|
|
|
200 |
}
|
|
|
201 |
elsif ( $file =~ m~^(.*)_(.*)\.([^_]+)_(.*)\.(deb|tgz|rpm)$~i)
|
|
|
202 |
{
|
|
|
203 |
$data->{name} = $1;
|
|
|
204 |
$data->{version} = $2;
|
|
|
205 |
$data->{prj} = $3;
|
|
|
206 |
$data->{arch} = $4;
|
|
|
207 |
$data->{type} = $5;
|
|
|
208 |
$data->{arch} =~ s~_[PD]~~;
|
|
|
209 |
}
|
|
|
210 |
elsif ( $file =~ m~^(.*)-($pv)\.(.*)\.(rpm)$~i)
|
|
|
211 |
{
|
|
|
212 |
# COTS package
|
|
|
213 |
$data->{name} = $1;
|
|
|
214 |
$data->{version} = $2;
|
|
|
215 |
$data->{arch} = $3;
|
|
|
216 |
$data->{type} = $4;
|
|
|
217 |
$data->{prj} = $proj;
|
|
|
218 |
}
|
|
|
219 |
elsif ( $file =~ m~^(.*)_($pv)\.(tgz)$~i)
|
|
|
220 |
{
|
|
|
221 |
# COTS package
|
|
|
222 |
$data->{name} = $1;
|
|
|
223 |
$data->{version} = $2;
|
|
|
224 |
$data->{arch} = 'UNKNOWN';
|
|
|
225 |
$data->{type} = $3;
|
|
|
226 |
$data->{prj} = $proj;
|
|
|
227 |
}
|
|
|
228 |
elsif ( $file =~ m~^(.*)-($pv)\.(.*)\.(deb)$~i)
|
|
|
229 |
{
|
|
|
230 |
# COTS package
|
|
|
231 |
$data->{name} = $1;
|
|
|
232 |
$data->{version} = $2;
|
|
|
233 |
$data->{arch} = $3;
|
|
|
234 |
$data->{type} = $4;
|
|
|
235 |
$data->{prj} = $proj;
|
|
|
236 |
}
|
|
|
237 |
elsif ( $file =~ m~^(.*)\.(sh|zip|msi|tar\.gz)$~i)
|
|
|
238 |
{
|
|
|
239 |
$data->{name} = $1;
|
|
|
240 |
$data->{arch} = 'NOARCH';
|
|
|
241 |
$data->{type} = $2;
|
|
|
242 |
|
|
|
243 |
$data->{version} = $pvfull;
|
|
|
244 |
$data->{prj} = $proj;
|
|
|
245 |
}
|
|
|
246 |
|
|
|
247 |
unless ($data && $data->{name} && $data->{prj} && $data->{type}) {
|
|
|
248 |
push @elist, $file;
|
|
|
249 |
next;
|
|
|
250 |
}
|
|
|
251 |
$data->{fullname} = $file;
|
|
|
252 |
|
|
|
253 |
#
|
|
|
254 |
# Create a nice alias
|
|
|
255 |
# ERG -> VIX (not done)
|
|
|
256 |
# All lowercase
|
|
|
257 |
#
|
|
|
258 |
$alias = join ('.', $data->{name}, $data->{prj}, $data->{type});
|
|
|
259 |
$alias = lc ($alias);
|
|
|
260 |
#$alias =~ s~^erg~vix~;
|
|
|
261 |
#$alias =~ s~^vix~vix-~;
|
|
|
262 |
#$alias =~ s~^vix--~vix-~;
|
|
|
263 |
push (@edup, join( ' : ', $alias, $file ,$bomInfo->{files}{$alias}{fullname}) ) if exists $bomInfo->{files}{$alias};
|
|
|
264 |
|
|
|
265 |
delete $data->{type};
|
|
|
266 |
$bomInfo->{files}{$alias} = $data;
|
|
|
267 |
}
|
|
|
268 |
|
|
|
269 |
ReportError ("Cannot extract file metadata from:", @elist) if (@elist);
|
|
|
270 |
ReportError ("Duplicate aliases for:", @edup) if (@edup);
|
|
|
271 |
ErrorDoExit();
|
|
|
272 |
}
|
|
|
273 |
|
|
|
274 |
#-------------------------------------------------------------------------------
|
|
|
275 |
# Function : CopyInNew
|
|
|
276 |
#
|
|
|
277 |
# Description : Copy in new files
|
|
|
278 |
# Don't copy in files that already exist - assume that the
|
|
|
279 |
# files don't chnage without a chnage to the file name
|
|
|
280 |
#
|
|
|
281 |
# Inputs :
|
|
|
282 |
#
|
|
|
283 |
# Returns :
|
|
|
284 |
#
|
|
|
285 |
sub CopyInNew
|
|
|
286 |
{
|
|
|
287 |
#
|
|
|
288 |
# Ensure the output directory exists
|
|
|
289 |
#
|
|
|
290 |
if ( ! -d $opt_rootdir )
|
|
|
291 |
{
|
|
|
292 |
if ( defined($opt_test) )
|
|
|
293 |
{
|
|
|
294 |
Message("mkdir $opt_rootdir");
|
|
|
295 |
}
|
|
|
296 |
else
|
|
|
297 |
{
|
|
|
298 |
eval { mkpath($opt_rootdir) };
|
|
|
299 |
Error("Failed to make project directory tree $opt_rootdir") if ( $@ || ! -d $opt_rootdir );
|
|
|
300 |
}
|
|
|
301 |
}
|
|
|
302 |
|
|
|
303 |
#
|
|
|
304 |
# Determine the files to be transferred
|
|
|
305 |
#
|
|
|
306 |
my @filelist;
|
|
|
307 |
foreach my $file ( keys %bomList)
|
|
|
308 |
{
|
|
|
309 |
push (@filelist, $file) unless ( -f "$opt_rootdir/$file" );
|
|
|
310 |
}
|
|
|
311 |
|
|
|
312 |
#
|
|
|
313 |
# Perform the actual copy
|
|
|
314 |
#
|
|
|
315 |
if ( @filelist )
|
|
|
316 |
{
|
|
|
317 |
#Message("Copying files for package $PKG_NAME version $PKG_VERSION");
|
|
|
318 |
if ( defined($opt_test) )
|
|
|
319 |
{
|
|
|
320 |
Message( map("$_...", @filelist) );
|
|
|
321 |
}
|
|
|
322 |
else
|
|
|
323 |
{
|
|
|
324 |
eval { mkpath($opt_rootdir) };
|
|
|
325 |
Error("Failed to make destination directory") if ( $@ || ! -d $opt_rootdir );
|
|
|
326 |
foreach my $file ( @filelist )
|
|
|
327 |
{
|
|
|
328 |
Verbose("Copy: $file...");
|
|
|
329 |
|
|
|
330 |
my $srcFile = $bomList{$file}{path};
|
|
|
331 |
if ( ! copy($srcFile, $opt_rootdir) )
|
|
|
332 |
{
|
|
|
333 |
Warning("Failed to copy $file ($!)");
|
|
|
334 |
}
|
|
|
335 |
}
|
|
|
336 |
}
|
|
|
337 |
}
|
|
|
338 |
}
|
|
|
339 |
|
|
|
340 |
#-------------------------------------------------------------------------------
|
|
|
341 |
# Function : RemoveExcess
|
|
|
342 |
#
|
|
|
343 |
# Description : Remove excess files from the output directory
|
|
|
344 |
#
|
|
|
345 |
# Inputs :
|
|
|
346 |
#
|
|
|
347 |
# Returns :
|
|
|
348 |
#
|
|
|
349 |
sub RemoveExcess
|
|
|
350 |
{
|
|
|
351 |
my @filelist;
|
|
|
352 |
my %keepList = map { $_ => 1 } @confFiles;
|
|
|
353 |
|
|
|
354 |
#
|
|
|
355 |
# Find all files in the output directory
|
|
|
356 |
# Use the 'keepList' so that we don't pickup files that should
|
|
|
357 |
# be in the directory. README.md, MANIFEST ...
|
|
|
358 |
#
|
|
|
359 |
foreach my $srcPath ( glob("$opt_rootdir/*") )
|
|
|
360 |
{
|
|
|
361 |
my $dstFile = basename($srcPath);
|
|
|
362 |
next if exists $keepList{$dstFile};
|
|
|
363 |
next unless ( -f $srcPath );
|
|
|
364 |
|
|
|
365 |
push (@filelist, $dstFile) unless (exists $bomList{$dstFile} );
|
|
|
366 |
}
|
|
|
367 |
|
|
|
368 |
if ( @filelist)
|
|
|
369 |
{
|
|
|
370 |
Message ("Delete execess files", @filelist );
|
|
|
371 |
unless ( defined($opt_test) )
|
|
|
372 |
{
|
|
|
373 |
foreach my $file ( @filelist )
|
|
|
374 |
{
|
|
|
375 |
Verbose("Delete: $file...");
|
|
|
376 |
if ( unlink("$opt_rootdir/$file") ne 1 )
|
|
|
377 |
{
|
|
|
378 |
Warning("Failed to delete: $file. ($!)");
|
|
|
379 |
}
|
|
|
380 |
}
|
|
|
381 |
}
|
|
|
382 |
}
|
|
|
383 |
}
|
|
|
384 |
|
|
|
385 |
#-------------------------------------------------------------------------------
|
|
|
386 |
# Function : RemoveDuplicates
|
|
|
387 |
#
|
|
|
388 |
# Description : Scan the BOM file list and remove duplicate installers
|
|
|
389 |
# Duplicate installers are that that have both a P and a D
|
|
|
390 |
# flavor of the installer
|
|
|
391 |
#
|
|
|
392 |
# This test has some nasty built-in knowledge (assumtions)
|
|
|
393 |
# It assumes that:
|
|
|
394 |
# Windows installers are only created for one flavor
|
|
|
395 |
# Don't need to worry about windoes installers
|
|
|
396 |
# Non windows installers are of the form:
|
|
|
397 |
# Name_Architecture_Type.deb
|
|
|
398 |
#
|
|
|
399 |
# Inputs :
|
|
|
400 |
#
|
|
|
401 |
# Returns :
|
|
|
402 |
#
|
|
|
403 |
sub RemoveDuplicates
|
|
|
404 |
{
|
|
|
405 |
my %baseNames;
|
|
|
406 |
foreach my $file ( keys %bomList)
|
|
|
407 |
{
|
|
|
408 |
#
|
|
|
409 |
# Only process files that are of the expected form
|
|
|
410 |
# ie: erg-udcrypt_1.0.3043.vss_UBUNTU16_P.deb
|
|
|
411 |
#
|
|
|
412 |
if( $file =~ m~(.*)_([PD])(\.(deb|rpm|tgz))$~ )
|
|
|
413 |
{
|
|
|
414 |
my $base=$1;
|
|
|
415 |
my $type=$2;
|
|
|
416 |
my $suf=$3;
|
|
|
417 |
|
|
|
418 |
if (exists $baseNames{$base} )
|
|
|
419 |
{
|
|
|
420 |
my $debugName = $base . '_D' . $suf;
|
|
|
421 |
Verbose("Remove debug installer: $file. Kill: $debugName");
|
|
|
422 |
delete $bomList{$debugName};
|
|
|
423 |
}
|
|
|
424 |
|
|
|
425 |
$baseNames{$base} = $type;
|
|
|
426 |
}
|
|
|
427 |
}
|
|
|
428 |
}
|
|
|
429 |
|
|
|
430 |
#-------------------------------------------------------------------------------
|
|
|
431 |
# Function : LoadFilterConfig
|
|
|
432 |
#
|
|
|
433 |
# Description : Load Filter Config
|
|
|
434 |
# Retain filter config for future reference
|
|
|
435 |
#
|
|
|
436 |
# Inputs :
|
|
|
437 |
#
|
|
|
438 |
# Returns :
|
|
|
439 |
#
|
|
|
440 |
sub LoadFilterConfig
|
|
|
441 |
{
|
|
|
442 |
if ( -f "$opt_rootdir/$CONFFILE" )
|
|
|
443 |
{
|
|
|
444 |
Message("Loading Config File");
|
|
|
445 |
|
|
|
446 |
local $/;
|
|
|
447 |
open(my $fh, "<$opt_rootdir/$CONFFILE") || Error("Failed to open config file");
|
|
|
448 |
my $json_text = <$fh>;
|
|
|
449 |
my $perl_scalar = decode_json( $json_text );
|
|
|
450 |
Error ("Invalid format in Config file")
|
|
|
451 |
unless (ref($perl_scalar->{filters}) eq 'ARRAY');
|
|
|
452 |
|
|
|
453 |
push (@confFilters, @{$perl_scalar->{filters}});
|
|
|
454 |
push (@confFiles, @{$perl_scalar->{keptfiles}}) if exists ($perl_scalar->{keptfiles});
|
|
|
455 |
close($fh);
|
|
|
456 |
}
|
|
|
457 |
}
|
|
|
458 |
|
|
|
459 |
#-------------------------------------------------------------------------------
|
|
|
460 |
# Function : ProcessFilterArgs
|
|
|
461 |
#
|
|
|
462 |
# Description : Process the filter based arguments
|
|
|
463 |
#
|
|
|
464 |
# Inputs :
|
|
|
465 |
#
|
|
|
466 |
# Returns :
|
|
|
467 |
#
|
|
|
468 |
sub ProcessFilterArgs
|
|
|
469 |
{
|
|
|
470 |
my $filterArgSeen;
|
|
|
471 |
my $writeConf;
|
|
|
472 |
|
|
|
473 |
|
|
|
474 |
unless ( @confFilters )
|
|
|
475 |
{
|
|
|
476 |
Error("No Filters defined.", "Add filters before creating BOM");
|
|
|
477 |
}
|
|
|
478 |
|
|
|
479 |
if ( @opt_addFilters )
|
|
|
480 |
{
|
|
|
481 |
Message ("Adding command line filters to the release config file");
|
|
|
482 |
foreach my $element (@opt_addFilters) {
|
|
|
483 |
UniquePush (\@confFilters, $_ ) foreach ( split(/,/, $element));
|
|
|
484 |
}
|
|
|
485 |
$writeConf = 1;
|
|
|
486 |
}
|
|
|
487 |
|
|
|
488 |
if ( @opt_delFilters )
|
|
|
489 |
{
|
|
|
490 |
Message ("Deleting command line filters to the release config file");
|
|
|
491 |
foreach my $element (@opt_delFilters) {
|
|
|
492 |
ArrayDelete (\@confFilters, $_ ) foreach ( split(/,/, $element));
|
|
|
493 |
}
|
|
|
494 |
$writeConf = 1;
|
|
|
495 |
}
|
|
|
496 |
|
|
|
497 |
if ( @opt_addFiles )
|
|
|
498 |
{
|
|
|
499 |
Message ("Adding command line files to the release config file");
|
|
|
500 |
foreach my $element (@opt_addFiles) {
|
|
|
501 |
UniquePush (\@confFiles, $_ ) foreach ( split(/,/, $element));
|
|
|
502 |
}
|
|
|
503 |
$writeConf = 1;
|
|
|
504 |
}
|
|
|
505 |
|
|
|
506 |
if ( @opt_delFiles )
|
|
|
507 |
{
|
|
|
508 |
Message ("Deleting command line files to the release config file");
|
|
|
509 |
foreach my $element (@opt_delFiles) {
|
|
|
510 |
ArrayDelete (\@confFiles, $_ ) foreach ( split(/,/, $element));
|
|
|
511 |
}
|
|
|
512 |
$writeConf = 1;
|
|
|
513 |
}
|
|
|
514 |
|
|
|
515 |
#
|
|
|
516 |
# Save filter information
|
|
|
517 |
#
|
|
|
518 |
if ( $writeConf && ! defined($opt_test) )
|
|
|
519 |
{
|
|
|
520 |
Verbose ("Write config file");
|
|
|
521 |
|
|
|
522 |
#
|
|
|
523 |
# Add known files
|
|
|
524 |
#
|
|
|
525 |
UniquePush (\@confFiles, $CONFFILE, $MANIFEST, $TFVARS);
|
|
|
526 |
|
|
|
527 |
my $config;
|
|
|
528 |
push @{$config->{filters}},@confFilters;
|
|
|
529 |
push @{$config->{keptfiles}},@confFiles;
|
|
|
530 |
FileCreate ("$opt_rootdir/$CONFFILE", to_json( $config, { ascii => 1, pretty => 1 }));
|
|
|
531 |
}
|
|
|
532 |
|
|
|
533 |
#
|
|
|
534 |
# Display information to the user
|
|
|
535 |
#
|
|
|
536 |
if ($opt_showFilters)
|
|
|
537 |
{
|
|
|
538 |
Message ("Configured Filters",@confFilters );
|
|
|
539 |
$filterArgSeen = 1;
|
|
|
540 |
}
|
|
|
541 |
|
|
|
542 |
if ($opt_showFiles)
|
|
|
543 |
{
|
|
|
544 |
Message ("Configured Files. Keep:",@confFiles );
|
|
|
545 |
$filterArgSeen = 1;
|
|
|
546 |
}
|
|
|
547 |
|
|
|
548 |
|
|
|
549 |
#
|
|
|
550 |
# Terminate program on any filter operations
|
|
|
551 |
#
|
|
|
552 |
exit 0 if ( $writeConf || $filterArgSeen);
|
|
|
553 |
}
|
|
|
554 |
|
|
|
555 |
#-------------------------------------------------------------------------------
|
|
|
556 |
# Function : WriteManifest
|
|
|
557 |
#
|
|
|
558 |
# Description : Save the filter config file if required
|
|
|
559 |
#
|
|
|
560 |
# Inputs :
|
|
|
561 |
#
|
|
|
562 |
# Returns :
|
|
|
563 |
#
|
|
|
564 |
sub WriteManifest
|
|
|
565 |
{
|
|
|
566 |
return if defined($opt_test);
|
|
|
567 |
|
|
|
568 |
#
|
|
|
569 |
# Create JSON metadata
|
|
|
570 |
#
|
|
|
571 |
Verbose ("Write JSON Manifest");
|
|
|
572 |
my $jsonString = to_json( $bomInfo, { ascii => 1, pretty => 1, canonical => 1 } );
|
|
|
573 |
FileCreate ($opt_rootdir . '/' . $MANIFEST, $jsonString);
|
|
|
574 |
|
|
|
575 |
#
|
|
|
576 |
# Create Terraform data
|
|
|
577 |
# Note: Terraform variable cannot have a '.' in them
|
|
|
578 |
#
|
|
|
579 |
my @tfData2;
|
|
|
580 |
|
|
|
581 |
push @tfData2, "// Terraform variable definitions to map clean package name to full file name";
|
|
|
582 |
push @tfData2, "variable vixFileName {";
|
|
|
583 |
push @tfData2, " type = \"map\"";
|
|
|
584 |
push @tfData2, " default = {" ;
|
|
|
585 |
|
|
|
586 |
foreach my $item ( sort keys $bomInfo->{files} )
|
|
|
587 |
{
|
|
|
588 |
push @tfData2, " \"". $item ."\" = \"" .$bomInfo->{files}{$item}{fullname} ."\"";
|
|
|
589 |
}
|
|
|
590 |
|
|
|
591 |
push @tfData2, " }" ;
|
|
|
592 |
push @tfData2, "}" ;
|
|
|
593 |
|
|
|
594 |
FileCreate ($opt_rootdir . '/' . $TFVARS, @tfData2);
|
|
|
595 |
|
|
|
596 |
|
|
|
597 |
}
|
|
|
598 |
|
|
|
599 |
#-------------------------------------------------------------------------------
|
|
|
600 |
# Function : GetReleaseInfo
|
|
|
601 |
#
|
|
|
602 |
# Description : Get Release Meta Data
|
|
|
603 |
#
|
|
|
604 |
# Inputs :
|
|
|
605 |
#
|
|
|
606 |
# Returns : Will exit on error
|
|
|
607 |
#
|
|
|
608 |
sub GetReleaseInfo
|
|
|
609 |
{
|
|
|
610 |
my $m_sqlstr = "SELECT p.PROJ_ID, rt.rtag_id, p.PROJ_NAME, rt.RTAG_NAME" .
|
|
|
611 |
" FROM release_tags rt, PROJECTS p" .
|
|
|
612 |
" WHERE p.PROJ_ID = rt.PROJ_ID" .
|
|
|
613 |
" and rt.RTAG_ID = " . $opt_rtagid;
|
|
|
614 |
|
|
|
615 |
my $sth = $DM_DB->prepare($m_sqlstr);
|
|
|
616 |
if ( defined($sth) )
|
|
|
617 |
{
|
|
|
618 |
if ( $sth->execute( ) )
|
|
|
619 |
{
|
|
|
620 |
if ( $sth->rows )
|
|
|
621 |
{
|
|
|
622 |
while ( my ( $proj_id, $xx, $pname, $rname ) = $sth->fetchrow_array )
|
|
|
623 |
{
|
|
|
624 |
my $data;
|
|
|
625 |
$data->{product_id} = $proj_id;
|
|
|
626 |
$data->{product_name} = $pname;
|
|
|
627 |
$data->{release_rtagid} = $opt_rtagid;
|
|
|
628 |
$data->{release_name} = $rname;
|
|
|
629 |
push @{$bomInfo->{release}}, $data;
|
|
|
630 |
}
|
|
|
631 |
}
|
|
|
632 |
else
|
|
|
633 |
{
|
|
|
634 |
Error("GetReleaseInfo: No rtagid found for " . $opt_rtagid);
|
|
|
635 |
}
|
|
|
636 |
$sth->finish();
|
|
|
637 |
}
|
|
|
638 |
else
|
|
|
639 |
{
|
|
|
640 |
Error("GetReleaseInfo: Execute failure", $sth->errstr(), $m_sqlstr );
|
|
|
641 |
}
|
|
|
642 |
}
|
|
|
643 |
else
|
|
|
644 |
{
|
|
|
645 |
Error("GetReleaseInfo: Prepare failure", $sth->errstr(), $m_sqlstr );
|
|
|
646 |
}
|
|
|
647 |
}
|
|
|
648 |
|
|
|
649 |
#-------------------------------------------------------------------------------
|
|
|
650 |
# Function : GetPackageData
|
|
|
651 |
#
|
|
|
652 |
# Description : Extract data from RM based on the provided rtag_id
|
|
|
653 |
#
|
|
|
654 |
# Inputs :
|
|
|
655 |
#
|
|
|
656 |
# Returns :
|
|
|
657 |
#
|
|
|
658 |
sub GetPackageData
|
|
|
659 |
{
|
|
|
660 |
my $m_sqlstr = "SELECT p.PKG_NAME, pv.PKG_VERSION" .
|
|
|
661 |
" FROM package_versions pv, RELEASE_CONTENT rc, PACKAGES p" .
|
|
|
662 |
" WHERE rc.rtag_id = " . $opt_rtagid .
|
|
|
663 |
" AND rc.pv_id = pv.pv_id" .
|
|
|
664 |
" and p.PKG_ID = pv.pkg_id" .
|
|
|
665 |
" and pv.IS_DEPLOYABLE = 'Y'";
|
|
|
666 |
# " and ( pv.IS_DEPLOYABLE = 'Y' or upper( p.PKG_NAME) like 'ERG%' or upper( p.PKG_NAME) like 'VIX%' )";
|
|
|
667 |
|
|
|
668 |
my ( $PKG_NAME, $PKG_VERSION );
|
|
|
669 |
my $sth = $DM_DB->prepare($m_sqlstr);
|
|
|
670 |
if ( defined($sth) )
|
|
|
671 |
{
|
|
|
672 |
if ( $sth->execute( ) )
|
|
|
673 |
{
|
|
|
674 |
if ( $sth->rows )
|
|
|
675 |
{
|
|
|
676 |
while ( ( $PKG_NAME, $PKG_VERSION ) = $sth->fetchrow_array )
|
|
|
677 |
{
|
|
|
678 |
Verbose ("Deployable: $PKG_NAME, $PKG_VERSION");
|
|
|
679 |
my $pkgDir = "$ENV{GBE_DPKG}/$PKG_NAME";
|
|
|
680 |
my $srcDir = "$ENV{GBE_DPKG}/$PKG_NAME/$PKG_VERSION";
|
|
|
681 |
my $dstDir = $opt_rootdir;
|
|
|
682 |
|
|
|
683 |
if ( -d "$srcDir" )
|
|
|
684 |
{
|
|
|
685 |
my $foundFiltered = 0;
|
|
|
686 |
|
|
|
687 |
# for each of the filter rules we glob the rule in the src pkg/version dir
|
|
|
688 |
# and if any of the globbed files dont exist in the dst dir add it to the
|
|
|
689 |
# the filelist array of files to copy
|
|
|
690 |
foreach my $filter ( @confFilters )
|
|
|
691 |
{
|
|
|
692 |
foreach my $srcPath ( glob("$srcDir/$filter") )
|
|
|
693 |
{
|
|
|
694 |
next unless ( -f $srcPath );
|
|
|
695 |
$foundFiltered = 1;
|
|
|
696 |
$filtersUsed{$filter} = 1;
|
|
|
697 |
my $dstFile = basename($srcPath);
|
|
|
698 |
my $srcFile = $srcPath;
|
|
|
699 |
$srcFile =~ s~^$srcDir/~~;
|
|
|
700 |
$bomList{$srcFile}{path} = $srcPath;
|
|
|
701 |
$bomList{$srcFile}{package} = $PKG_NAME;
|
|
|
702 |
$bomList{$srcFile}{version} = $PKG_VERSION;
|
|
|
703 |
}
|
|
|
704 |
}
|
|
|
705 |
|
|
|
706 |
# if no files found using filters then issue warning
|
|
|
707 |
Warning("No Files found for Package Version $PKG_NAME/$PKG_VERSION using supplied filters")
|
|
|
708 |
unless ( $foundFiltered );
|
|
|
709 |
|
|
|
710 |
if ($foundFiltered)
|
|
|
711 |
{
|
|
|
712 |
$bomInfo->{packages}{$PKG_NAME} = $PKG_VERSION;
|
|
|
713 |
}
|
|
|
714 |
}
|
|
|
715 |
elsif ( ! -d "$pkgDir" )
|
|
|
716 |
{
|
|
|
717 |
# if srcDir and pkgDir dont exist then package is not in dpkg_archive so display message
|
|
|
718 |
Warning("Skipping Package $PKG_NAME/$PKG_VERSION as it does not exist in dpkg_archive");
|
|
|
719 |
}
|
|
|
720 |
else
|
|
|
721 |
{
|
|
|
722 |
# However if srcDir does not exist but pkgDir does exist then the package version is missing which maybe an issue
|
|
|
723 |
Warning("Missing Version $PKG_VERSION for Package $PKG_NAME in dpkg_archive");
|
|
|
724 |
}
|
|
|
725 |
}
|
|
|
726 |
|
|
|
727 |
#
|
|
|
728 |
# Report filter elements that where not used.
|
|
|
729 |
#
|
|
|
730 |
my @notUsed;
|
|
|
731 |
foreach my $filter ( @confFilters )
|
|
|
732 |
{
|
|
|
733 |
next if ( exists $filtersUsed{$filter} );
|
|
|
734 |
push @notUsed, $filter
|
|
|
735 |
}
|
|
|
736 |
Warning ("Unused filter rules:", @notUsed )
|
|
|
737 |
if ( @notUsed );
|
|
|
738 |
|
|
|
739 |
}
|
|
|
740 |
else
|
|
|
741 |
{
|
|
|
742 |
Error("No Packages for rtagid: $opt_rtagid");
|
|
|
743 |
}
|
|
|
744 |
$sth->finish();
|
|
|
745 |
}
|
|
|
746 |
else
|
|
|
747 |
{
|
|
|
748 |
Error("Execute failure", $sth->errstr(), $m_sqlstr );
|
|
|
749 |
}
|
|
|
750 |
}
|
|
|
751 |
else
|
|
|
752 |
{
|
|
|
753 |
Error("Prepare failure", $sth->errstr(), $m_sqlstr );
|
|
|
754 |
}
|
|
|
755 |
}
|
|
|
756 |
|
|
|
757 |
#-------------------------------------------------------------------------------
|
|
|
758 |
# Documentation
|
|
|
759 |
#
|
|
|
760 |
|
|
|
761 |
=pod
|
|
|
762 |
|
|
|
763 |
=for htmltoc DEPLOY::generate_deployable
|
|
|
764 |
|
|
|
765 |
=head1 NAME
|
|
|
766 |
|
|
|
767 |
jats_generate_deployable - Extracts current package version list from Release Manager RtagId
|
|
|
768 |
and copy resultant packages to a specific directory.
|
|
|
769 |
|
|
|
770 |
=head1 SYNOPSIS
|
|
|
771 |
|
|
|
772 |
jats generate_deployable [options]
|
|
|
773 |
|
|
|
774 |
Options:
|
|
|
775 |
-help - Brief help message
|
|
|
776 |
-help -help - Detailed help message
|
|
|
777 |
-man - Full documentation
|
|
|
778 |
-rtagid=xxx - Specify the Release Manager RtagId to process
|
|
|
779 |
-rootdir=xxx - Specifies the root of the releases directory
|
|
|
780 |
|
|
|
781 |
-showfilters - Display current filter set and exit
|
|
|
782 |
-addfilter=xxx[,yyy] - Add a new filter to the existing filter set
|
|
|
783 |
-delfilter=xxx[,yyy] - Delete a filter from the existing filter set
|
|
|
784 |
|
|
|
785 |
-showfiles - Display current kept file set and exit
|
|
|
786 |
-addfiles=xxx[,yyy] - Add a new file to the kept file set
|
|
|
787 |
-delfiles=xxx[,yyy] - Delete a file from the kept file set
|
|
|
788 |
|
|
|
789 |
-test - Just log actions without copying files.
|
|
|
790 |
-verbose - Enable verbose output
|
|
|
791 |
|
|
|
792 |
=head1 OPTIONS
|
|
|
793 |
|
|
|
794 |
=over 8
|
|
|
795 |
|
|
|
796 |
=item B<-help>
|
|
|
797 |
|
|
|
798 |
Print a brief help message and exits.
|
|
|
799 |
|
|
|
800 |
=item B<-help -help>
|
|
|
801 |
|
|
|
802 |
Print a detailed help message with an explanation for each option.
|
|
|
803 |
|
|
|
804 |
=item B<-man>
|
|
|
805 |
|
|
|
806 |
Prints the manual page and exits.
|
|
|
807 |
|
|
|
808 |
=item B<-rtagid=xxx>
|
|
|
809 |
|
|
|
810 |
This option specifies one or more RTAG_ID's to use as the source of packages that will be copied.
|
|
|
811 |
The ID will be used to get a unique list of package/versions that can be copied from dpkg_archive.
|
|
|
812 |
|
|
|
813 |
This option is Mandatory, for non-filter command.
|
|
|
814 |
|
|
|
815 |
=item B<-rootdir=xxx>
|
|
|
816 |
|
|
|
817 |
This option specifies the root directory where the packages will be copied to.
|
|
|
818 |
|
|
|
819 |
The specified directory must exist.
|
|
|
820 |
|
|
|
821 |
The default value is the current directory.
|
|
|
822 |
|
|
|
823 |
=item B<-showfilters>
|
|
|
824 |
|
|
|
825 |
This option will display the current filter set. If it is combined with another filter operation
|
|
|
826 |
then the other operations will be performed before the display.
|
|
|
827 |
|
|
|
828 |
=item B<-addFilter=xxx[,yyy]>
|
|
|
829 |
|
|
|
830 |
This option allows new filters to be added to the set of filters. This
|
|
|
831 |
option can be specified multiple times.
|
|
|
832 |
|
|
|
833 |
This option specifies a comma separated list of shell wildcard filter rule that
|
|
|
834 |
will be used to determine which files are copied from package version directory in
|
|
|
835 |
GBE_DPKG to the release directory. This can be supplied multiple times to
|
|
|
836 |
specify rules for copying.
|
|
|
837 |
|
|
|
838 |
Filters must be added the first time this command is run against a release
|
|
|
839 |
and packages are copied to the project/release directory. These values are then written to a
|
|
|
840 |
config file in the release directory so the same values can be used on subsequent runs.
|
|
|
841 |
In these subsequent runs this option need not be specified as the config items will be used, however
|
|
|
842 |
they can be changed by specifying them again on the command line and the config will be re-written.
|
|
|
843 |
|
|
|
844 |
The values of these will depend on what builds are required for each project. Some examples are
|
|
|
845 |
--filter='*-WIN32.exe,*.deb'
|
|
|
846 |
|
|
|
847 |
=item B<-delFilter=xxx[,yyy]>
|
|
|
848 |
|
|
|
849 |
This option deletes one or more filter rules from an existing set of filters. This
|
|
|
850 |
option can be specified multiple times.
|
|
|
851 |
|
|
|
852 |
=item B<-showfiles>
|
|
|
853 |
|
|
|
854 |
This option will display the current file set. If it is combined with another file operations
|
|
|
855 |
then the other operations will be performed before the display.
|
|
|
856 |
|
|
|
857 |
=item B<-addFile=xxx[,yyy]>
|
|
|
858 |
|
|
|
859 |
This option allows new files to be added to the set of kept files. This
|
|
|
860 |
option can be specified multiple times.
|
|
|
861 |
|
|
|
862 |
This option specifies a comma separated list of file names (No wild cards) that
|
|
|
863 |
will be used to specify a list of files that shold be kept in the directory. These
|
|
|
864 |
files do not form a part of the manifest, but are not deleted by the tool.
|
|
|
865 |
|
|
|
866 |
=item B<-delFile=xxx[,yyy]>
|
|
|
867 |
|
|
|
868 |
This option deletes one or more files from the set of kept files. This
|
|
|
869 |
option can be specified multiple times.
|
|
|
870 |
|
|
|
871 |
=item B<-test>
|
|
|
872 |
|
|
|
873 |
This option will display what would be copied without actually copying anything
|
|
|
874 |
|
|
|
875 |
=item B<-verbose>
|
|
|
876 |
|
|
|
877 |
This option will display progress information as the program executes.
|
|
|
878 |
|
|
|
879 |
=back
|
|
|
880 |
|
|
|
881 |
=head1 DESCRIPTION
|
|
|
882 |
|
|
|
883 |
This program is used to update a Distribution 'bin' directory with the versions of
|
|
|
884 |
packages as indicated by the specified Deployment Manager SBoms.
|
|
|
885 |
|
|
|
886 |
There are two modes of operation: Filter modification operations and BOM creation.
|
|
|
887 |
|
|
|
888 |
In 'Filter modification' mode the current filter set will be updated and the program will
|
|
|
889 |
exit.
|
|
|
890 |
|
|
|
891 |
In BOM creation mode an sbomid must be provided.
|
|
|
892 |
|
|
|
893 |
The rtagid is used to get all the required information from Release Manager about
|
|
|
894 |
which package version are required, as well as the project name and release name.
|
|
|
895 |
|
|
|
896 |
|
|
|
897 |
In addition to using Release Manager information to determine which Package/Versions are
|
|
|
898 |
required to be copied this script also uses a set of shell wildcard filters that are
|
|
|
899 |
used to determine which files are actually copied when invoked.
|
|
|
900 |
|
|
|
901 |
The filter rules can be supplied on the command line if available read from a
|
|
|
902 |
configuration file saved in the output diretory the last time the script was run
|
|
|
903 |
on this release directory.
|
|
|
904 |
|
|
|
905 |
One or more filter rules must be specified on the command line the first time this command
|
|
|
906 |
is run against a project release directory. These filter values are then written to a config
|
|
|
907 |
file in the output directory so the same values can be used on subsequent runs.
|
|
|
908 |
In subsequent runs the filter rules will be loaded from the config file and need not be specified
|
|
|
909 |
on the command line, however the filter rules in the config file can be changed by specifying
|
|
|
910 |
them again on the command line and the config will be re-written.
|
|
|
911 |
|
|
|
912 |
=cut
|
|
|
913 |
|