| 7396 |
dpurdie |
1 |
#! /usr/bin/perl
|
|
|
2 |
########################################################################
|
|
|
3 |
# COPYRIGHT - VIX IP PTY LTD ("VIX"). ALL RIGHTS RESERVED.
|
|
|
4 |
#
|
|
|
5 |
# Module name : blatS3Sync.pl
|
|
|
6 |
# Module type :
|
|
|
7 |
# Compiler(s) : Perl
|
|
|
8 |
# Environment(s):
|
|
|
9 |
#
|
|
|
10 |
# Description : This is a blat related task that will perform S3 SYNC
|
|
|
11 |
# transfers for configured releases
|
|
|
12 |
#
|
|
|
13 |
# Usage : ARGV[0] - Path to config file for this instance
|
|
|
14 |
#
|
|
|
15 |
#......................................................................#
|
|
|
16 |
|
|
|
17 |
require 5.008_002;
|
|
|
18 |
use strict;
|
|
|
19 |
use warnings;
|
|
|
20 |
use Getopt::Long;
|
|
|
21 |
use File::Basename;
|
|
|
22 |
use Data::Dumper;
|
|
|
23 |
use File::Spec::Functions;
|
|
|
24 |
use POSIX ":sys_wait_h";
|
|
|
25 |
use File::Temp qw/tempfile/;
|
|
|
26 |
use Digest::MD5 qw(md5_base64 md5_hex);
|
|
|
27 |
use File::Path qw( rmtree );
|
|
|
28 |
use Archive::Zip qw( :ERROR_CODES :CONSTANTS );
|
|
|
29 |
use JSON;
|
|
|
30 |
|
|
|
31 |
use FindBin; # Determine the current directory
|
|
|
32 |
use lib "$FindBin::Bin/lib"; # Allow local libraries
|
|
|
33 |
|
|
|
34 |
use Utils;
|
|
|
35 |
use StdLogger; # Log to sdtout
|
|
|
36 |
use Logger; # Log to file
|
|
|
37 |
|
|
|
38 |
#
|
|
|
39 |
# Database interface
|
|
|
40 |
# Pinched from jats and modified so that this software is not dependent on JATS
|
|
|
41 |
#
|
|
|
42 |
use IO::Handle;
|
|
|
43 |
use JatsRmApi;
|
|
|
44 |
use DBI;
|
|
|
45 |
|
|
|
46 |
#
|
|
|
47 |
# Globals
|
|
|
48 |
#
|
|
|
49 |
my $logger = StdLogger->new(); # Stdout logger. Only during config
|
|
|
50 |
$logger->err("No config file specified") unless (defined $ARGV[0]);
|
|
|
51 |
$logger->err("Config File does not exist: $ARGV[0]") unless (-f $ARGV[0]);
|
|
|
52 |
my $name = basename( $ARGV[0]);
|
|
|
53 |
$name =~ s~.conf$~~;
|
|
|
54 |
my $now = 0;
|
|
|
55 |
my $startTime = 0;
|
|
|
56 |
my $tagDirTime = 0;
|
|
|
57 |
my $lastDirScan = 0;
|
|
|
58 |
my $lastS3Refresh = 0;
|
|
|
59 |
my $lastTagListUpdate = 0;
|
|
|
60 |
my $mtimeConfig = 0;
|
|
|
61 |
my $conf;
|
|
|
62 |
my $yday = -1;
|
|
|
63 |
my $linkUp = 1;
|
|
|
64 |
my $RM_DB;
|
|
|
65 |
my $activeReleases;
|
| 7397 |
dpurdie |
66 |
my $wedgedCount = 0;
|
| 7396 |
dpurdie |
67 |
|
|
|
68 |
#
|
|
|
69 |
# Contain statisics maintained while operating
|
|
|
70 |
# Can be dumped with a kill -USR2
|
|
|
71 |
# List here for documentation
|
|
|
72 |
#
|
|
|
73 |
|
|
|
74 |
my %statistics = (
|
|
|
75 |
SeqNum => 0, # Bumped when $statistics are dumped
|
|
|
76 |
timeStamp => 0, # DateTime when statistics are dumped
|
|
|
77 |
upTime => 0, # Seconds since program start
|
|
|
78 |
Cycle => 0, # Major process loop counter
|
|
|
79 |
phase => 'Init', # Current phase of operation
|
|
|
80 |
state => 'OK', # Nagios state
|
| 7397 |
dpurdie |
81 |
wedged => 0, # Wedge indication - main loop not cycling
|
| 7396 |
dpurdie |
82 |
#
|
|
|
83 |
# The following are reset each day
|
|
|
84 |
dayStart => 0, # DateTime when daily data was reset
|
|
|
85 |
txCount => 0, # Packages Transferred
|
| 7397 |
dpurdie |
86 |
txBytes => 0, # Bytes Transferred
|
| 7396 |
dpurdie |
87 |
delCount => 0, # Packages marked for deletion
|
|
|
88 |
linkErrors => 0, # Transfer (S3) errors
|
|
|
89 |
#
|
|
|
90 |
# Per Cycle Data - Calculated each processing Cycle
|
| 7406 |
dpurdie |
91 |
total => 0, # Number targets
|
| 7396 |
dpurdie |
92 |
);
|
|
|
93 |
|
|
|
94 |
#
|
|
|
95 |
# Describe configuration parameters
|
|
|
96 |
#
|
|
|
97 |
my %cdata = (
|
|
|
98 |
'piddir' => {'mandatory' => 1 , 'fmt' => 'dir'},
|
|
|
99 |
'sleep' => {'default' => 5 , 'fmt' => 'period'},
|
|
|
100 |
'sleepLinkDown' => {'default' => '1m' , 'fmt' => 'period'},
|
|
|
101 |
'dpkg_archive' => {'mandatory' => 1 , 'fmt' => 'dir'},
|
|
|
102 |
'logfile' => {'mandatory' => 1 , 'fmt' => 'vfile'},
|
|
|
103 |
'logfile.size' => {'default' => '1M' , 'fmt' => 'size'},
|
|
|
104 |
'logfile.count' => {'default' => 9 , 'fmt' => 'int'},
|
|
|
105 |
|
|
|
106 |
'verbose' => {'default' => 0 , 'fmt' => 'int'}, # Debug ...
|
|
|
107 |
'active' => {'default' => 1 , 'fmt' => 'bool'}, # Disable alltogether
|
|
|
108 |
'debug' => {'default' => 0 , 'fmt' => 'bool'}, # Log to screen
|
|
|
109 |
'txdetail' => {'default' => 0 , 'fmt' => 'bool'}, # Show transfer times
|
|
|
110 |
'noTransfers' => {'default' => 0 , 'fmt' => 'bool'}, # Debugging option to prevent transfers
|
| 7413 |
dpurdie |
111 |
's3File' => {'default' => 's3Transfer.json', 'fmt' => 'text'}, # Undocumented - debug only
|
| 7396 |
dpurdie |
112 |
|
| 7397 |
dpurdie |
113 |
'tagdir' => {'mandatory' => 1 , 'fmt' => 'mkdir'},
|
|
|
114 |
'workdir' => {'mandatory' => 1 , 'fmt' => 'mkdir'},
|
| 7396 |
dpurdie |
115 |
'forcedirscan' => {'default' => 100 , 'fmt' => 'period'},
|
|
|
116 |
'forces3update' => {'default' => '30m' , 'fmt' => 'period'},
|
|
|
117 |
'tagListUpdate' => {'default' => '1h' , 'fmt' => 'period'},
|
|
|
118 |
'S3Bucket' => {'mandatory' => 1 , 'fmt' => 'text'},
|
|
|
119 |
'S3Profile' => {'mandatory' => 1 , 'fmt' => 'text'},
|
| 7406 |
dpurdie |
120 |
'S3Region' => {'default' => undef , 'fmt' => 'text'},
|
| 7396 |
dpurdie |
121 |
|
|
|
122 |
);
|
|
|
123 |
|
|
|
124 |
|
|
|
125 |
#
|
|
|
126 |
# Read in the configuration
|
|
|
127 |
# Set up a logger
|
|
|
128 |
# Write a pidfile - thats not used
|
|
|
129 |
$now = $startTime = time();
|
|
|
130 |
readConfig();
|
|
|
131 |
Utils::writepid($conf);
|
|
|
132 |
$logger->logmsg("Starting...");
|
|
|
133 |
readStatistics();
|
| 7423 |
dpurdie |
134 |
sighandlers();
|
| 7396 |
dpurdie |
135 |
|
|
|
136 |
#
|
|
|
137 |
# Main processing loop
|
|
|
138 |
# Will exit when terminated by parent
|
|
|
139 |
#
|
|
|
140 |
while (1)
|
|
|
141 |
{
|
|
|
142 |
$logger->verbose3("Processing");
|
|
|
143 |
$statistics{Cycle}++;
|
| 7397 |
dpurdie |
144 |
$wedgedCount = 0;
|
| 7396 |
dpurdie |
145 |
$now = time();
|
|
|
146 |
|
|
|
147 |
$statistics{phase} = 'ReadConfig';
|
|
|
148 |
readConfig();
|
|
|
149 |
if ( $conf->{'active'} )
|
|
|
150 |
{
|
|
|
151 |
$statistics{phase} = 'Refresh S3 Info';
|
|
|
152 |
refreshS3Info();
|
|
|
153 |
if( $linkUp )
|
|
|
154 |
{
|
|
|
155 |
$statistics{phase} = 'Monitor Requests';
|
|
|
156 |
monitorRequests();
|
|
|
157 |
|
|
|
158 |
$statistics{phase} = 'maintainTagList';
|
|
|
159 |
maintainTagList();
|
|
|
160 |
}
|
|
|
161 |
}
|
|
|
162 |
|
|
|
163 |
$statistics{phase} = 'Sleep';
|
|
|
164 |
sleep( $linkUp ? $conf->{'sleep'} : $conf->{'sleepLinkDown'} );
|
|
|
165 |
reapChildren();
|
|
|
166 |
|
|
|
167 |
# If my PID file ceases to be, then exit the daemon
|
|
|
168 |
# Used to force daemon to restart
|
|
|
169 |
#
|
|
|
170 |
unless ( -f $conf->{'pidfile'} )
|
|
|
171 |
{
|
|
|
172 |
$logger->logmsg("Terminate. Pid file removed");
|
|
|
173 |
last;
|
|
|
174 |
}
|
|
|
175 |
}
|
|
|
176 |
$statistics{phase} = 'Terminated';
|
|
|
177 |
$logger->logmsg("Child End");
|
|
|
178 |
exit 0;
|
|
|
179 |
|
|
|
180 |
#-------------------------------------------------------------------------------
|
|
|
181 |
# Function : reapChildren
|
|
|
182 |
#
|
|
|
183 |
# Description : Reap any and all dead children
|
|
|
184 |
# Call in major loops to prevent zombies accumulating
|
|
|
185 |
#
|
|
|
186 |
# Inputs : None
|
|
|
187 |
#
|
|
|
188 |
# Returns :
|
|
|
189 |
#
|
|
|
190 |
sub reapChildren
|
|
|
191 |
{
|
|
|
192 |
my $currentPhase = $statistics{phase};
|
|
|
193 |
$statistics{phase} = 'Reaping';
|
|
|
194 |
|
|
|
195 |
my $kid;
|
|
|
196 |
do {
|
|
|
197 |
$kid = waitpid(-1, WNOHANG);
|
|
|
198 |
} while ( $kid > 0 );
|
|
|
199 |
|
|
|
200 |
$statistics{phase} = $currentPhase;
|
|
|
201 |
}
|
|
|
202 |
|
|
|
203 |
|
|
|
204 |
#-------------------------------------------------------------------------------
|
|
|
205 |
# Function : readConfig
|
|
|
206 |
#
|
|
|
207 |
# Description : Re read the config file if it modification time has changed
|
|
|
208 |
#
|
|
|
209 |
# Inputs : Nothing
|
|
|
210 |
#
|
|
|
211 |
# Returns : 0 - Config not read
|
|
|
212 |
# 1 - Config read
|
|
|
213 |
# Config file has changed
|
|
|
214 |
#
|
|
|
215 |
sub readConfig
|
|
|
216 |
{
|
|
|
217 |
my ($mtime) = Utils::mtime($ARGV[0]);
|
|
|
218 |
my $rv = 0;
|
|
|
219 |
|
|
|
220 |
if ( $mtimeConfig != $mtime )
|
|
|
221 |
{
|
|
|
222 |
$logger->logmsg("Reading config file: $ARGV[0]");
|
|
|
223 |
$mtimeConfig = $mtime;
|
|
|
224 |
my $errors;
|
|
|
225 |
($conf, $errors) = Utils::readconf ( $ARGV[0], \%cdata );
|
|
|
226 |
if ( scalar @{$errors} > 0 )
|
|
|
227 |
{
|
|
|
228 |
warn "$_\n" foreach (@{$errors});
|
|
|
229 |
die ("Config contained errors\n");
|
|
|
230 |
}
|
|
|
231 |
|
|
|
232 |
#
|
|
|
233 |
# Reset some information
|
|
|
234 |
# Create a new logger
|
|
|
235 |
#
|
|
|
236 |
$logger = Logger->new($conf) unless $conf->{debug};
|
|
|
237 |
$conf->{logger} = $logger;
|
|
|
238 |
$conf->{'pidfile'} = $conf->{'piddir'} . '/' . $name . '.pid';
|
|
|
239 |
$logger->setVerbose($conf->{verbose});
|
|
|
240 |
$logger->verbose("Log Levl: $conf->{verbose}");
|
|
|
241 |
|
|
|
242 |
#
|
|
|
243 |
# Setup statistics filename
|
|
|
244 |
$conf->{'statsfile'} = $conf->{'piddir'} . '/' . $name . '.stats';
|
|
|
245 |
$conf->{'statsfiletmp'} = $conf->{'piddir'} . '/' . $name . '.stats.tmp';
|
|
|
246 |
|
|
|
247 |
#
|
|
|
248 |
# When config is read force some actions
|
|
|
249 |
# - Force tagList to be created
|
|
|
250 |
# - Force refresh from S3
|
|
|
251 |
$lastTagListUpdate = 0;
|
|
|
252 |
$lastS3Refresh = 0;
|
| 7406 |
dpurdie |
253 |
$rv = 1;
|
| 7396 |
dpurdie |
254 |
|
| 7406 |
dpurdie |
255 |
#
|
|
|
256 |
# When config is read force some actions
|
| 7396 |
dpurdie |
257 |
#Utils::DebugDumpData ("Config", $conf);
|
|
|
258 |
|
| 7406 |
dpurdie |
259 |
$logger->warn("All Transfers disabled") if ( $conf->{'noTransfers'} );
|
|
|
260 |
$logger->warn("S3Sync is inactive") unless ( $conf->{'active'} );
|
|
|
261 |
}
|
|
|
262 |
|
| 7396 |
dpurdie |
263 |
return $rv;
|
|
|
264 |
}
|
|
|
265 |
|
|
|
266 |
#-------------------------------------------------------------------------------
|
|
|
267 |
# Function : refreshS3Info
|
|
|
268 |
#
|
|
|
269 |
# Description : At startup, and at time after startup examine the S3 bucket
|
|
|
270 |
# and recover information from it
|
|
|
271 |
#
|
|
|
272 |
# Inputs :
|
|
|
273 |
#
|
|
|
274 |
# Returns : 0 - Gross error ( Bucket access)
|
|
|
275 |
#
|
|
|
276 |
sub refreshS3Info
|
|
|
277 |
{
|
|
|
278 |
my $rv = 1;
|
|
|
279 |
if ( !$linkUp || ($now > ($lastS3Refresh + $conf->{'forces3update'})) )
|
|
|
280 |
{
|
| 7423 |
dpurdie |
281 |
$logger->verbose("refreshS3Info");
|
| 7396 |
dpurdie |
282 |
$lastS3Refresh = $now;
|
|
|
283 |
|
|
|
284 |
#
|
|
|
285 |
# Examine the s3 bucket and extract useful information
|
|
|
286 |
#
|
|
|
287 |
my $startTime = time;
|
|
|
288 |
$rv = examineS3Bucket();
|
|
|
289 |
unless ($rv) {
|
|
|
290 |
$statistics{linkErrors}++;
|
|
|
291 |
$linkUp = 0;
|
| 7397 |
dpurdie |
292 |
} else {
|
|
|
293 |
$linkUp = 1;
|
| 7396 |
dpurdie |
294 |
}
|
|
|
295 |
|
|
|
296 |
#
|
|
|
297 |
# Display the duration of the refresh
|
|
|
298 |
# Diagnostic use
|
|
|
299 |
#
|
|
|
300 |
if ($conf->{txdetail}) {
|
|
|
301 |
my $duration = time - $startTime;
|
|
|
302 |
$logger->logmsg("refreshS3Info: Stats: $duration Secs");
|
|
|
303 |
}
|
|
|
304 |
|
|
|
305 |
}
|
|
|
306 |
return $rv;
|
|
|
307 |
}
|
|
|
308 |
|
|
|
309 |
|
|
|
310 |
|
|
|
311 |
#-------------------------------------------------------------------------------
|
|
|
312 |
# Function : monitorRequests
|
|
|
313 |
#
|
|
|
314 |
# Description : Monitor S3Sync requests
|
|
|
315 |
# This is simply done my polling Release Manager - at the moment
|
|
|
316 |
#
|
|
|
317 |
# Inputs : None
|
|
|
318 |
#
|
|
|
319 |
# Returns : Nothing
|
|
|
320 |
#
|
|
|
321 |
sub monitorRequests
|
|
|
322 |
{
|
|
|
323 |
#
|
|
|
324 |
# Determine if new tags are present by examining the time
|
|
|
325 |
# that the directory was last modified.
|
|
|
326 |
#
|
|
|
327 |
# Allow for a forced scan to catch packages that did not transfer
|
|
|
328 |
# on the first attempt
|
|
|
329 |
#
|
|
|
330 |
my $tagCount = 0;
|
|
|
331 |
my ($mtime) = Utils::mtime($conf->{'tagdir'} );
|
|
|
332 |
if ( ($mtime > $tagDirTime) || ($now > ($lastDirScan + $conf->{'forcedirscan'})) )
|
|
|
333 |
{
|
|
|
334 |
$logger->verbose2("monitorRequests: $conf->{'tagdir'}");
|
|
|
335 |
#$logger->verbose2("monitorRequests: mtime:" . ($mtime > $tagDirTime));
|
|
|
336 |
#$logger->verbose2("monitorRequests: last:" . ($now > ($lastDirScan + $conf->{'forcedirscan'})));
|
|
|
337 |
|
|
|
338 |
#
|
|
|
339 |
# Package tags information is not really used
|
|
|
340 |
# Just delete all the tags
|
|
|
341 |
# Used to trigger the scan - rather than rely on the slow data
|
|
|
342 |
# base poll. Still need a change in release sequence number
|
|
|
343 |
#
|
|
|
344 |
my $dh;
|
|
|
345 |
unless (opendir($dh, $conf->{'tagdir'}))
|
|
|
346 |
{
|
|
|
347 |
$logger->warn ("can't opendir $conf->{'tagdir'}: $!");
|
|
|
348 |
return;
|
|
|
349 |
}
|
|
|
350 |
|
|
|
351 |
#
|
|
|
352 |
# Process each entry
|
|
|
353 |
# Ignore those that start with a .
|
|
|
354 |
# Remove all files
|
|
|
355 |
#
|
|
|
356 |
while (my $tag = readdir($dh) )
|
|
|
357 |
{
|
|
|
358 |
next if ( $tag =~ m~^\.~ );
|
|
|
359 |
my $file = "$conf->{'tagdir'}/$tag";
|
|
|
360 |
$logger->verbose3("processTags: $file");
|
|
|
361 |
|
|
|
362 |
next unless ( -f $file );
|
|
|
363 |
unlink $file;
|
|
|
364 |
}
|
|
|
365 |
|
|
|
366 |
#
|
|
|
367 |
# Reset the scan time triggers
|
|
|
368 |
#
|
|
|
369 |
$tagDirTime = $mtime;
|
|
|
370 |
$lastDirScan = $now;
|
|
|
371 |
|
|
|
372 |
#
|
|
|
373 |
# Examine Release Manager looking for active releases that have S3Sync support
|
|
|
374 |
# Purpose is to:
|
|
|
375 |
# Detect new Releases
|
|
|
376 |
# Detect dead Releases
|
|
|
377 |
# Detect changed Releases
|
|
|
378 |
#
|
|
|
379 |
connectRM(\$RM_DB, $conf->{verbose} > 3);
|
|
|
380 |
|
|
|
381 |
foreach my $rtag_id (keys %{$activeReleases}) {
|
|
|
382 |
$activeReleases->{$rtag_id}{exists} = 0;
|
|
|
383 |
}
|
|
|
384 |
my $m_sqlstr = "SELECT rt.rtag_id,rm.seqnum, rt.s3sync, rt.official, rm.timestamp " .
|
|
|
385 |
"FROM RELEASE_MANAGER.release_tags rt, RELEASE_MANAGER.release_modified rm " .
|
|
|
386 |
"WHERE rt.s3sync = 'Y' AND rm.rtag_id = rt.rtag_id AND rt.official in ('N', 'R', 'C')";
|
|
|
387 |
|
|
|
388 |
my $curData = getDataFromRm ('monitorRequests', $m_sqlstr, {data => 0} );
|
| 7406 |
dpurdie |
389 |
$statistics{total} = scalar @{$curData};
|
|
|
390 |
|
| 7396 |
dpurdie |
391 |
foreach my $entry (@{$curData}) {
|
|
|
392 |
my ($rtag_id, $seqnum) = @{$entry};
|
| 7406 |
dpurdie |
393 |
$logger->verbose3("rtagid: $rtag_id, seqnumm: $seqnum");
|
| 7396 |
dpurdie |
394 |
|
|
|
395 |
if (! exists $activeReleases->{$rtag_id} || ! exists $activeReleases->{$rtag_id}{s3} ) {
|
|
|
396 |
$logger->logmsg("New Release Detected. rtag_id: $rtag_id, seq:$seqnum");
|
|
|
397 |
processChangedRelease($rtag_id, $seqnum);
|
|
|
398 |
$lastTagListUpdate = 0;
|
|
|
399 |
|
|
|
400 |
} elsif (($activeReleases->{$rtag_id}{seqnum} || 0) ne ($seqnum || 0) ) {
|
|
|
401 |
$logger->logmsg("Change Release Detected. rtag_id: $rtag_id, seq:$seqnum");
|
|
|
402 |
processChangedRelease($rtag_id, $seqnum);
|
|
|
403 |
}
|
|
|
404 |
|
|
|
405 |
# Update activeReleases so that changes will be detected
|
|
|
406 |
$activeReleases->{$rtag_id}{exists} = 1;
|
|
|
407 |
}
|
|
|
408 |
|
|
|
409 |
# Detect Releases that are no longer active
|
|
|
410 |
foreach my $rtag_id (keys %{$activeReleases}) {
|
|
|
411 |
unless ($activeReleases->{$rtag_id}{exists}) {
|
|
|
412 |
$logger->logmsg("Dead Release Detected. rtag_id: $rtag_id");
|
|
|
413 |
removeDeadRelease($rtag_id);
|
|
|
414 |
delete $activeReleases->{$rtag_id};
|
|
|
415 |
$lastTagListUpdate = 0;
|
|
|
416 |
}
|
|
|
417 |
}
|
|
|
418 |
|
|
|
419 |
disconnectRM(\$RM_DB);
|
|
|
420 |
}
|
|
|
421 |
}
|
|
|
422 |
|
|
|
423 |
#-------------------------------------------------------------------------------
|
|
|
424 |
# Function : examineS3Bucket
|
|
|
425 |
#
|
|
|
426 |
# Description : Scan the S3 bucket looking for Releases
|
|
|
427 |
# Used to pre-populate the process so that we:
|
|
|
428 |
# - Delete dead releases
|
|
|
429 |
# - Don't do excessive work on startup
|
|
|
430 |
#
|
|
|
431 |
# Inputs : Nothing
|
|
|
432 |
#
|
|
|
433 |
# Returns : Updates global structure ($activeReleases)
|
|
|
434 |
# Returns : 0 - Gross error ( Bucket access)
|
|
|
435 |
#
|
|
|
436 |
sub examineS3Bucket
|
|
|
437 |
{
|
| 7406 |
dpurdie |
438 |
my $bucket;
|
|
|
439 |
my $prefix;
|
|
|
440 |
|
| 7396 |
dpurdie |
441 |
#
|
|
|
442 |
# Remove data collected from s3
|
|
|
443 |
#
|
|
|
444 |
foreach my $rtag_id (keys %{$activeReleases}) {
|
|
|
445 |
delete $activeReleases->{$rtag_id}{s3} ;
|
|
|
446 |
}
|
|
|
447 |
|
| 7406 |
dpurdie |
448 |
if ($conf->{'S3Bucket'} =~ m~(.*?)/(.*)~) {
|
|
|
449 |
$bucket = $1;
|
|
|
450 |
$prefix = $2;
|
|
|
451 |
} else {
|
|
|
452 |
$bucket = $conf->{'S3Bucket'};
|
|
|
453 |
}
|
| 7396 |
dpurdie |
454 |
|
| 7406 |
dpurdie |
455 |
my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
|
|
|
456 |
$s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
|
|
|
457 |
$s3_cmd .= " s3api list-objects --bucket $bucket";
|
|
|
458 |
$s3_cmd .= " --prefix '$prefix'" if (defined $prefix);
|
|
|
459 |
|
| 7396 |
dpurdie |
460 |
$logger->verbose2("examineS3Bucket:s3_cmd:$s3_cmd");
|
|
|
461 |
|
|
|
462 |
my $ph;
|
|
|
463 |
my $jsontxt = "";
|
|
|
464 |
open ($ph, "$s3_cmd |");
|
|
|
465 |
while ( <$ph> ) {
|
|
|
466 |
chomp;
|
|
|
467 |
$logger->verbose3("examineS3Bucket:Data: $_");
|
|
|
468 |
$jsontxt .= $_;
|
|
|
469 |
}
|
|
|
470 |
close ($ph);
|
|
|
471 |
my $cmdRv = $?;
|
|
|
472 |
if ($cmdRv != 0) {
|
|
|
473 |
$logger->warn("Cannot read S3 Bucket Data");
|
|
|
474 |
return 0;
|
|
|
475 |
}
|
|
|
476 |
|
|
|
477 |
if ($jsontxt) {
|
|
|
478 |
my $json = from_json ($jsontxt);
|
|
|
479 |
#Utils::DebugDumpData("JSON",$json->{'Contents'});
|
|
|
480 |
foreach my $item ( @{$json->{'Contents'}})
|
|
|
481 |
{
|
| 7406 |
dpurdie |
482 |
if ($item->{Key} =~ m~(?:^|/)Release-(.*)\.zip$~ ) {
|
| 7396 |
dpurdie |
483 |
|
|
|
484 |
my $rtag_id = $1;
|
|
|
485 |
my $metaData = gets3ObjectMetaData($item->{Key});
|
|
|
486 |
|
|
|
487 |
#
|
|
|
488 |
# Update info in the global structure ($activeReleases)
|
|
|
489 |
# This data could be discarded - only needed for diagnostics
|
|
|
490 |
#
|
|
|
491 |
$activeReleases->{$rtag_id}{s3}{seqnum} = $metaData->{'releaseseq'};
|
|
|
492 |
$activeReleases->{$rtag_id}{s3}{md5} = $metaData->{'md5'};
|
|
|
493 |
$activeReleases->{$rtag_id}{s3}{depsig} = $metaData->{'depsig'};
|
|
|
494 |
|
|
|
495 |
#
|
|
|
496 |
# Recover information from S3
|
|
|
497 |
# Should only be done on the first call after restart
|
|
|
498 |
#
|
|
|
499 |
if (! exists $activeReleases->{$rtag_id}{md5} ) {
|
|
|
500 |
$activeReleases->{$rtag_id}{md5} = $activeReleases->{$rtag_id}{s3}{md5};
|
|
|
501 |
$activeReleases->{$rtag_id}{depsig} = $activeReleases->{$rtag_id}{s3}{depsig};
|
|
|
502 |
}
|
|
|
503 |
|
|
|
504 |
|
|
|
505 |
} else {
|
|
|
506 |
$logger->warn("Unknown item in bucket: $item->{Key}");
|
|
|
507 |
}
|
|
|
508 |
}
|
|
|
509 |
}
|
|
|
510 |
#Utils::DebugDumpData("activeReleases",$activeReleases);
|
|
|
511 |
return 1;
|
|
|
512 |
}
|
|
|
513 |
|
|
|
514 |
#-------------------------------------------------------------------------------
|
|
|
515 |
# Function : gets3ObjectMetaData
|
|
|
516 |
#
|
|
|
517 |
# Description : Get Metadata about one object
|
|
|
518 |
# Must do object by object :(
|
|
|
519 |
#
|
|
|
520 |
# Inputs : $key - Key
|
|
|
521 |
#
|
|
|
522 |
# Returns :
|
|
|
523 |
#
|
|
|
524 |
|
|
|
525 |
sub gets3ObjectMetaData
|
|
|
526 |
{
|
|
|
527 |
my ($key) = @_;
|
| 7406 |
dpurdie |
528 |
my $bucket;
|
|
|
529 |
my $prefix;
|
| 7396 |
dpurdie |
530 |
|
| 7406 |
dpurdie |
531 |
if ($conf->{'S3Bucket'} =~ m~(.*?)/(.*)~) {
|
|
|
532 |
$bucket = $1;
|
|
|
533 |
$prefix = $2;
|
|
|
534 |
} else {
|
|
|
535 |
$bucket = $conf->{'S3Bucket'};
|
|
|
536 |
$prefix = '';
|
|
|
537 |
}
|
| 7396 |
dpurdie |
538 |
|
| 7406 |
dpurdie |
539 |
my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
|
|
|
540 |
$s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
|
|
|
541 |
$s3_cmd .= " s3api head-object --bucket $bucket --key $key";
|
|
|
542 |
|
| 7396 |
dpurdie |
543 |
$logger->verbose2("gets3ObjectMetaData:s3_cmd:$s3_cmd");
|
|
|
544 |
|
|
|
545 |
my $ph;
|
|
|
546 |
my $jsontxt = "";
|
|
|
547 |
open ($ph, "$s3_cmd |");
|
|
|
548 |
while ( <$ph> ) {
|
|
|
549 |
chomp;
|
|
|
550 |
$logger->verbose3("gets3ObjectMetaData:Data: $_");
|
|
|
551 |
$jsontxt .= $_;
|
|
|
552 |
}
|
|
|
553 |
close ($ph);
|
|
|
554 |
|
|
|
555 |
my $json;
|
|
|
556 |
$json->{Metadata} = {};
|
|
|
557 |
|
|
|
558 |
if ($jsontxt) {
|
|
|
559 |
$json = from_json ($jsontxt);
|
|
|
560 |
#Utils::DebugDumpData("JSON",$json);
|
|
|
561 |
}
|
|
|
562 |
return $json->{Metadata};
|
|
|
563 |
}
|
|
|
564 |
|
|
|
565 |
|
|
|
566 |
#-------------------------------------------------------------------------------
|
|
|
567 |
# Function : removeDeadRelease
|
|
|
568 |
#
|
|
|
569 |
# Description : Remove a Dead Release from the S3 bucket
|
|
|
570 |
#
|
|
|
571 |
# Inputs : $rtag_id - Release identifier
|
|
|
572 |
#
|
|
|
573 |
# Returns : 0 - Nothing deleted
|
|
|
574 |
# 1 - Something deleted
|
|
|
575 |
#
|
|
|
576 |
sub removeDeadRelease {
|
|
|
577 |
my ($rtag_id) = @_;
|
|
|
578 |
my $cmdRv;
|
|
|
579 |
my $rv = 0;
|
|
|
580 |
|
|
|
581 |
# Create the process pipe to delete the package
|
|
|
582 |
|
|
|
583 |
my $targetPath = generateBucketZipName($rtag_id);
|
| 7406 |
dpurdie |
584 |
|
|
|
585 |
my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
|
|
|
586 |
$s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
|
|
|
587 |
$s3_cmd .= " s3 rm s3://$targetPath";
|
|
|
588 |
|
| 7396 |
dpurdie |
589 |
$logger->logmsg("removeDeadRelease:$targetPath");
|
|
|
590 |
$logger->verbose2("removeDeadRelease:s3_cmd:$s3_cmd");
|
|
|
591 |
|
|
|
592 |
my $ph;
|
|
|
593 |
open ($ph, "$s3_cmd |");
|
|
|
594 |
while ( <$ph> ) {
|
|
|
595 |
chomp;
|
|
|
596 |
$logger->verbose2("removeDeadRelease:Data: $_");
|
|
|
597 |
}
|
|
|
598 |
close ($ph);
|
|
|
599 |
$cmdRv = $?;
|
|
|
600 |
|
|
|
601 |
#
|
|
|
602 |
# Common code
|
|
|
603 |
#
|
|
|
604 |
$logger->verbose("removeDeadRelease:End: $cmdRv");
|
|
|
605 |
if ( $cmdRv == 0 ) {
|
|
|
606 |
$rv = 1;
|
|
|
607 |
$statistics{delCount}++;
|
|
|
608 |
|
|
|
609 |
} else {
|
|
|
610 |
$logger->warn("removeDeadRelease:Error: $rtag_id, $?");
|
|
|
611 |
}
|
|
|
612 |
return $rv;
|
|
|
613 |
}
|
|
|
614 |
|
|
|
615 |
#-------------------------------------------------------------------------------
|
|
|
616 |
# Function : processChangedRelease
|
|
|
617 |
#
|
|
|
618 |
# Description : Create/Update a release to the S3 bucket
|
|
|
619 |
#
|
|
|
620 |
# Various attempts are made to reduce the work that needs to be done
|
|
|
621 |
# There are three checks to skip a transfer
|
|
|
622 |
# 1) The Release sequence number - must be diff for processing to occur
|
|
|
623 |
# 2) Packages inserted into the image
|
|
|
624 |
# Dependent package versions are used to generate a MD5
|
|
|
625 |
# If this does not change then there is no need to do work
|
|
|
626 |
# 3) An MD5 over the zip Image
|
|
|
627 |
# If this is the same as the one in S3, then don't upload
|
|
|
628 |
#
|
|
|
629 |
# These three pices of information are held as metadata along with the
|
|
|
630 |
# package. These are read at start up.
|
|
|
631 |
#
|
|
|
632 |
#
|
|
|
633 |
# Inputs : $rtag_id - Release identifier
|
|
|
634 |
# $seqnum - Release sequence number
|
|
|
635 |
# Added as metadata to objects
|
|
|
636 |
#
|
|
|
637 |
# Returns : Nothing
|
|
|
638 |
#
|
|
|
639 |
sub processChangedRelease {
|
|
|
640 |
my ($rtag_id, $seqnum) = @_;
|
|
|
641 |
|
|
|
642 |
#
|
|
|
643 |
# Cleanout previous zip files
|
|
|
644 |
#
|
|
|
645 |
my @files = glob($conf->{'workdir'} . '/*.zip');
|
|
|
646 |
$logger->verbose("Delete old zips: @files");
|
|
|
647 |
unlink @files;
|
|
|
648 |
|
|
|
649 |
#
|
|
|
650 |
# Create an image of the data to be transferred
|
|
|
651 |
# Based on packages that support S3Sync
|
|
|
652 |
#
|
|
|
653 |
my $m_sqlstr = "SELECT p.pkg_name, pv.pkg_version, pv.pv_id " .
|
|
|
654 |
" FROM RELEASE_MANAGER.release_content rc, RELEASE_MANAGER.packages p, RELEASE_MANAGER.package_versions pv " .
|
|
|
655 |
" WHERE rc.rtag_id = " . $rtag_id .
|
|
|
656 |
" AND rc.s3sync = 'Y' " .
|
|
|
657 |
" AND rc.pv_id = pv.pv_id " .
|
|
|
658 |
" AND pv.pkg_id = p.pkg_id " .
|
|
|
659 |
" ORDER by pv.pv_id";
|
|
|
660 |
|
|
|
661 |
my $curData = getDataFromRm ('s3Pkgs', $m_sqlstr, {data => 0, dump => 0} );
|
|
|
662 |
#Utils::DebugDumpData("activeReleases",$activeReleases);
|
|
|
663 |
|
|
|
664 |
#
|
|
|
665 |
# Generate a md5 of the PVIDs of the packages that will go into the image
|
|
|
666 |
# Used to detect true changes - only of the packages we are interested in
|
|
|
667 |
#
|
|
|
668 |
my $signature = Digest::MD5->new;
|
|
|
669 |
foreach my $entry (@{$curData}) {
|
|
|
670 |
$signature->add( $entry->[2] )
|
|
|
671 |
}
|
|
|
672 |
my $depsig = $signature->hexdigest();
|
|
|
673 |
my $reason = "";
|
|
|
674 |
if ( !exists $activeReleases->{$rtag_id}{s3} ) {
|
|
|
675 |
$reason = 'NoS3Data';
|
|
|
676 |
|
|
|
677 |
} elsif (! exists $activeReleases->{$rtag_id}{depsig}) {
|
|
|
678 |
$reason = 'NoSavedData';
|
|
|
679 |
|
| 7413 |
dpurdie |
680 |
} elsif ( $activeReleases->{$rtag_id}{depsig} ne $depsig ) {
|
| 7396 |
dpurdie |
681 |
$reason = "Mismatch: $activeReleases->{$rtag_id}{depsig} ne $depsig";
|
|
|
682 |
|
|
|
683 |
} else {
|
|
|
684 |
$logger->verbose("Dependencies unchanged - upload skipped");
|
|
|
685 |
$activeReleases->{$rtag_id}{seqnum} = $seqnum;
|
|
|
686 |
return;
|
|
|
687 |
}
|
|
|
688 |
$logger->verbose("Dependency Test: $reason");
|
|
|
689 |
|
|
|
690 |
#
|
| 7397 |
dpurdie |
691 |
# Create a monifest to go into the zip
|
| 7396 |
dpurdie |
692 |
#
|
| 7397 |
dpurdie |
693 |
my $manifest;
|
|
|
694 |
$manifest->{Packages} = [];
|
|
|
695 |
$manifest->{rtag_id} = $rtag_id;
|
|
|
696 |
|
|
|
697 |
#
|
|
|
698 |
# Generate the zip of the objects to be pushed to S3
|
|
|
699 |
# Add directories
|
|
|
700 |
# Update the manifest entries
|
|
|
701 |
#
|
| 7396 |
dpurdie |
702 |
my $startTime = time;
|
|
|
703 |
my $zip = Archive::Zip->new();
|
|
|
704 |
foreach my $entry (@{$curData}) {
|
|
|
705 |
|
| 7413 |
dpurdie |
706 |
#
|
|
|
707 |
# The TxList will contain information on every file to be transferred
|
|
|
708 |
# The data is an array of entries
|
|
|
709 |
# Each entry is a hash of name, version, an array of files
|
|
|
710 |
# Each file entry will be of the form
|
|
|
711 |
# srcFilePath [targetFilePath]
|
|
|
712 |
# The [targetFilePath] is optional. If not present, then the target is the same as the source
|
|
|
713 |
#
|
|
|
714 |
my $pkgList = getPackageTxList($entry->[0], $entry->[1]);
|
|
|
715 |
if ($pkgList) {
|
|
|
716 |
|
| 7397 |
dpurdie |
717 |
my %data;
|
|
|
718 |
$data{name} = $entry->[0];
|
|
|
719 |
$data{version} = $entry->[1];
|
|
|
720 |
$data{pvid} = $entry->[2];
|
|
|
721 |
push @{$manifest->{Packages}}, \%data;
|
|
|
722 |
|
| 7413 |
dpurdie |
723 |
foreach my $pentry (@{$pkgList->{packages}}) {
|
|
|
724 |
my $pkgBase = getPackageBase($pentry->{name}, $pentry->{version});
|
|
|
725 |
if ($pkgBase) {
|
|
|
726 |
my @txList = @{$pentry->{files}};
|
|
|
727 |
foreach my $fileEntry (@txList) {
|
|
|
728 |
my ($src,$dst);
|
|
|
729 |
if ($fileEntry =~ m~(.*) \[(.*)\]$~) {
|
|
|
730 |
$src = $1;
|
|
|
731 |
$dst = $2;
|
|
|
732 |
$logger->verbose("Zip add: $pentry->{name}:$pentry->{version}:$src as $dst");
|
|
|
733 |
} else {
|
|
|
734 |
$dst = $src = $fileEntry;
|
|
|
735 |
$logger->verbose("Zip add: $pentry->{name}:$pentry->{version}:$src");
|
|
|
736 |
}
|
|
|
737 |
$src = catdir($pkgBase, $src);
|
|
|
738 |
unless (-f $src) {
|
|
|
739 |
$logger->warn("Cannot find file: $src");
|
|
|
740 |
return;
|
|
|
741 |
}
|
|
|
742 |
$zip->addFile($src, $dst);
|
|
|
743 |
}
|
|
|
744 |
}
|
| 7396 |
dpurdie |
745 |
}
|
|
|
746 |
}
|
|
|
747 |
}
|
|
|
748 |
|
| 7397 |
dpurdie |
749 |
# Add the manifest into the zip
|
| 7413 |
dpurdie |
750 |
# Set its modifiedTime to a fixed value, so that it doesn't keep changing
|
|
|
751 |
# Must use a time >= 1980.
|
|
|
752 |
#
|
|
|
753 |
my $jsonText = to_json( $manifest, { ascii => 1, pretty => 1, canonical => 1 });
|
|
|
754 |
$logger->verbose("Zip add: ReleaseManifest.json" );
|
|
|
755 |
my $member = $zip->addString( $jsonText, 'ReleaseManifest.json' );
|
|
|
756 |
$member->setLastModFileDateTimeFromUnix( 900000000 );
|
|
|
757 |
$logger->verbose2("ManifestJson: $jsonText");
|
| 7397 |
dpurdie |
758 |
|
|
|
759 |
# Generate the zip file
|
| 7396 |
dpurdie |
760 |
my $zipFile = catdir( $conf->{'workdir'} , 'Images-' . $rtag_id . '.zip');
|
|
|
761 |
if ( $zip->writeToFileNamed($zipFile) != AZ_OK ) {
|
|
|
762 |
$logger->warn("Zip write Error: $rtag_id");
|
|
|
763 |
return;
|
|
|
764 |
}
|
|
|
765 |
$logger->verbose("Zip created: $zipFile");
|
|
|
766 |
|
|
|
767 |
#
|
|
|
768 |
# Display the size of the package (zipped)
|
|
|
769 |
# Diagnostic use
|
|
|
770 |
#
|
|
|
771 |
if ($conf->{txdetail}) {
|
|
|
772 |
my $tzfsize = -s $zipFile;
|
|
|
773 |
my $size = sprintf "%.3f", $tzfsize / 1024 / 1024 / 1024 ;
|
|
|
774 |
my $duration = time - $startTime;
|
|
|
775 |
$logger->logmsg("zipImage: Stats: $rtag_id, $size Gb, $duration Secs");
|
|
|
776 |
}
|
|
|
777 |
|
|
|
778 |
#
|
|
|
779 |
# Have a ZIP file of the desired contents
|
|
|
780 |
# Could try to detect if it differs from the one already in the bucket
|
|
|
781 |
# Don't want to trigger CI/CD pipeline operations unless we need to
|
| 7410 |
dpurdie |
782 |
#
|
|
|
783 |
my $digest;
|
|
|
784 |
my $md5Digestor = Digest::MD5->new;
|
|
|
785 |
if (open( my $zipFileHandle, '<', $zipFile )) {
|
|
|
786 |
binmode ($zipFileHandle);
|
|
|
787 |
$md5Digestor->addfile($zipFileHandle);
|
|
|
788 |
$digest = $md5Digestor->hexdigest();
|
|
|
789 |
} else {
|
|
|
790 |
$logger->warn("Cannot open ZIP: $rtag_id, $!");
|
|
|
791 |
return;
|
|
|
792 |
}
|
|
|
793 |
$logger->verbose("Zip MD5: $digest");
|
| 7396 |
dpurdie |
794 |
$reason = "";
|
|
|
795 |
if ( !exists $activeReleases->{$rtag_id}{s3} ) {
|
|
|
796 |
$reason = 'NoS3Data';
|
|
|
797 |
|
|
|
798 |
} elsif (! exists $activeReleases->{$rtag_id}{md5}) {
|
|
|
799 |
$reason = 'NoSavedMd5';
|
|
|
800 |
|
|
|
801 |
} elsif ($activeReleases->{$rtag_id}{md5} ne $digest ) {
|
|
|
802 |
$reason = "Mismatch: $activeReleases->{$rtag_id}{md5} ne $digest";
|
|
|
803 |
|
|
|
804 |
} else {
|
|
|
805 |
$logger->verbose("Zip file has same md5 hash - upload skipped");
|
|
|
806 |
#
|
|
|
807 |
# Update the known signature
|
|
|
808 |
$activeReleases->{$rtag_id}{depsig} = $depsig;
|
|
|
809 |
$activeReleases->{$rtag_id}{seqnum} = $seqnum;
|
|
|
810 |
return;
|
|
|
811 |
}
|
|
|
812 |
$logger->verbose("ZipMd5 Test: $reason");
|
|
|
813 |
|
|
|
814 |
# Create a command to transfer the file to AWS use the cli tools
|
|
|
815 |
# Note: Ive seen problem with this when used from Perth to AWS (Sydney)
|
|
|
816 |
# If this is an issue use curl - see the savePkgToS3.sh for an implementation
|
|
|
817 |
#
|
|
|
818 |
$startTime = time;
|
|
|
819 |
my $targetPath = generateBucketZipName($rtag_id);
|
| 7406 |
dpurdie |
820 |
|
|
|
821 |
my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
|
|
|
822 |
$s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
|
|
|
823 |
$s3_cmd .= " s3 cp $zipFile s3://$targetPath --metadata releaseseq=$seqnum,md5=$digest,depsig=$depsig";
|
|
|
824 |
|
| 7396 |
dpurdie |
825 |
$logger->logmsg("transferPackage:$targetPath");
|
|
|
826 |
$logger->verbose2("transferPackage:s3_cmd:$s3_cmd");
|
|
|
827 |
|
|
|
828 |
my $cmdRv;
|
|
|
829 |
unless ($conf->{'noTransfers'}) {
|
|
|
830 |
my $ph;
|
|
|
831 |
open ($ph, "$s3_cmd |");
|
|
|
832 |
while ( <$ph> )
|
|
|
833 |
{
|
|
|
834 |
chomp;
|
|
|
835 |
$logger->verbose2("transferPackage:Data: $_");
|
|
|
836 |
}
|
|
|
837 |
close ($ph);
|
|
|
838 |
$cmdRv = $?;
|
|
|
839 |
$logger->verbose("transferPackage:End: $cmdRv");
|
|
|
840 |
}
|
|
|
841 |
#
|
|
|
842 |
# Display the size of the package (zipped)
|
|
|
843 |
# Diagnostic use
|
|
|
844 |
#
|
|
|
845 |
if ($conf->{txdetail}) {
|
|
|
846 |
my $tzfsize = -s $zipFile;
|
|
|
847 |
my $size = sprintf "%.3f", $tzfsize / 1024 / 1024 / 1024 ;
|
|
|
848 |
my $duration = time - $startTime;
|
|
|
849 |
$logger->logmsg("S3 Copy: Stats: $rtag_id, $size Gb, $duration Secs");
|
|
|
850 |
}
|
|
|
851 |
|
|
|
852 |
if ($cmdRv == 0) {
|
|
|
853 |
$statistics{txCount}++;
|
| 7397 |
dpurdie |
854 |
$statistics{txBytes} += -s $zipFile;
|
| 7396 |
dpurdie |
855 |
|
|
|
856 |
#
|
|
|
857 |
# Mark the current entry as having been processed
|
|
|
858 |
#
|
|
|
859 |
$activeReleases->{$rtag_id}{depsig} = $depsig;
|
|
|
860 |
$activeReleases->{$rtag_id}{md5} = $digest;
|
|
|
861 |
$activeReleases->{$rtag_id}{seqnum} = $seqnum;
|
|
|
862 |
$activeReleases->{$rtag_id}{s3}{sent} = 1;
|
|
|
863 |
}
|
|
|
864 |
}
|
|
|
865 |
|
|
|
866 |
#-------------------------------------------------------------------------------
|
| 7413 |
dpurdie |
867 |
# Function : getPackageTxList
|
| 7396 |
dpurdie |
868 |
#
|
| 7413 |
dpurdie |
869 |
# Description : Locate and read in the s3Transfer.json file found with
|
|
|
870 |
# the specified package
|
|
|
871 |
#
|
|
|
872 |
# Inputs : $pname - Package name
|
|
|
873 |
# $pver - Package version
|
|
|
874 |
#
|
|
|
875 |
#
|
|
|
876 |
# Returns : undef - bad
|
|
|
877 |
# Ref to a hash of data
|
|
|
878 |
sub getPackageTxList {
|
|
|
879 |
my ($pname, $pver) = @_;
|
|
|
880 |
|
|
|
881 |
#
|
|
|
882 |
# Locate package
|
|
|
883 |
#
|
|
|
884 |
my $pkgBase = getPackageBase($pname, $pver);
|
|
|
885 |
return undef unless $pkgBase;
|
|
|
886 |
|
|
|
887 |
my $src = catdir( $pkgBase, $conf->{s3File});
|
|
|
888 |
unless ( -f $src ) {
|
|
|
889 |
$logger->verbose("getPackageTxList: Package has no $conf->{s3File}: $pname, $pver");
|
|
|
890 |
return undef;
|
|
|
891 |
}
|
|
|
892 |
|
|
|
893 |
#
|
|
|
894 |
# Read the entire file into a string
|
|
|
895 |
#
|
|
|
896 |
my $fh;
|
|
|
897 |
unless (open ( $fh, '<', $src) ) {
|
|
|
898 |
$logger->verbose("getPackageTxList: Cannot open s3Transfer.json: $pname, $pver, $!");
|
|
|
899 |
return undef;
|
|
|
900 |
}
|
|
|
901 |
my $jsontext = do { local $/; <$fh> };
|
|
|
902 |
close $fh;
|
|
|
903 |
|
|
|
904 |
my $json = from_json($jsontext);
|
|
|
905 |
unless ( exists $json->{format}) {
|
|
|
906 |
$logger->verbose("getPackageTxList: s3Transfer.json has no format: $pname, $pver");
|
|
|
907 |
return undef;
|
|
|
908 |
}
|
|
|
909 |
unless ( exists $json->{packages}) {
|
|
|
910 |
$logger->verbose("getPackageTxList: s3Transfer.json has no packages: $pname, $pver");
|
|
|
911 |
return undef;
|
|
|
912 |
}
|
|
|
913 |
return $json;
|
|
|
914 |
}
|
|
|
915 |
|
|
|
916 |
#-------------------------------------------------------------------------------
|
|
|
917 |
# Function : getPackageBase
|
|
|
918 |
#
|
| 7396 |
dpurdie |
919 |
# Description : Calculate the base of a package in dpkg_archive
|
|
|
920 |
# With errors and wanings
|
|
|
921 |
#
|
|
|
922 |
# Inputs : $pname - Package name
|
|
|
923 |
# $pver - Package version
|
|
|
924 |
#
|
|
|
925 |
#
|
|
|
926 |
# Returns : undef - bad
|
| 7413 |
dpurdie |
927 |
# Path to root of the package in dpkg_archive
|
| 7396 |
dpurdie |
928 |
sub getPackageBase {
|
|
|
929 |
my ($pname, $pver) = @_;
|
|
|
930 |
|
|
|
931 |
#
|
|
|
932 |
# Locate package
|
|
|
933 |
#
|
|
|
934 |
unless ( -d $conf->{'dpkg_archive'}) {
|
| 7413 |
dpurdie |
935 |
$logger->warn("getPackageBase: dpkg_archive not found");
|
| 7396 |
dpurdie |
936 |
return undef;
|
|
|
937 |
}
|
|
|
938 |
|
|
|
939 |
my $src = catdir($conf->{'dpkg_archive'}, $pname, $pver);
|
|
|
940 |
unless ( -d $src ) {
|
| 7413 |
dpurdie |
941 |
$logger->warn("getPackageBase: Package not found: $pname, $pver");
|
| 7396 |
dpurdie |
942 |
return undef;
|
|
|
943 |
}
|
|
|
944 |
|
|
|
945 |
return $src;
|
|
|
946 |
}
|
|
|
947 |
|
|
|
948 |
#-------------------------------------------------------------------------------
|
|
|
949 |
# Function : generateBucketZipName
|
|
|
950 |
#
|
|
|
951 |
# Description : Generate the name of the zipfile created within the bucket
|
|
|
952 |
#
|
|
|
953 |
# Inputs : $rtag_id
|
|
|
954 |
#
|
|
|
955 |
# Returns : Full name - including bucket name
|
|
|
956 |
#
|
|
|
957 |
sub generateBucketZipName
|
|
|
958 |
{
|
|
|
959 |
my ($rtag_id) = @_;
|
|
|
960 |
my $targetName = 'Release-' . $rtag_id . '.zip';
|
|
|
961 |
my $targetPath = catdir ($conf->{'S3Bucket'}, $targetName );
|
|
|
962 |
return $targetPath;
|
|
|
963 |
}
|
|
|
964 |
|
|
|
965 |
|
|
|
966 |
#-------------------------------------------------------------------------------
|
|
|
967 |
# Function : getDataFromRm
|
|
|
968 |
#
|
|
|
969 |
# Description : Get an array of data from RM
|
|
|
970 |
# Normally an array of arrays
|
|
|
971 |
#
|
|
|
972 |
# Inputs : $name - Query Name
|
|
|
973 |
# $m_sqlstr - Query
|
|
|
974 |
# $options - Ref to a hash of options
|
|
|
975 |
# sql - show sql
|
|
|
976 |
# data - show data
|
|
|
977 |
# dump - show results
|
|
|
978 |
# oneRow - Only fetch one row
|
|
|
979 |
# error - Must find data
|
|
|
980 |
#
|
|
|
981 |
# Returns : ref to array of data
|
|
|
982 |
#
|
|
|
983 |
sub getDataFromRm
|
|
|
984 |
{
|
|
|
985 |
my ($name,$m_sqlstr, $options ) = @_;
|
|
|
986 |
my @row;
|
|
|
987 |
my $data;
|
|
|
988 |
|
|
|
989 |
if (ref $options ne 'HASH') {
|
|
|
990 |
$options = {};
|
|
|
991 |
}
|
|
|
992 |
|
|
|
993 |
if ($options->{sql}) {
|
|
|
994 |
$logger->logmsg("$name: $m_sqlstr")
|
|
|
995 |
}
|
|
|
996 |
my $sth = $RM_DB->prepare($m_sqlstr);
|
|
|
997 |
if ( defined($sth) )
|
|
|
998 |
{
|
|
|
999 |
if ( $sth->execute( ) ) {
|
|
|
1000 |
if ( $sth->rows ) {
|
|
|
1001 |
while ( @row = $sth->fetchrow_array ) {
|
|
|
1002 |
if ($options->{data}) {
|
|
|
1003 |
$logger->warn ("$name: @row");
|
|
|
1004 |
}
|
|
|
1005 |
#Debug0("$name: @row");
|
|
|
1006 |
push @{$data}, [@row];
|
|
|
1007 |
|
|
|
1008 |
last if $options->{oneRow};
|
|
|
1009 |
}
|
|
|
1010 |
}
|
|
|
1011 |
$sth->finish();
|
|
|
1012 |
} else {
|
|
|
1013 |
$logger->warn("Execute failure:$name: $m_sqlstr", $sth->errstr() );
|
|
|
1014 |
}
|
|
|
1015 |
} else {
|
|
|
1016 |
$logger->warn("Prepare failure:$name" );
|
|
|
1017 |
}
|
|
|
1018 |
|
|
|
1019 |
if (!$data && $options->{error}) {
|
|
|
1020 |
$logger->warn( $options->{error} );
|
|
|
1021 |
}
|
|
|
1022 |
|
|
|
1023 |
if ($data && $options->{oneRow}) {
|
|
|
1024 |
$data = $data->[0];
|
|
|
1025 |
}
|
|
|
1026 |
|
|
|
1027 |
if ($options->{dump}) {
|
|
|
1028 |
Utils::DebugDumpData("$name", $data);
|
|
|
1029 |
}
|
|
|
1030 |
return $data;
|
|
|
1031 |
}
|
|
|
1032 |
|
|
|
1033 |
#-------------------------------------------------------------------------------
|
|
|
1034 |
# Function : maintainTagList
|
|
|
1035 |
#
|
|
|
1036 |
# Description : Maintain a data structure for the maintenance of the
|
|
|
1037 |
# tags directory
|
|
|
1038 |
#
|
|
|
1039 |
# Inputs : None
|
|
|
1040 |
#
|
|
|
1041 |
# Returns : Nothing
|
|
|
1042 |
#
|
|
|
1043 |
sub maintainTagList
|
|
|
1044 |
{
|
|
|
1045 |
#
|
|
|
1046 |
# Time to perform the scan
|
|
|
1047 |
# Will do at startup and every time period there after
|
|
|
1048 |
#
|
|
|
1049 |
return unless ( $now > ($lastTagListUpdate + $conf->{tagListUpdate} ));
|
|
|
1050 |
$logger->verbose("maintainTagList");
|
|
|
1051 |
$lastTagListUpdate = $now;
|
|
|
1052 |
|
|
|
1053 |
#
|
|
|
1054 |
# Generate new configuration
|
|
|
1055 |
#
|
|
|
1056 |
my %config;
|
|
|
1057 |
$config{s3Sync} = 1; # Indicate that it may be special
|
|
|
1058 |
|
|
|
1059 |
%{$config{releases}} = map { $_ => 1 } keys %{$activeReleases};
|
|
|
1060 |
|
|
|
1061 |
#
|
|
|
1062 |
# Save data
|
|
|
1063 |
#
|
|
|
1064 |
my $dump = Data::Dumper->new([\%config], [qw(*config)]);
|
|
|
1065 |
#print $dump->Dump;
|
|
|
1066 |
#$dump->Reset;
|
|
|
1067 |
|
|
|
1068 |
#
|
|
|
1069 |
# Save config data
|
|
|
1070 |
#
|
|
|
1071 |
my $conf_file = catfile( $conf->{'tagdir'},'.config' );
|
|
|
1072 |
$logger->verbose3("maintainTagList: Writting $conf_file");
|
|
|
1073 |
|
|
|
1074 |
my $fh;
|
|
|
1075 |
open ( $fh, '>', $conf_file ) or $logger->err("Can't create $conf_file: $!");
|
|
|
1076 |
print $fh $dump->Dump;
|
|
|
1077 |
close $fh;
|
|
|
1078 |
}
|
|
|
1079 |
|
|
|
1080 |
#-------------------------------------------------------------------------------
|
|
|
1081 |
# Function : resetDailyStatistics
|
|
|
1082 |
#
|
|
|
1083 |
# Description : Called periodically to reset the daily statistics
|
|
|
1084 |
#
|
|
|
1085 |
# Inputs : $time - Current time
|
|
|
1086 |
#
|
|
|
1087 |
# Returns :
|
|
|
1088 |
#
|
|
|
1089 |
sub resetDailyStatistics
|
|
|
1090 |
{
|
|
|
1091 |
my ($time) = @_;
|
|
|
1092 |
|
|
|
1093 |
#
|
|
|
1094 |
# Detect a new day
|
|
|
1095 |
#
|
|
|
1096 |
my $today = (localtime($time))[7];
|
|
|
1097 |
if ($yday != $today)
|
|
|
1098 |
{
|
|
|
1099 |
$yday = $today;
|
|
|
1100 |
$logger->logmsg('Resetting daily statistics' );
|
|
|
1101 |
|
|
|
1102 |
# Note: Must match @recoverTags in readStatistics
|
|
|
1103 |
$statistics{dayStart} = $time;
|
|
|
1104 |
$statistics{txCount} = 0;
|
| 7397 |
dpurdie |
1105 |
$statistics{txBytes} = 0;
|
| 7396 |
dpurdie |
1106 |
$statistics{delCount} = 0;
|
|
|
1107 |
$statistics{linkErrors} = 0;
|
|
|
1108 |
}
|
|
|
1109 |
}
|
|
|
1110 |
|
|
|
1111 |
#-------------------------------------------------------------------------------
|
|
|
1112 |
# Function : readStatistics
|
|
|
1113 |
#
|
|
|
1114 |
# Description : Read in the last set of stats
|
|
|
1115 |
# Used after a restart to recover daily statistics
|
|
|
1116 |
#
|
|
|
1117 |
# Inputs :
|
|
|
1118 |
#
|
|
|
1119 |
# Returns :
|
|
|
1120 |
#
|
|
|
1121 |
sub readStatistics
|
|
|
1122 |
{
|
| 7397 |
dpurdie |
1123 |
my @recoverTags = qw(dayStart txCount txBytes delCount linkErrors);
|
| 7396 |
dpurdie |
1124 |
|
|
|
1125 |
if ($conf->{'statsfile'} and -f $conf->{'statsfile'})
|
|
|
1126 |
{
|
|
|
1127 |
if (open my $fh, $conf->{'statsfile'})
|
|
|
1128 |
{
|
|
|
1129 |
while (<$fh>)
|
|
|
1130 |
{
|
|
|
1131 |
m~(.*):(.*)~;
|
|
|
1132 |
if ( grep( /^$1$/, @recoverTags ) )
|
|
|
1133 |
{
|
|
|
1134 |
$statistics{$1} = $2;
|
|
|
1135 |
$logger->verbose("readStatistics $1, $2");
|
|
|
1136 |
}
|
|
|
1137 |
}
|
|
|
1138 |
close $fh;
|
|
|
1139 |
$yday = (localtime($statistics{dayStart}))[7];
|
|
|
1140 |
}
|
|
|
1141 |
}
|
|
|
1142 |
}
|
|
|
1143 |
|
|
|
1144 |
|
|
|
1145 |
#-------------------------------------------------------------------------------
|
|
|
1146 |
# Function : periodicStatistics
|
|
|
1147 |
#
|
|
|
1148 |
# Description : Called on a regular basis to write out statistics
|
|
|
1149 |
# Used to feed information into Nagios
|
|
|
1150 |
#
|
|
|
1151 |
# This function is called via an alarm and may be outside the normal
|
|
|
1152 |
# processing loop. Don't make assumptions on the value of $now
|
|
|
1153 |
#
|
|
|
1154 |
# Inputs :
|
|
|
1155 |
#
|
|
|
1156 |
# Returns :
|
|
|
1157 |
#
|
|
|
1158 |
sub periodicStatistics
|
|
|
1159 |
{
|
|
|
1160 |
#
|
|
|
1161 |
# A few local stats
|
|
|
1162 |
#
|
|
|
1163 |
$statistics{SeqNum}++;
|
|
|
1164 |
$statistics{timeStamp} = time();
|
|
|
1165 |
$statistics{upTime} = $statistics{timeStamp} - $startTime;
|
| 7397 |
dpurdie |
1166 |
$statistics{wedged} = $wedgedCount++ > 30 ? 1 : 0;
|
| 7396 |
dpurdie |
1167 |
|
| 7406 |
dpurdie |
1168 |
if ( $statistics{wedged}) {
|
|
|
1169 |
$statistics{state} = 'Wedged';
|
|
|
1170 |
} elsif(!$linkUp){
|
|
|
1171 |
$statistics{state} = 'S3 Bucket Read Error';
|
|
|
1172 |
} else {
|
|
|
1173 |
$statistics{state} = 'OK';
|
|
|
1174 |
}
|
|
|
1175 |
|
| 7396 |
dpurdie |
1176 |
# Reset daily accumulations - on first use each day
|
|
|
1177 |
resetDailyStatistics($statistics{timeStamp});
|
|
|
1178 |
|
|
|
1179 |
#
|
|
|
1180 |
# Write statistics to a file
|
|
|
1181 |
# Write to a tmp file, then rename.
|
|
|
1182 |
# Attempt to make the operation atomic - so that the file consumer
|
|
|
1183 |
# doesn't get a badly formed file.
|
|
|
1184 |
#
|
|
|
1185 |
if ($conf->{'statsfiletmp'})
|
|
|
1186 |
{
|
|
|
1187 |
my $fh;
|
|
|
1188 |
unless (open ($fh, '>', $conf->{'statsfiletmp'}))
|
|
|
1189 |
{
|
|
|
1190 |
$fh = undef;
|
|
|
1191 |
$logger->warn("Cannot create temp stats file: $!");
|
|
|
1192 |
}
|
|
|
1193 |
else
|
|
|
1194 |
{
|
|
|
1195 |
foreach my $key ( sort { lc($a) cmp lc($b) } keys %statistics)
|
|
|
1196 |
{
|
|
|
1197 |
print $fh $key . ':' . $statistics{$key} . "\n";
|
|
|
1198 |
$logger->verbose2('Statistics:'. $key . ':' . $statistics{$key});
|
|
|
1199 |
}
|
|
|
1200 |
close $fh;
|
|
|
1201 |
|
|
|
1202 |
# Rename temp to real file
|
|
|
1203 |
rename $conf->{'statsfiletmp'}, $conf->{'statsfile'} ;
|
|
|
1204 |
}
|
|
|
1205 |
}
|
|
|
1206 |
}
|
|
|
1207 |
|
|
|
1208 |
#-------------------------------------------------------------------------------
|
|
|
1209 |
# Function : sighandlers
|
|
|
1210 |
#
|
|
|
1211 |
# Description : Install signal handlers
|
|
|
1212 |
#
|
| 7423 |
dpurdie |
1213 |
# Inputs : Uses gobals
|
| 7396 |
dpurdie |
1214 |
#
|
|
|
1215 |
# Returns : Nothing
|
|
|
1216 |
#
|
|
|
1217 |
sub sighandlers
|
|
|
1218 |
{
|
|
|
1219 |
$SIG{TERM} = sub {
|
|
|
1220 |
# On shutdown
|
|
|
1221 |
$logger->logmsg('Received SIGTERM. Shutting down....' );
|
|
|
1222 |
unlink $conf->{'pidfile'} if (-f $conf->{'pidfile'});
|
|
|
1223 |
exit 0;
|
|
|
1224 |
};
|
|
|
1225 |
|
|
|
1226 |
$SIG{HUP} = sub {
|
|
|
1227 |
# On logrotate
|
|
|
1228 |
$logger->logmsg('Received SIGHUP.');
|
|
|
1229 |
$logger->rotatelog();
|
|
|
1230 |
};
|
|
|
1231 |
|
|
|
1232 |
$SIG{USR1} = sub {
|
|
|
1233 |
# On Force Rescans
|
|
|
1234 |
$logger->logmsg('Received SIGUSR1.');
|
|
|
1235 |
$lastTagListUpdate = 0;
|
|
|
1236 |
$lastS3Refresh = 0;
|
|
|
1237 |
};
|
|
|
1238 |
|
|
|
1239 |
alarm 60;
|
|
|
1240 |
$SIG{ALRM} = sub {
|
|
|
1241 |
# On Dump Statistics
|
|
|
1242 |
$logger->verbose2('Received SIGUSR2.');
|
|
|
1243 |
periodicStatistics();
|
|
|
1244 |
alarm 60;
|
|
|
1245 |
};
|
|
|
1246 |
|
|
|
1247 |
$SIG{__WARN__} = sub { $logger->warn("@_") };
|
|
|
1248 |
$SIG{__DIE__} = sub { $logger->err("@_") };
|
|
|
1249 |
}
|
|
|
1250 |
|
|
|
1251 |
|
|
|
1252 |
#-------------------------------------------------------------------------------
|
|
|
1253 |
# Function : Error, Verbose, Warning
|
|
|
1254 |
#
|
|
|
1255 |
# Description : Support for JatsRmApi
|
|
|
1256 |
#
|
|
|
1257 |
# Inputs : Message
|
|
|
1258 |
#
|
|
|
1259 |
# Returns : Nothing
|
|
|
1260 |
#
|
|
|
1261 |
sub Error
|
|
|
1262 |
{
|
|
|
1263 |
$logger->err("@_");
|
|
|
1264 |
}
|
|
|
1265 |
|
|
|
1266 |
sub Verbose
|
|
|
1267 |
{
|
|
|
1268 |
$logger->verbose2("@_");
|
|
|
1269 |
}
|
|
|
1270 |
|
|
|
1271 |
sub Warning
|
|
|
1272 |
{
|
|
|
1273 |
$logger->warn("@_");
|
|
|
1274 |
}
|
|
|
1275 |
|
|
|
1276 |
|