Subversion Repositories DevTools

Rev

Rev 7397 | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 7397 Rev 7406
Line 86... Line 86...
86
    txBytes => 0,                       # Bytes Transferred
86
    txBytes => 0,                       # Bytes Transferred
87
    delCount => 0,                      # Packages marked for deletion
87
    delCount => 0,                      # Packages marked for deletion
88
    linkErrors => 0,                    # Transfer (S3) errors
88
    linkErrors => 0,                    # Transfer (S3) errors
89
                                        # 
89
                                        # 
90
                                        # Per Cycle Data - Calculated each processing Cycle
90
                                        # Per Cycle Data - Calculated each processing Cycle
-
 
91
    total => 0,                         # Number targets
91
);
92
);
92
 
93
 
93
#
94
#
94
#   Describe configuration parameters
95
#   Describe configuration parameters
95
#
96
#
Line 114... Line 115...
114
    'forcedirscan'    => {'default'   => 100    , 'fmt' => 'period'},
115
    'forcedirscan'    => {'default'   => 100    , 'fmt' => 'period'},
115
    'forces3update'   => {'default'   => '30m'  , 'fmt' => 'period'},
116
    'forces3update'   => {'default'   => '30m'  , 'fmt' => 'period'},
116
    'tagListUpdate'   => {'default'   => '1h'   , 'fmt' => 'period'},
117
    'tagListUpdate'   => {'default'   => '1h'   , 'fmt' => 'period'},
117
    'S3Bucket'        => {'mandatory' => 1      , 'fmt' => 'text'},
118
    'S3Bucket'        => {'mandatory' => 1      , 'fmt' => 'text'},
118
    'S3Profile'       => {'mandatory' => 1      , 'fmt' => 'text'},
119
    'S3Profile'       => {'mandatory' => 1      , 'fmt' => 'text'},
-
 
120
    'S3Region'        => {'default' => undef    , 'fmt' => 'text'},
119
 
121
 
120
);
122
);
121
 
123
 
122
 
124
 
123
#
125
#
Line 246... Line 248...
246
        #   When config is read force some actions
248
        #   When config is read force some actions
247
        #       - Force tagList to be created
249
        #       - Force tagList to be created
248
        #       - Force refresh from S3
250
        #       - Force refresh from S3
249
        $lastTagListUpdate = 0;
251
        $lastTagListUpdate = 0;
250
        $lastS3Refresh = 0;
252
        $lastS3Refresh = 0;
251
    }
-
 
252
 
-
 
253
    #
253
        $rv = 1;
254
    #   When config is read force some actions
-
 
255
 
254
 
-
 
255
        #
-
 
256
        #   When config is read force some actions
256
#Utils::DebugDumpData ("Config", $conf);
257
#Utils::DebugDumpData ("Config", $conf);
257
 
258
 
258
    $logger->warn("All Transfers disabled") if ( $conf->{'noTransfers'} );
259
        $logger->warn("All Transfers disabled") if ( $conf->{'noTransfers'} );
259
    $logger->warn("S3Sync is inactive") unless ( $conf->{'active'} );
260
        $logger->warn("S3Sync is inactive") unless ( $conf->{'active'} );
-
 
261
    }
-
 
262
 
260
    return $rv;
263
    return $rv;
261
}
264
}
262
 
265
 
263
#-------------------------------------------------------------------------------
266
#-------------------------------------------------------------------------------
264
# Function        : refreshS3Info 
267
# Function        : refreshS3Info 
Line 381... Line 384...
381
        my $m_sqlstr = "SELECT rt.rtag_id,rm.seqnum, rt.s3sync, rt.official, rm.timestamp " . 
384
        my $m_sqlstr = "SELECT rt.rtag_id,rm.seqnum, rt.s3sync, rt.official, rm.timestamp " . 
382
                       "FROM RELEASE_MANAGER.release_tags rt, RELEASE_MANAGER.release_modified rm " .
385
                       "FROM RELEASE_MANAGER.release_tags rt, RELEASE_MANAGER.release_modified rm " .
383
                       "WHERE rt.s3sync = 'Y' AND rm.rtag_id = rt.rtag_id AND rt.official in ('N', 'R', 'C')";
386
                       "WHERE rt.s3sync = 'Y' AND rm.rtag_id = rt.rtag_id AND rt.official in ('N', 'R', 'C')";
384
 
387
 
385
        my $curData = getDataFromRm ('monitorRequests', $m_sqlstr, {data => 0} );
388
        my $curData = getDataFromRm ('monitorRequests', $m_sqlstr, {data => 0} );
-
 
389
        $statistics{total} = scalar @{$curData};
-
 
390
 
386
        foreach my $entry (@{$curData}) {
391
        foreach my $entry (@{$curData}) {
387
            my ($rtag_id, $seqnum) = @{$entry};
392
            my ($rtag_id, $seqnum) = @{$entry};
-
 
393
            $logger->verbose3("rtagid: $rtag_id, seqnumm: $seqnum");
388
 
394
 
389
            if (! exists $activeReleases->{$rtag_id} || ! exists $activeReleases->{$rtag_id}{s3}  ) {
395
            if (! exists $activeReleases->{$rtag_id} || ! exists $activeReleases->{$rtag_id}{s3}  ) {
390
                $logger->logmsg("New Release Detected. rtag_id: $rtag_id, seq:$seqnum");
396
                $logger->logmsg("New Release Detected. rtag_id: $rtag_id, seq:$seqnum");
391
                processChangedRelease($rtag_id, $seqnum);
397
                processChangedRelease($rtag_id, $seqnum);
392
                $lastTagListUpdate = 0;
398
                $lastTagListUpdate = 0;
Line 427... Line 433...
427
# Returns         : Updates global structure ($activeReleases) 
433
# Returns         : Updates global structure ($activeReleases) 
428
# Returns         : 0 - Gross error ( Bucket access) 
434
# Returns         : 0 - Gross error ( Bucket access) 
429
#
435
#
430
sub examineS3Bucket
436
sub examineS3Bucket
431
{
437
{
-
 
438
    my $bucket;
-
 
439
    my $prefix;
-
 
440
 
432
    #
441
    #
433
    #   Remove data collected from s3
442
    #   Remove data collected from s3
434
    #
443
    #
435
    foreach my $rtag_id (keys %{$activeReleases}) {
444
    foreach my $rtag_id (keys %{$activeReleases}) {
436
        delete $activeReleases->{$rtag_id}{s3}  ;
445
        delete $activeReleases->{$rtag_id}{s3}  ;
437
    }
446
    }
438
 
447
 
439
    $conf->{'S3Bucket'} =~ m~(.*?)/(.*)~;
448
    if ($conf->{'S3Bucket'} =~ m~(.*?)/(.*)~) {
440
    my $bucket = $1;
449
        $bucket = $1;
441
    my $prefix = $2;
450
        $prefix = $2;
-
 
451
    } else {
-
 
452
        $bucket = $conf->{'S3Bucket'};
-
 
453
    }
-
 
454
 
-
 
455
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
-
 
456
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
-
 
457
    $s3_cmd .= " s3api list-objects --bucket $bucket";
-
 
458
    $s3_cmd .= " --prefix '$prefix'" if (defined $prefix);
442
 
459
 
443
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} s3api list-objects --bucket $bucket --prefix $prefix";
-
 
444
    $logger->verbose2("examineS3Bucket:s3_cmd:$s3_cmd");
460
    $logger->verbose2("examineS3Bucket:s3_cmd:$s3_cmd");
445
 
461
 
446
    my $ph;
462
    my $ph;
447
    my $jsontxt = "";
463
    my $jsontxt = "";
448
    open ($ph, "$s3_cmd |");
464
    open ($ph, "$s3_cmd |");
Line 461... Line 477...
461
    if ($jsontxt) {
477
    if ($jsontxt) {
462
        my $json = from_json ($jsontxt);
478
        my $json = from_json ($jsontxt);
463
        #Utils::DebugDumpData("JSON",$json->{'Contents'});
479
        #Utils::DebugDumpData("JSON",$json->{'Contents'});
464
        foreach my $item ( @{$json->{'Contents'}})
480
        foreach my $item ( @{$json->{'Contents'}})
465
        {
481
        {
466
            if ($item->{Key} =~ m~/Release-(.*)\.zip$~ ) {
482
            if ($item->{Key} =~ m~(?:^|/)Release-(.*)\.zip$~ ) {
467
 
483
 
468
                my $rtag_id = $1;
484
                my $rtag_id = $1;
469
                my $metaData = gets3ObjectMetaData($item->{Key});
485
                my $metaData = gets3ObjectMetaData($item->{Key});
470
 
486
 
471
                #
487
                #
Line 507... Line 523...
507
#
523
#
508
 
524
 
509
sub gets3ObjectMetaData
525
sub gets3ObjectMetaData
510
{
526
{
511
    my ($key) = @_;
527
    my ($key) = @_;
-
 
528
    my $bucket;
-
 
529
    my $prefix;
-
 
530
 
-
 
531
    if ($conf->{'S3Bucket'} =~ m~(.*?)/(.*)~) {
-
 
532
        $bucket = $1;
-
 
533
        $prefix = $2;
-
 
534
    } else {
-
 
535
        $bucket = $conf->{'S3Bucket'};
-
 
536
        $prefix = '';
-
 
537
    }
512
 
538
 
513
    $conf->{'S3Bucket'} =~ m~(.*?)/(.*)~;
539
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
514
    my $bucket = $1;
540
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
515
    my $prefix = $2;
541
    $s3_cmd .= " s3api head-object --bucket $bucket --key $key";
516
 
542
 
517
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} s3api head-object --bucket $bucket --key $key";
-
 
518
    $logger->verbose2("gets3ObjectMetaData:s3_cmd:$s3_cmd");
543
    $logger->verbose2("gets3ObjectMetaData:s3_cmd:$s3_cmd");
519
 
544
 
520
    my $ph;
545
    my $ph;
521
    my $jsontxt = "";
546
    my $jsontxt = "";
522
    open ($ph, "$s3_cmd |");
547
    open ($ph, "$s3_cmd |");
Line 554... Line 579...
554
    my $rv = 0;
579
    my $rv = 0;
555
 
580
 
556
    #   Create the process pipe to delete the package
581
    #   Create the process pipe to delete the package
557
 
582
 
558
    my $targetPath = generateBucketZipName($rtag_id);
583
    my $targetPath = generateBucketZipName($rtag_id);
-
 
584
 
559
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} s3 rm s3://$targetPath";
585
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
-
 
586
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
-
 
587
    $s3_cmd .= " s3 rm s3://$targetPath";
-
 
588
 
560
    $logger->logmsg("removeDeadRelease:$targetPath");
589
    $logger->logmsg("removeDeadRelease:$targetPath");
561
    $logger->verbose2("removeDeadRelease:s3_cmd:$s3_cmd");
590
    $logger->verbose2("removeDeadRelease:s3_cmd:$s3_cmd");
562
 
591
 
563
    my $ph;
592
    my $ph;
564
    open ($ph, "$s3_cmd |");
593
    open ($ph, "$s3_cmd |");
Line 745... Line 774...
745
    #   Note: Ive seen problem with this when used from Perth to AWS (Sydney)
774
    #   Note: Ive seen problem with this when used from Perth to AWS (Sydney)
746
    #         If this is an issue use curl - see the savePkgToS3.sh for an implementation
775
    #         If this is an issue use curl - see the savePkgToS3.sh for an implementation
747
    #
776
    #
748
    $startTime = time;
777
    $startTime = time;
749
    my $targetPath = generateBucketZipName($rtag_id);
778
    my $targetPath = generateBucketZipName($rtag_id);
-
 
779
 
-
 
780
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
-
 
781
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
750
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} s3 cp $zipFile s3://$targetPath --metadata releaseseq=$seqnum,md5=$digest,depsig=$depsig";
782
    $s3_cmd .= " s3 cp $zipFile s3://$targetPath --metadata releaseseq=$seqnum,md5=$digest,depsig=$depsig";
-
 
783
 
751
    $logger->logmsg("transferPackage:$targetPath");
784
    $logger->logmsg("transferPackage:$targetPath");
752
    $logger->verbose2("transferPackage:s3_cmd:$s3_cmd");
785
    $logger->verbose2("transferPackage:s3_cmd:$s3_cmd");
753
 
786
 
754
    my $cmdRv;
787
    my $cmdRv;
755
    unless ($conf->{'noTransfers'}) {
788
    unless ($conf->{'noTransfers'}) {
Line 1043... Line 1076...
1043
    #
1076
    #
1044
    $statistics{SeqNum}++;
1077
    $statistics{SeqNum}++;
1045
    $statistics{timeStamp} = time();
1078
    $statistics{timeStamp} = time();
1046
    $statistics{upTime} = $statistics{timeStamp} - $startTime;
1079
    $statistics{upTime} = $statistics{timeStamp} - $startTime;
1047
    $statistics{wedged} = $wedgedCount++ > 30  ? 1 : 0;
1080
    $statistics{wedged} = $wedgedCount++ > 30  ? 1 : 0;
-
 
1081
 
-
 
1082
    if ( $statistics{wedged}) {
1048
    $statistics{state} = $statistics{wedged} ? 'Wedged' : $statistics{state}; 
1083
         $statistics{state} = 'Wedged';
-
 
1084
    } elsif(!$linkUp){
1049
    $statistics{state} = $linkUp ? $statistics{state} : 'S3 Link Errors' ; 
1085
        $statistics{state} = 'S3 Bucket Read Error';
-
 
1086
    } else {
-
 
1087
        $statistics{state} = 'OK';
-
 
1088
    }
1050
 
1089
 
1051
    #   Reset daily accumulations - on first use each day
1090
    #   Reset daily accumulations - on first use each day
1052
    resetDailyStatistics($statistics{timeStamp});
1091
    resetDailyStatistics($statistics{timeStamp});
1053
    
1092
    
1054
    #
1093
    #