Subversion Repositories DevTools

Rev

Rev 7397 | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 7397 Rev 7406
Line 86... Line 86...
86 
    txBytes => 0,                       # Bytes Transferred
 86 
    txBytes => 0,                       # Bytes Transferred
87 
    delCount => 0,                      # Packages marked for deletion
 87 
    delCount => 0,                      # Packages marked for deletion
88 
    linkErrors => 0,                    # Transfer (S3) errors
 88 
    linkErrors => 0,                    # Transfer (S3) errors
89 
                                        #
89 
                                        #
90 
                                        # Per Cycle Data - Calculated each processing Cycle
 90 
                                        # Per Cycle Data - Calculated each processing Cycle
- 
 
 91 
    total => 0,                         # Number targets
91 
);
 92 
);
92 
 
 93 
 
93 
#
 94 
#
94 
#   Describe configuration parameters
 95 
#   Describe configuration parameters
95 
#
 96 
#
Line 114... Line 115...
114 
    'forcedirscan'    => {'default'   => 100    , 'fmt' => 'period'},
 115 
    'forcedirscan'    => {'default'   => 100    , 'fmt' => 'period'},
115 
    'forces3update'   => {'default'   => '30m'  , 'fmt' => 'period'},
 116 
    'forces3update'   => {'default'   => '30m'  , 'fmt' => 'period'},
116 
    'tagListUpdate'   => {'default'   => '1h'   , 'fmt' => 'period'},
 117 
    'tagListUpdate'   => {'default'   => '1h'   , 'fmt' => 'period'},
117 
    'S3Bucket'        => {'mandatory' => 1      , 'fmt' => 'text'},
 118 
    'S3Bucket'        => {'mandatory' => 1      , 'fmt' => 'text'},
118 
    'S3Profile'       => {'mandatory' => 1      , 'fmt' => 'text'},
 119 
    'S3Profile'       => {'mandatory' => 1      , 'fmt' => 'text'},
- 
 
 120 
    'S3Region'        => {'default' => undef    , 'fmt' => 'text'},
119 
 
 121 
 
120 
);
 122 
);
121 
 
 123 
 
122 
 
 124 
 
123 
#
 125 
#
Line 246... Line 248...
246 
        #   When config is read force some actions
 248 
        #   When config is read force some actions
247 
        #       - Force tagList to be created
 249 
        #       - Force tagList to be created
248 
        #       - Force refresh from S3
 250 
        #       - Force refresh from S3
249 
        $lastTagListUpdate = 0;
 251 
        $lastTagListUpdate = 0;
250 
        $lastS3Refresh = 0;
 252 
        $lastS3Refresh = 0;
251 
    }
 - 
 
252 
 
 - 
 
253 
    #
 253 
        $rv = 1;
254 
    #   When config is read force some actions
 - 
 
255 
 
 254 
 
- 
 
 255 
        #
- 
 
 256 
        #   When config is read force some actions
256 
#Utils::DebugDumpData ("Config", $conf);
 257 
#Utils::DebugDumpData ("Config", $conf);
257 
 
 258 
 
258 
    $logger->warn("All Transfers disabled") if ( $conf->{'noTransfers'} );
 259 
        $logger->warn("All Transfers disabled") if ( $conf->{'noTransfers'} );
259 
    $logger->warn("S3Sync is inactive") unless ( $conf->{'active'} );
 260 
        $logger->warn("S3Sync is inactive") unless ( $conf->{'active'} );
- 
 
 261 
    }
- 
 
 262 
 
260 
    return $rv;
 263 
    return $rv;
261 
}
 264 
}
262 
 
 265 
 
263 
#-------------------------------------------------------------------------------
 266 
#-------------------------------------------------------------------------------
264 
# Function        : refreshS3Info
267 
# Function        : refreshS3Info
Line 381... Line 384...
381 
        my $m_sqlstr = "SELECT rt.rtag_id,rm.seqnum, rt.s3sync, rt.official, rm.timestamp " .
384 
        my $m_sqlstr = "SELECT rt.rtag_id,rm.seqnum, rt.s3sync, rt.official, rm.timestamp " .
382 
                       "FROM RELEASE_MANAGER.release_tags rt, RELEASE_MANAGER.release_modified rm " .
 385 
                       "FROM RELEASE_MANAGER.release_tags rt, RELEASE_MANAGER.release_modified rm " .
383 
                       "WHERE rt.s3sync = 'Y' AND rm.rtag_id = rt.rtag_id AND rt.official in ('N', 'R', 'C')";
 386 
                       "WHERE rt.s3sync = 'Y' AND rm.rtag_id = rt.rtag_id AND rt.official in ('N', 'R', 'C')";
384 
 
 387 
 
385 
        my $curData = getDataFromRm ('monitorRequests', $m_sqlstr, {data => 0} );
 388 
        my $curData = getDataFromRm ('monitorRequests', $m_sqlstr, {data => 0} );
- 
 
 389 
        $statistics{total} = scalar @{$curData};
- 
 
 390 
 
386 
        foreach my $entry (@{$curData}) {
 391 
        foreach my $entry (@{$curData}) {
387 
            my ($rtag_id, $seqnum) = @{$entry};
 392 
            my ($rtag_id, $seqnum) = @{$entry};
- 
 
 393 
            $logger->verbose3("rtagid: $rtag_id, seqnumm: $seqnum");
388 
 
 394 
 
389 
            if (! exists $activeReleases->{$rtag_id} || ! exists $activeReleases->{$rtag_id}{s3}  ) {
 395 
            if (! exists $activeReleases->{$rtag_id} || ! exists $activeReleases->{$rtag_id}{s3}  ) {
390 
                $logger->logmsg("New Release Detected. rtag_id: $rtag_id, seq:$seqnum");
 396 
                $logger->logmsg("New Release Detected. rtag_id: $rtag_id, seq:$seqnum");
391 
                processChangedRelease($rtag_id, $seqnum);
 397 
                processChangedRelease($rtag_id, $seqnum);
392 
                $lastTagListUpdate = 0;
 398 
                $lastTagListUpdate = 0;
Line 427... Line 433...
427 
# Returns         : Updates global structure ($activeReleases)
433 
# Returns         : Updates global structure ($activeReleases)
428 
# Returns         : 0 - Gross error ( Bucket access)
434 
# Returns         : 0 - Gross error ( Bucket access)
429 
#
 435 
#
430 
sub examineS3Bucket
 436 
sub examineS3Bucket
431 
{
 437 
{
- 
 
 438 
    my $bucket;
- 
 
 439 
    my $prefix;
- 
 
 440 
 
432 
    #
 441 
    #
433 
    #   Remove data collected from s3
 442 
    #   Remove data collected from s3
434 
    #
 443 
    #
435 
    foreach my $rtag_id (keys %{$activeReleases}) {
 444 
    foreach my $rtag_id (keys %{$activeReleases}) {
436 
        delete $activeReleases->{$rtag_id}{s3}  ;
 445 
        delete $activeReleases->{$rtag_id}{s3}  ;
437 
    }
 446 
    }
438 
 
 447 
 
439 
    $conf->{'S3Bucket'} =~ m~(.*?)/(.*)~;
 448 
    if ($conf->{'S3Bucket'} =~ m~(.*?)/(.*)~) {
440 
    my $bucket = $1;
 449 
        $bucket = $1;
441 
    my $prefix = $2;
 450 
        $prefix = $2;
- 
 
 451 
    } else {
- 
 
 452 
        $bucket = $conf->{'S3Bucket'};
- 
 
 453 
    }
- 
 
 454 
 
- 
 
 455 
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
- 
 
 456 
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
- 
 
 457 
    $s3_cmd .= " s3api list-objects --bucket $bucket";
- 
 
 458 
    $s3_cmd .= " --prefix '$prefix'" if (defined $prefix);
442 
 
 459 
 
443 
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} s3api list-objects --bucket $bucket --prefix $prefix";
 - 
 
444 
    $logger->verbose2("examineS3Bucket:s3_cmd:$s3_cmd");
 460 
    $logger->verbose2("examineS3Bucket:s3_cmd:$s3_cmd");
445 
 
 461 
 
446 
    my $ph;
 462 
    my $ph;
447 
    my $jsontxt = "";
 463 
    my $jsontxt = "";
448 
    open ($ph, "$s3_cmd |");
 464 
    open ($ph, "$s3_cmd |");
Line 461... Line 477...
461 
    if ($jsontxt) {
 477 
    if ($jsontxt) {
462 
        my $json = from_json ($jsontxt);
 478 
        my $json = from_json ($jsontxt);
463 
        #Utils::DebugDumpData("JSON",$json->{'Contents'});
 479 
        #Utils::DebugDumpData("JSON",$json->{'Contents'});
464 
        foreach my $item ( @{$json->{'Contents'}})
 480 
        foreach my $item ( @{$json->{'Contents'}})
465 
        {
 481 
        {
466 
            if ($item->{Key} =~ m~/Release-(.*)\.zip$~ ) {
 482 
            if ($item->{Key} =~ m~(?:^|/)Release-(.*)\.zip$~ ) {
467 
 
 483 
 
468 
                my $rtag_id = $1;
 484 
                my $rtag_id = $1;
469 
                my $metaData = gets3ObjectMetaData($item->{Key});
 485 
                my $metaData = gets3ObjectMetaData($item->{Key});
470 
 
 486 
 
471 
                #
 487 
                #
Line 507... Line 523...
507 
#
 523 
#
508 
 
 524 
 
509 
sub gets3ObjectMetaData
 525 
sub gets3ObjectMetaData
510 
{
 526 
{
511 
    my ($key) = @_;
 527 
    my ($key) = @_;
- 
 
 528 
    my $bucket;
- 
 
 529 
    my $prefix;
- 
 
 530 
 
- 
 
 531 
    if ($conf->{'S3Bucket'} =~ m~(.*?)/(.*)~) {
- 
 
 532 
        $bucket = $1;
- 
 
 533 
        $prefix = $2;
- 
 
 534 
    } else {
- 
 
 535 
        $bucket = $conf->{'S3Bucket'};
- 
 
 536 
        $prefix = '';
- 
 
 537 
    }
512 
 
 538 
 
513 
    $conf->{'S3Bucket'} =~ m~(.*?)/(.*)~;
 539 
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
514 
    my $bucket = $1;
 540 
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
515 
    my $prefix = $2;
 541 
    $s3_cmd .= " s3api head-object --bucket $bucket --key $key";
516 
 
 542 
 
517 
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} s3api head-object --bucket $bucket --key $key";
 - 
 
518 
    $logger->verbose2("gets3ObjectMetaData:s3_cmd:$s3_cmd");
 543 
    $logger->verbose2("gets3ObjectMetaData:s3_cmd:$s3_cmd");
519 
 
 544 
 
520 
    my $ph;
 545 
    my $ph;
521 
    my $jsontxt = "";
 546 
    my $jsontxt = "";
522 
    open ($ph, "$s3_cmd |");
 547 
    open ($ph, "$s3_cmd |");
Line 554... Line 579...
554 
    my $rv = 0;
 579 
    my $rv = 0;
555 
 
 580 
 
556 
    #   Create the process pipe to delete the package
 581 
    #   Create the process pipe to delete the package
557 
 
 582 
 
558 
    my $targetPath = generateBucketZipName($rtag_id);
 583 
    my $targetPath = generateBucketZipName($rtag_id);
- 
 
 584 
 
559 
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} s3 rm s3://$targetPath";
 585 
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
- 
 
 586 
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
- 
 
 587 
    $s3_cmd .= " s3 rm s3://$targetPath";
- 
 
 588 
 
560 
    $logger->logmsg("removeDeadRelease:$targetPath");
 589 
    $logger->logmsg("removeDeadRelease:$targetPath");
561 
    $logger->verbose2("removeDeadRelease:s3_cmd:$s3_cmd");
 590 
    $logger->verbose2("removeDeadRelease:s3_cmd:$s3_cmd");
562 
 
 591 
 
563 
    my $ph;
 592 
    my $ph;
564 
    open ($ph, "$s3_cmd |");
 593 
    open ($ph, "$s3_cmd |");
Line 745... Line 774...
745 
    #   Note: Ive seen problem with this when used from Perth to AWS (Sydney)
 774 
    #   Note: Ive seen problem with this when used from Perth to AWS (Sydney)
746 
    #         If this is an issue use curl - see the savePkgToS3.sh for an implementation
 775 
    #         If this is an issue use curl - see the savePkgToS3.sh for an implementation
747 
    #
 776 
    #
748 
    $startTime = time;
 777 
    $startTime = time;
749 
    my $targetPath = generateBucketZipName($rtag_id);
 778 
    my $targetPath = generateBucketZipName($rtag_id);
- 
 
 779 
 
- 
 
 780 
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
- 
 
 781 
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
750 
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} s3 cp $zipFile s3://$targetPath --metadata releaseseq=$seqnum,md5=$digest,depsig=$depsig";
 782 
    $s3_cmd .= " s3 cp $zipFile s3://$targetPath --metadata releaseseq=$seqnum,md5=$digest,depsig=$depsig";
- 
 
 783 
 
751 
    $logger->logmsg("transferPackage:$targetPath");
 784 
    $logger->logmsg("transferPackage:$targetPath");
752 
    $logger->verbose2("transferPackage:s3_cmd:$s3_cmd");
 785 
    $logger->verbose2("transferPackage:s3_cmd:$s3_cmd");
753 
 
 786 
 
754 
    my $cmdRv;
 787 
    my $cmdRv;
755 
    unless ($conf->{'noTransfers'}) {
 788 
    unless ($conf->{'noTransfers'}) {
Line 1043... Line 1076...
1043 
    #
 1076 
    #
1044 
    $statistics{SeqNum}++;
 1077 
    $statistics{SeqNum}++;
1045 
    $statistics{timeStamp} = time();
 1078 
    $statistics{timeStamp} = time();
1046 
    $statistics{upTime} = $statistics{timeStamp} - $startTime;
 1079 
    $statistics{upTime} = $statistics{timeStamp} - $startTime;
1047 
    $statistics{wedged} = $wedgedCount++ > 30  ? 1 : 0;
 1080 
    $statistics{wedged} = $wedgedCount++ > 30  ? 1 : 0;
- 
 
 1081 
 
- 
 
 1082 
    if ( $statistics{wedged}) {
1048 
    $statistics{state} = $statistics{wedged} ? 'Wedged' : $statistics{state};
1083 
         $statistics{state} = 'Wedged';
- 
 
 1084 
    } elsif(!$linkUp){
1049 
    $statistics{state} = $linkUp ? $statistics{state} : 'S3 Link Errors' ;
1085 
        $statistics{state} = 'S3 Bucket Read Error';
- 
 
 1086 
    } else {
- 
 
 1087 
        $statistics{state} = 'OK';
- 
 
 1088 
    }
1050 
 
 1089 
 
1051 
    #   Reset daily accumulations - on first use each day
 1090 
    #   Reset daily accumulations - on first use each day
1052 
    resetDailyStatistics($statistics{timeStamp});
 1091 
    resetDailyStatistics($statistics{timeStamp});
1053
1092
1054 
    #
 1093 
    #