Subversion Repositories DevTools

Rev

Rev 7469 | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 7469 Rev 7485
Line 59... Line 59...
59
my $lastTagListUpdate = 0;
59
my $lastTagListUpdate = 0;
60
my $mtimeConfig = 0;
60
my $mtimeConfig = 0;
61
my $conf;
61
my $conf;
62
my $yday = -1;
62
my $yday = -1;
63
my $linkUp = 1;
63
my $linkUp = 1;
-
 
64
my $dbUp = 1;
64
my $RM_DB;
65
my $RM_DB;
65
my $activeReleases;
66
my $activeReleases;
-
 
67
my $RMerror = 0;                        # Error on last RM DB access
66
 
68
 
67
#
69
#
68
#   Contain statisics maintained while operating
70
#   Contain statisics maintained while operating
69
#       Can be dumped with a kill -USR2
71
#       Can be dumped with a kill -USR2
70
#       List here for documentation
72
#       List here for documentation
Line 83... Line 85...
83
                                        # The following are reset each day
85
                                        # The following are reset each day
84
    dayStart => 0,                      # DateTime when daily data was reset
86
    dayStart => 0,                      # DateTime when daily data was reset
85
    txCount => 0,                       # Packages Transferred
87
    txCount => 0,                       # Packages Transferred
86
    txBytes => 0,                       # Bytes Transferred
88
    txBytes => 0,                       # Bytes Transferred
87
    linkErrors => 0,                    # Transfer (S3) errors
89
    linkErrors => 0,                    # Transfer (S3) errors
-
 
90
    dbErrors => 0,                      # Database errors
88
                                        # 
91
                                        # 
89
                                        # Per Cycle Data - Calculated each processing Cycle
92
                                        # Per Cycle Data - Calculated each processing Cycle
90
    total => 0,                         # Number targets
93
    total => 0,                         # Number targets
91
);
94
);
92
 
95
 
Line 164... Line 167...
164
            maintainTagList();
167
            maintainTagList();
165
        }
168
        }
166
    }
169
    }
167
 
170
 
168
    $statistics{phase} = 'Sleep';
171
    $statistics{phase} = 'Sleep';
169
    sleep( $linkUp ? $conf->{'sleep'} : $conf->{'sleepLinkDown'} );
172
    sleep( ($linkUp && $dbUp) ? $conf->{'sleep'} : $conf->{'sleepLinkDown'} );
170
    reapChildren();
173
    reapChildren();
171
 
174
 
172
    #   If my PID file ceases to be, then exit the daemon
175
    #   If my PID file ceases to be, then exit the daemon
173
    #   Used to force daemon to restart
176
    #   Used to force daemon to restart
174
    #
177
    #
Line 380... Line 383...
380
        $ENV{GBE_RM_LOCATION} = $conf->{RM_LOCATION} if defined $conf->{RM_LOCATION};
383
        $ENV{GBE_RM_LOCATION} = $conf->{RM_LOCATION} if defined $conf->{RM_LOCATION};
381
        
384
        
382
        connectRM(\$RM_DB, $conf->{verbose} > 3);
385
        connectRM(\$RM_DB, $conf->{verbose} > 3);
383
 
386
 
384
        my $m_sqlstr = "SELECT rtag_id, parent_rtag_id, s3manifest, s3manifest_done " .
387
        my $m_sqlstr = "SELECT rtag_id, parent_rtag_id, s3manifest, s3manifest_done " .
385
                       "FROM release_tags rt " .
388
                       "FROM release_manager.release_tags rt " .
386
                       "WHERE rt.OFFICIAL = 'S' " .
389
                       "WHERE rt.OFFICIAL = 'S' " .
387
                       "      AND rt.S3MANIFEST = 'Y' " .
390
                       "      AND rt.S3MANIFEST = 'Y' " .
388
                       "      AND rt.S3MANIFEST_DONE != 'Y' " .
391
                       "      AND rt.S3MANIFEST_DONE != 'Y' " .
389
                       "ORDER BY rtag_id ";
392
                       "ORDER BY rtag_id ";
390
 
-
 
391
        my $curData = getDataFromRm ('monitorRequests', $m_sqlstr, {data => 0, sql => 1, dump => 1} );
-
 
392
        return unless defined $curData;
-
 
393
        $statistics{total} = scalar @{$curData};
-
 
394
 
-
 
395
        # Add new enties to the set to process
-
 
396
        foreach my $entry (@{$curData}) {
-
 
397
            my ($rtag_id, $parent_rtag_id) = @{$entry};
-
 
398
            unless ( exists $activeReleases->{$rtag_id}) {
-
 
399
                $logger->logmsg("New Release Detected. rtag_id: $rtag_id, parent_rtag_id:$parent_rtag_id");
-
 
400
                $activeReleases->{$rtag_id}{parent_rtag_id} = $parent_rtag_id;
-
 
401
                $activeReleases->{$rtag_id}{rtag_id} = $rtag_id;
-
 
402
                $activeReleases->{$rtag_id}{startTime} = $now;
-
 
403
            }
-
 
404
        }
-
 
405
 
-
 
406
        # Process all the entries
-
 
407
        my $stuckEntry = 0;
393
        my $stuckEntry = 0;
408
        foreach my $rtag_id ( sort keys %{$activeReleases} ) {
394
        my $curData = getDataFromRm ('monitorRequests', $m_sqlstr, {data => 0, sql => 0, dump => 0} );
-
 
395
        $dbUp = !$RMerror;
-
 
396
 
-
 
397
        if ($curData) {
409
            processSnapshot($rtag_id);
398
            $statistics{total} = scalar @{$curData};
410
 
399
 
411
            # Warn if stuck entry
400
            # Add new enties to the set to process
412
            # Repeat the warning on periodic basis
401
            foreach my $entry (@{$curData}) {
413
            if (exists $activeReleases->{$rtag_id}) {
402
                my ($rtag_id, $parent_rtag_id) = @{$entry};
414
                my $entry = $activeReleases->{$rtag_id};
403
                unless ( exists $activeReleases->{$rtag_id}) {
415
                 
404
                    $logger->logmsg("New Release Detected. rtag_id: $rtag_id, parent_rtag_id:$parent_rtag_id");
416
                if ($now - $activeReleases->{$rtag_id}{startTime} > $conf->{waitTime}) {
405
                    $activeReleases->{$rtag_id}{parent_rtag_id} = $parent_rtag_id;
417
                    $logger->warn("Max waitTime exceeded: rtag_id: $rtag_id, parent_rtag_id:$entry->{parent_rtag_id}");
406
                    $activeReleases->{$rtag_id}{rtag_id} = $rtag_id;
418
                    $activeReleases->{$rtag_id}{startTime} = $now;
407
                    $activeReleases->{$rtag_id}{startTime} = $now;
419
                    $entry->{isStuck} = 1;
408
                    $lastTagListUpdate = 0;
420
                }
409
                }
-
 
410
            }
421
 
411
 
-
 
412
            # Process all the entries
-
 
413
            foreach my $rtag_id ( sort keys %{$activeReleases} ) {
-
 
414
                processSnapshot($rtag_id);
-
 
415
 
-
 
416
                # Warn if stuck entry
-
 
417
                # Repeat the warning on periodic basis
-
 
418
                if (exists $activeReleases->{$rtag_id}) {
-
 
419
                    my $entry = $activeReleases->{$rtag_id};
-
 
420
                     
-
 
421
                    if ($now - $activeReleases->{$rtag_id}{startTime} > $conf->{waitTime}) {
-
 
422
                        $logger->warn("Max waitTime exceeded: rtag_id: $rtag_id, parent_rtag_id:$entry->{parent_rtag_id}");
-
 
423
                        $activeReleases->{$rtag_id}{startTime} = $now;
-
 
424
                        $entry->{isStuck} = 1;
-
 
425
                    }
-
 
426
 
422
                if ($entry->{isStuck}) {
427
                    if ($entry->{isStuck}) {
423
                    $stuckEntry++;
428
                        $stuckEntry++;
424
                }
429
                    }
425
 
430
 
-
 
431
                }
426
            }
432
            }
427
        }
433
        }
428
        $statistics{slowGen} = $stuckEntry; 
434
        $statistics{slowGen} = $stuckEntry; 
429
        $lastTagListUpdate = 0;
-
 
430
        disconnectRM(\$RM_DB);
435
        disconnectRM(\$RM_DB);
431
    }
436
    }
432
}
437
}
433
 
438
 
434
#-------------------------------------------------------------------------------
439
#-------------------------------------------------------------------------------
Line 503... Line 508...
503
 
508
 
504
        #   Looks like a new entry - get the packages flagged for the manifest
509
        #   Looks like a new entry - get the packages flagged for the manifest
505
        #   Determine all the package version that should go into the manifest
510
        #   Determine all the package version that should go into the manifest
506
        #
511
        #
507
        my $m_sqlstr = "SELECT rc.pv_id, p.PKG_NAME, pv.PKG_VERSION, pv.v_ext, 0" .
512
        my $m_sqlstr = "SELECT rc.pv_id, p.PKG_NAME, pv.PKG_VERSION, pv.v_ext, 0" .
508
                        " FROM RELEASE_CONTENT rc, PACKAGE_VERSIONS pv, PACKAGES p " .
513
                        " FROM release_manager.RELEASE_CONTENT rc, release_manager.PACKAGE_VERSIONS pv, release_manager.PACKAGES p " .
509
                        " WHERE rc.RTAG_ID = $rtagId " .
514
                        " WHERE rc.RTAG_ID = $rtagId " .
510
                        "  AND rc.IN_MANIFEST = 'Y' " .
515
                        "  AND rc.IN_MANIFEST = 'Y' " .
511
                        "  AND rc.pv_id = pv.pv_id " .
516
                        "  AND rc.pv_id = pv.pv_id " .
512
                        "  AND p.pkg_id = pv.pkg_id";
517
                        "  AND p.pkg_id = pv.pkg_id";
513
 
518
 
Line 559... Line 564...
559
        my $depEntry;
564
        my $depEntry;
560
        $depEntry->{package} = $entry->[1];
565
        $depEntry->{package} = $entry->[1];
561
        $depEntry->{version} = $entry->[2];
566
        $depEntry->{version} = $entry->[2];
562
        $depEntry->{alias} = $entry->[1] . $entry->[3];
567
        $depEntry->{alias} = $entry->[1] . $entry->[3];
563
        $depEntry->{pv_id} = $entry->[0];
568
        $depEntry->{pv_id} = $entry->[0];
564
        $depEntry->{stored} = $entry->[1] . '__' . $entry->[2] . 'tgz';
569
        $depEntry->{stored} = $entry->[1] . '__' . $entry->[2] . '.tgz';
565
        push @{$manifest->{deployed}}, $depEntry;
570
        push @{$manifest->{deployed}}, $depEntry;
566
    }
571
    }
567
 
572
 
568
    my $jsonText = to_json( $manifest, { ascii => 1, pretty => 1, canonical => 1 });
573
    my $jsonText = to_json( $manifest, { ascii => 1, pretty => 1, canonical => 1 });
569
 
574
 
Line 591... Line 596...
591
    $startTime = time;
596
    $startTime = time;
592
    my $targetPath = catdir ($conf->{'S3Bucket'}, $targetName );
597
    my $targetPath = catdir ($conf->{'S3Bucket'}, $targetName );
593
 
598
 
594
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
599
    my $s3_cmd = "aws --profile $conf->{'S3Profile'} --output json";
595
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
600
    $s3_cmd .= " --region $conf->{'S3Region'}" if (defined $conf->{'S3Region'});
596
    $s3_cmd .= " s3 cp $file s3://$targetPath";
601
    $s3_cmd .= " s3 cp --quiet $file s3://$targetPath";
597
 
602
 
598
    $logger->logmsg("transferManifest:$targetPath");
603
    $logger->logmsg("transferManifest:$targetPath");
599
    $logger->verbose2("transferManifest:s3_cmd:$s3_cmd");
604
    $logger->verbose2("transferManifest:s3_cmd:$s3_cmd");
600
 
605
 
601
    my $cmdRv;
606
    my $cmdRv;
602
    unless ($conf->{'noTransfers'}) {
607
    unless ($conf->{'noTransfers'}) {
603
        my $ph;
608
        my $ph;
604
        open ($ph, "$s3_cmd |");
609
        open ($ph, "$s3_cmd 2>&1 |");
605
        while ( <$ph> )
610
        while ( <$ph> )
606
        {
611
        {
607
            chomp;
612
            chomp;
608
            $logger->verbose2("transferManifest:Data: $_");
613
            $logger->verbose2("transferManifest:Data: $_");
609
        }
614
        }
Line 633... Line 638...
633
    #
638
    #
634
    #   Mark the Release Manager entry as done
639
    #   Mark the Release Manager entry as done
635
    #       Need Write access to do this
640
    #       Need Write access to do this
636
    #
641
    #
637
    $logger->verbose("Update database: $rtagId");
642
    $logger->verbose("Update database: $rtagId");
638
    my $rv = executeRmQuery('MarkDone', "UPDATE RELEASE_TAGS SET S3MANIFEST_DONE = 'Y' where RTAG_ID = $rtagId") ;
643
    my $rv = executeRmQuery('MarkDone', "UPDATE release_manager.RELEASE_TAGS SET S3MANIFEST_DONE = 'Y' where RTAG_ID = $rtagId") ;
639
    if ($rv) {
644
    if ($rv) {
640
        return;
645
        return;
641
    }
646
    }
642
 
647
 
643
    #
648
    #
644
    #   Remove the entry from hash of items to be processed
649
    #   Remove the entry from hash of items to be processed
645
    #
650
    #
646
    $logger->logmsg("Complete: $targetName");
651
    $logger->logmsg("Complete: $targetName");
647
    delete $activeReleases->{$rtagId};
652
    delete $activeReleases->{$rtagId};
648
    unlink $file;
653
    unlink $file;
-
 
654
    $lastTagListUpdate = 0;
649
    return;
655
    return;
650
}
656
}
651
 
657
 
652
#-------------------------------------------------------------------------------
658
#-------------------------------------------------------------------------------
653
# Function        : checkPackageVersion 
659
# Function        : checkPackageVersion 
Line 716... Line 722...
716
sub getDataFromRm
722
sub getDataFromRm
717
{
723
{
718
    my ($name,$m_sqlstr, $options ) = @_;
724
    my ($name,$m_sqlstr, $options ) = @_;
719
    my @row;
725
    my @row;
720
    my $data;
726
    my $data;
-
 
727
    $RMerror = 0;
721
 
728
 
722
    if (ref $options ne 'HASH') {
729
    if (ref $options ne 'HASH') {
723
        $options = {}; 
730
        $options = {}; 
724
    }
731
    }
725
 
732
 
Line 742... Line 749...
742
                }
749
                }
743
            }
750
            }
744
            $sth->finish();
751
            $sth->finish();
745
        } else {
752
        } else {
746
            $logger->warn("Execute failure:$name: $m_sqlstr", $sth->errstr() );
753
            $logger->warn("Execute failure:$name: $m_sqlstr", $sth->errstr() );
-
 
754
            $RMerror++;
-
 
755
            $statistics{dbErrors}++;
747
        }
756
        }
748
    } else {
757
    } else {
749
        $logger->warn("Prepare failure:$name" );
758
        $logger->warn("Prepare failure:$name" );
-
 
759
        $RMerror++;
-
 
760
        $statistics{dbErrors}++;
750
    }
761
    }
751
 
762
 
752
    if (!$data && $options->{error}) {
763
    if (!$data && $options->{error}) {
753
        $logger->warn( $options->{error} );
764
        $logger->warn( $options->{error} );
754
    }
765
    }
Line 779... Line 790...
779
sub executeRmQuery
790
sub executeRmQuery
780
{
791
{
781
    my ($fname, $m_sqlstr) = @_;
792
    my ($fname, $m_sqlstr) = @_;
782
 
793
 
783
    $logger->verbose3('ExecuteQuery:', $fname);
794
    $logger->verbose3('ExecuteQuery:', $fname);
-
 
795
    $RMerror = 0;
784
    #
796
    #
785
    #   Create the full SQL statement
797
    #   Create the full SQL statement
786
    #
798
    #
787
    my $sth = $RM_DB->prepare($m_sqlstr);
799
    my $sth = $RM_DB->prepare($m_sqlstr);
788
    if ( defined($sth) )
800
    if ( defined($sth) )
Line 792... Line 804...
792
            $sth->finish();
804
            $sth->finish();
793
        }
805
        }
794
        else
806
        else
795
        {
807
        {
796
            $logger->warn("$fname: Execute failure: $m_sqlstr", $sth->errstr() );
808
            $logger->warn("$fname: Execute failure: $m_sqlstr", $sth->errstr() );
-
 
809
            $RMerror++;
-
 
810
            $statistics{dbErrors}++;
797
            return 1;
811
            return 1;
798
        }
812
        }
799
    }
813
    }
800
    else
814
    else
801
    {
815
    {
802
        $logger->warn("$fname: Prepare failure");
816
        $logger->warn("$fname: Prepare failure");
-
 
817
        $RMerror++;
-
 
818
        $statistics{dbErrors}++;
803
        return 1;
819
        return 1;
804
    }
820
    }
805
 
821
 
806
    return 0;
822
    return 0;
807
}
823
}
Line 878... Line 894...
878
        # Note: Must match @recoverTags in readStatistics
894
        # Note: Must match @recoverTags in readStatistics
879
        $statistics{dayStart} = $time;
895
        $statistics{dayStart} = $time;
880
        $statistics{txCount} = 0;
896
        $statistics{txCount} = 0;
881
        $statistics{txBytes} = 0;
897
        $statistics{txBytes} = 0;
882
        $statistics{linkErrors} = 0;
898
        $statistics{linkErrors} = 0;
-
 
899
        $statistics{dbErrors} = 0;
883
    }
900
    }
884
}
901
}
885
 
902
 
886
#-------------------------------------------------------------------------------
903
#-------------------------------------------------------------------------------
887
# Function        : readStatistics 
904
# Function        : readStatistics 
Line 893... Line 910...
893
#
910
#
894
# Returns         : 
911
# Returns         : 
895
#
912
#
896
sub readStatistics
913
sub readStatistics
897
{
914
{
898
    my @recoverTags = qw(dayStart txCount txBytes linkErrors);
915
    my @recoverTags = qw(dayStart txCount txBytes linkErrors dbErrors);
899
 
916
 
900
    if ($conf->{'statsfile'} and -f $conf->{'statsfile'})
917
    if ($conf->{'statsfile'} and -f $conf->{'statsfile'})
901
    {
918
    {
902
        if (open my $fh, $conf->{'statsfile'})
919
        if (open my $fh, $conf->{'statsfile'})
903
        {
920
        {
Line 940... Line 957...
940
    $statistics{upTime} = $statistics{timeStamp} - $startTime;
957
    $statistics{upTime} = $statistics{timeStamp} - $startTime;
941
    $statistics{wedged} = Utils::isWedged($conf);
958
    $statistics{wedged} = Utils::isWedged($conf);
942
 
959
 
943
    if ( $statistics{wedged}) {
960
    if ( $statistics{wedged}) {
944
         $statistics{state} = 'Wedged';
961
         $statistics{state} = 'Wedged';
-
 
962
    } elsif(!$dbUp){
-
 
963
        $statistics{state} = 'RM Access error';
945
    } elsif(!$linkUp){
964
    } elsif(!$linkUp){
946
        $statistics{state} = 'S3 Bucket Read Error';
965
        $statistics{state} = 'S3 Bucket Read Error';
947
    } elsif ($statistics{slowGen}) {
966
    } elsif ($statistics{slowGen}) {
948
        $statistics{state} = 'Slow manifest generation detected';
967
        $statistics{state} = 'Slow manifest generation detected';
949
    } else {
968
    } else {