WebSVN – DevTools – Diff – /dpkg_blat/trunk/src/blat/blatDaemon.pl

+#
 #   Globals
+#
 my $logger = StdLogger->new();                  # Stdout logger. Only during config
 $logger->err("No config file specified") unless (defined $ARGV[0]);
+$logger->err("Config File does not exist: $ARGV[0]") unless (-f $ARGV[0]);
 my $name = basename( $ARGV[0]);
    $name =~ s~.conf$~~;
 my $now = 0;
 my $startTime = 0;
 my $tar = 'tar';
 my $RemotePkgList = {};
 my $targetBinDir = "$FindBin::Bin/targetBin";
 my $server_id;
 my @projectList;
 my @releaseList;
+my $isS3Target;
+#
 #   Contain statisics maintained while operating
 #       Can be dumped with a kill -USR2
 #       List here for documentation
+#
 #   Describe configuration parameters
+#
 my %cdata = (
     '.ignore'         => {'pkg\.(.+)' => 'pkgs' },
+    '.oneOf'          => [['hostname','S3Bucket']] ,
     'piddir'          => {'mandatory' => 1      , 'fmt' => 'dir'},
     'sleep'           => {'default'   => 5      , 'fmt' => 'period'},
     'dpkg_archive'    => {'mandatory' => 1      , 'fmt' => 'dir'},
     'logfile'         => {'mandatory' => 1      , 'fmt' => 'vfile'},
     'logfile.size'    => {'default'   => '1M'   , 'fmt' => 'size'},
     'logfile.count'   => {'default'   => 9      , 'fmt' => 'int'},
     'verbose'         => {'default'   => 0      , 'fmt' => 'int'},
-    'user'            => {'mandatory' => 1      , 'fmt' => 'text'},
-    'hostname'        => {'mandatory' => 1      , 'fmt' => 'text'},
     'rmHostName'      => {'default'   => undef  , 'fmt' => 'text'},
     'sshport'         => {'default'   => 0      , 'fmt' => 'int'},
-    'identity'        => {'mandatory' => 1      , 'fmt' => 'file'},
-    'bindir'          => {'mandatory' => 1      , 'fmt' => 'text'},
     'tagdir'          => {'mandatory' => 1      , 'fmt' => 'dir'},
     'forcedirscan'    => {'default'   => 100    , 'fmt' => 'period'},
     'tagage'          => {'default'   => '10m'  , 'fmt' => 'period'},
     'tagListUpdate'   => {'default'   => '1h'   , 'fmt' => 'period'},
-    'tagMaxPackages'  => {'default'   => 10      , 'fmt' => 'int'},
+    'tagMaxPackages'  => {'default'   => 10     , 'fmt' => 'int'},
     'rmConfigCheck'   => {'default'   => '60'   , 'fmt' => 'period'},
     'rmConfFullRead'  => {'default'   => '1h'   , 'fmt' => 'period'},
     'synctime'        => {'default'   => '2h'   , 'fmt' => 'period'},
     'syncretry'       => {'default'   => '5m'   , 'fmt' => 'period'},
     'allProjects'     => {'default'   => 0      , 'fmt' => 'bool'},
     'packageFilter'   => {'default'   => undef  , 'fmt' => 'text'},
     'active'          => {'default'   => 1      , 'fmt' => 'bool'},
     'debug'           => {'default'   => 0      , 'fmt' => 'bool'},                 # Log to screen
     'txdetail'        => {'default'   => 0      , 'fmt' => 'bool'},
     'noTransfers'     => {'default'   => 0      , 'fmt' => 'bool'},                 # Debugging option to prevent transfers
+    #
+    #   Transfer via ssh
+    #   Cannot be used in conjunction with S3Bucket
+    #
+    'hostname'        => {'fmt' => 'text', requires => 'user,identity,bindir'},
+    'user'            => {'fmt' => 'text', requires => 'hostname'},
+    'identity'        => {'fmt' => 'file'},
+    'bindir'          => {'fmt' => 'text'},
+    #
+    #   Transfer to S3 configuration items
+    #       Cannot be used in conjunction with hostname
+    #       Many other options will be ignored
+    #
+    'S3Bucket'        => {'fmt' => 'text', requires => 'S3Profile'},
+    'S3Profile'       => {'fmt' => 'text', requires => 'S3Bucket' },
 );
+#
 #   Read in the configuration
         if ( scalar @{$errors} > 0 )
+        {
             warn "$_\n" foreach (@{$errors});
             die ("Config contained errors\n");
+        }
+        $isS3Target = defined $conf->{'S3Bucket'};
+        #
         #   Reset some information
         #   Create a new logger
+        #
-        $logger = Logger->new($conf);
+        $logger = Logger->new($conf) unless $conf->{debug};
         $conf->{logger} = $logger;
         $conf->{'pidfile'} = $conf->{'piddir'} . '/' . $name . '.pid';
+        $logger->setVerbose($conf->{verbose});
         $logger->verbose("Log Levl: $conf->{verbose}");
+        #
         #   Setup statistics filename
         $conf->{'statsfile'} = $conf->{'piddir'} . '/' . $name . '.stats';
+        #
         #   Extract extra package config
         #       Ignore ALL and Version info if transferring the entire archive
         #       Honor the EXCLUDE - for bandwidth reasons
+        #
+        #   NOTE: Package inclusion will not be processed in allArchive mode
+        #   BUG:  This processing should be done after the RmConfig has been included
+        #
         $extraPkgs = {};
         $excludePkgs = {};
         while (my($key, $data) = each ( %{$conf->{pkgs}} ))
+        {
         $logger->warn("Transfer session configured as not active") unless ( $conf->{'active'} );
         $logger->warn("Transfer all project packages") if ( $conf->{'allProjects'} );
         $logger->warn("Transfer entire package archive") if ( $conf->{'allArchive'} );
         $logger->warn("All Transfers disabled") if ( $conf->{'noTransfers'} );
+        $logger->warn("Transfer to AWS S3 Bucket") if ( $isS3Target );
+    }
     return $rv;
+}
+#
 # Returns         : Nothing
+#
 sub checkForBasicTools
+{
+    return if $isS3Target;
     my $ph;
     my $found;
     my $tgt_cmd = "if [ -x  $conf->{'bindir'}/get_plist.pl ] ; then echo :FOUND:; fi";
     my $ssh_cmd = sshCmd($tgt_cmd);
+    #
     #   Is Release List Processing active
     #   Can configure blat to disable release sync
     #   This will then allow 'new' packages to be sent
+    #
-    if ( $conf->{maxpackages} == 0 || $conf->{'synctime'} <= 0)
+    if ( $conf->{maxpackages} == 0 || $conf->{'synctime'} <= 0 || $isS3Target )
+    {
         $logger->verbose2("processReleaseList disabled");
+        $RemotePkgList = {};
         return;
+    }
+    #
     #   Time to perform the scan
                 } elsif ( transferPackage( $package, $version )) {
                     unlink $file;
+                }
                 else
+                {
+                    if ($conf->{'tagage'} > 0) {
-                    my ($mtime) = Utils::mtime( $file );
+                        my ($mtime) = Utils::mtime( $file );
-                    if ( $now - $mtime > $conf->{'tagage'} )
+                        if ( $now - $mtime > $conf->{'tagage'} )
-                    {
+                        {
-                        $logger->warn ("Delete unsatisfied tag: $package::$version");
+                            $logger->warn ("Delete unsatisfied tag: $package::$version after $conf->{'tagage'}" );
-                        unlink $file;
+                            unlink $file;
-                        $statistics{staleTags}++;
+                            $statistics{staleTags}++;
+                        }
+                    }
+                }
                 $tagCount--;
                 reapChildren();
+    {
         $logger->warn("transferPackage:Package not found: $pname, $pver");
         return $rv;
+    }
+    my $tzdir = catfile( $conf->{'dpkg_archive'} , '.dpkg_archive', 'tarStore' );
+    my $tzfile = $pname . '__' . $pver . '.tgz';
+    my $tzpath = catfile($tzdir, $tzfile);
+    unless (-f $tzpath) {
+        $logger->verbose("transferPackage: tarZip not found - $tzpath");
+        $conf->{'tagdir'} =~ m~^(.*)/~;
+        my $tagRoot = $1;
+        my $tag = "$pname::$pver";
+        Utils::TouchFile($conf, catfile($tagRoot, 'tarZip', $tag));
+        return 0;
+    }
     ###########################################################################
     #   Transfer the package / symlink
+    #
+    if ($isS3Target) {
+        $cmdRv = transferPackageS3($tzdir, $tzfile, $pname, $pver);
+    } else {
+        $cmdRv = transferPackageSsh($tzdir, $tzfile, $sfile, $pname, $pver);
+    }
+    #
+    #   Display the size of the package
+    #       Diagnostic use
+    #
+    if ($conf->{txdetail}) {
+        my $ph;
+        open ( $ph, "du -bs $sfile 2>/dev/null |" );
+        my $line = <$ph>;
+        $line =~ m/^([0-9]+)/;
+        $line = $1 || 0;
+        my $size = sprintf "%.3f", $line / 1024 / 1024 / 1024 ;
+        close $ph;
+        my $duration = time - $startTime;
+        $logger->logmsg("transferPackage: Stats: $pname, $pver, $size Gb, $duration Secs");
+    }
+    if ( $cmdRv == 0 ) {
+        #
+        #   Mark has having been transferred in the current cycle
+        #
+        $RemotePkgList->{$pname}{$pver}{transferred} = 1;
-    my $tar_cmd;
+        $rv = 1;
+        $statistics{txCount}++;
+    } else {
+        $logger->warn("transferPackage:Transfer Error: $pname, $pver, $?");
+    }
+    LogTxError ($?);
+    return $rv;
+}
+#-------------------------------------------------------------------------------
+# Function        : transferPackageSsh
+#
+# Description     : Transfer a package via an ssh connection
+#
+# Inputs          : $tzdir  - Directory that contains the tarZip file
+#                   $tzname - Name of tarZip File
+#                   $sfile  - Full path to the source file
+#                   $pname  - Package Name
+#                   $pver   - Package Version
+#
+# Returns         : Result Code
+#                   0   - Transfer OK
+#                   <0  - Skip transfer
+#                   >0  - Command error code
+#
+sub transferPackageSsh
+{
+    my ($tzdir, $tzname, $sfile, $pname, $pver) = @_;
     my $tgt_cmd;
     my $ssh_cmd;
+    my $cmdRv = 0;
+    my $tzfile = catfile($tzdir, $tzname);
     if (-l $sfile) {
+        #
         #   Determine the value of the symlink
-        #   Only support simple symlinks - this in the same directory
+        #   Only support simple symlinks - that are in the same directory
+        #
         my $lver = readlink( $sfile );
         if ( ! defined $lver ) {
             $logger->warn("Can't resolve symlink: $pname, $pver");
-            next;
+            return -1;
+        }
         if ( $lver =~ m ~/~ ) {
             $logger->warn("Won't resolve symlink: $pname, $pver, $lver");
-            next;
+            return -1;
+        }
         $tgt_cmd = "$conf->{'bindir'}/receive_symlink \"$pname\" \"$pver\" \"$lver\"";
         $ssh_cmd = sshCmd($tgt_cmd);
     } else {
+        #
         #   Create the process pipe to transfer the package
-        #   Tar the directory and pipe the result through a ssh session to
+        #   Pipe the tarZip of the package through a ssh session to the target machine
-        #   the target machine
-        #   $tar -czf - -C "$dpkg/${pname}/${pver}" . |  ssh  ... "./receive_package pname pver"
+        #   cat $tzpath | ssh  ... "./receive_package pname pver"
+        #
-        $tar_cmd = "$tar -czf - -C \"$sfile\" .";
         $tgt_cmd = "$conf->{'bindir'}/receive_package \"$pname\" \"$pver\"";
         $ssh_cmd = sshCmd($tgt_cmd);
+        $ssh_cmd .= " <$tzfile"
+    }
-    $logger->verbose2("transferPackage:tar_cmd:$tar_cmd") if defined $tar_cmd;
     $logger->verbose2("transferPackage:tgt_cmd:$tgt_cmd");
     $logger->verbose2("transferPackage:ssh_cmd:$ssh_cmd");
     unless ($conf->{'noTransfers'}) {
         my $ph;
-        my @cmd_list;
-        push (@cmd_list, $tar_cmd) if defined $tar_cmd;
-        push (@cmd_list, $ssh_cmd);
-        my $cmd = join (' | ', @cmd_list);
-        open ($ph, "$cmd |");
+        open ($ph, "$ssh_cmd |");
         while ( <$ph> )
+        {
             chomp;
             $logger->verbose2("transferPackage:Data: $_");
+        }
         close ($ph);
         $cmdRv = $?;
         $logger->verbose("transferPackage:End: $?");
+    }
+    return $cmdRv;
+}
+#-------------------------------------------------------------------------------
+# Function        : transferPackageS3
+#
+# Description     : Transfer a package to an AWS S3 bucket
+#                   Requires that the package already be tarZip-ed
+#
+# Inputs          : $tzdir  - Directory that contains the tarZip file
+#                   $tzfile - Name of tarZip File
+#                   $pname  - Package Name
+#                   $pver   - Package Version
+#
+# Returns         : Result Code
+#                   0   - Transfer OK
+#                   <0  - Skip transfer
+#                   >0  - Command error code
+#
+sub transferPackageS3
+{
+    my ($tzdir, $tzfile, $pname, $pver) = @_;
+    my $cmdRv = 0;
+    #
-    #   Display the size of the package
+    #   Locate the file on the dpkgArchive tarZip store
+    #
+    my $sfile = catfile($tzdir, $tzfile);
+    if (-l $sfile) {
+        $logger->warn("Will not transfer symlink: $pname, $pver");
-    #       Diagnostic use
+        return -1;
+    }
+    #
+    #   Create a command to transfer the file to AWS use the cli tools
+    #   Note: Ive seen problem with this when used from Perth to AWS (Sydney)
+    #         If this is an issue use curl - see the savePkgToS3.sh for an implementation
+    #
+    my $s3_cmd = "aws --profile $conf->{'S3Profile'} s3 cp $sfile s3://$conf->{'S3Bucket'}/$tzfile";
+    $logger->verbose2("transferPackage:s3_cmd:$s3_cmd");
-    if ($conf->{txdetail}) {
+    unless ($conf->{'noTransfers'}) {
         my $ph;
-        open ( $ph, "du -bs $sfile 2>/dev/null |" );
+        open ($ph, "$s3_cmd |");
-        my $line = <$ph>;
+        while ( <$ph> )
-        $line =~ m/^([0-9]+)/;
+        {
-        $line = $1 || 0;
+            chomp;
-        my $size = sprintf "%.3f", $line / 1024 / 1024 / 1024 ;
+            $logger->verbose2("transferPackage:Data: $_");
+        }
-        close $ph;
+        close ($ph);
-        my $duration = time - $startTime;
+        $cmdRv = $?;
-        $logger->logmsg("transferPackage: Stats: $pname, $pver, $size Gb, $duration Secs");
+        $logger->verbose("transferPackage:End: $?");
+    }
-    if ( $cmdRv == 0 )
-    {
-        #
-        #   Mark has having been transferred in the current cycle
-        #
-        $RemotePkgList->{$pname}{$pver}{transferred} = 1;
-        $rv = 1;
-        $statistics{txCount}++;
-    }
-    else
-    {
-        $logger->warn("transferPackage:Transfer Error: $pname, $pver, $?");
-    }
-    LogTxError ($?);
-    return $rv;
+    return $cmdRv;
+}
 #-------------------------------------------------------------------------------
 # Function        : deletePackage
+#
 # Description     : Delete specified package to target system
+#
         $lastReleaseScan = 0;
         $lastTagListUpdate = 0;
         $lastRmConfRead = 0;
     };
-    alarm 60;
+    alarm 60 unless $conf->{debug};
     $SIG{ALRM} = sub {
         # On Dump Statistics
         $logger->verbose2('Received SIGUSR2.');
         periodicStatistics();
         alarm 60;

Subversion Repositories DevTools

(root)/dpkg_blat/trunk/src/blat/blatDaemon.pl – Rev 6779 → 7387