WebSVN – DevTools – Diff – /core_devl/trunk/TOOLS/LOCAL/cc2svn_gendata.pl

 } else {
     Error ("Mode not specified: all, hops, standard");
+}
 GetAllPackageNames();
+getPackagesInSvn();
 getReleaseDetails();
 getPkgDetailsByRTAG_ID();
 my ($pcount, $vcount) = countPackages();
 print "Directly referenced Packages: $pcount Versions: $vcount\n";
 LocateStrays() unless ($opt_quick);
 ($pcount, $vcount) = countPackages();
 print "Indirectly referenced Packages: $pcount Versions: $vcount\n";
+getSourcePaths();
 processData();
 outputData();
 if ( $opt_verbose > 1 )
+{
 ($pcount, $vcount) = countPackages();
 print "Total References Packages: $pcount Versions: $vcount\n";
 exit;
 #-------------------------------------------------------------------------------
+# Function        : getPackagesInSvn
+#
+# Description     : Determine packages that have been migrated
+#                   Done by looking up RM for any version that has been migrated
+#
+# Inputs          :
+#
+# Returns         :
+#
+sub getPackagesInSvn
+{
+    my (@row);
+    # if we are not or cannot connect then return 0 as we have not found anything
+    connectRM(\$RM_DB) unless $RM_DB;
+    # First get all packages that are referenced in a Release
+    # This will only get the top level packages
+    # From non-archived releases
+    my $m_sqlstr = "SELECT UNIQUE" .
+                        " pv.PKG_ID ".                                         #[0]
+                   " FROM RELEASE_MANAGER.PACKAGE_VERSIONS pv".
+                   " WHERE pv.VCS_TYPE_ID=23";
+    my $sth = $RM_DB->prepare($m_sqlstr);
+    if ( defined($sth) )
+    {
+        if ( $sth->execute( ) )
+        {
+#            print "--- Execute\n";
+            if ( $sth->rows )
+            {
+                #print "--- Execute ROWS\n";
+                while ( @row = $sth->fetchrow_array )
+                {
+                    my $id = $row[0];
+                    next unless ( $id );
+                    $AllPackages{$id}{inSvn} = 1;
+#print "Data: @row\n";
+                }
+            }
+#            print "--- Finish\n";
+            $sth->finish();
+        }
+        else
+        {
+            Error("Execute failure: $m_sqlstr", $sth->errstr() );
+        }
+    }
+    else
+    {
+        Error("Prepare failure" );
+    }
+}
+#-------------------------------------------------------------------------------
 # Function        : getReleaseDetails
+#
 # Description     : Determine all candiate releases
+#
 # Inputs          :
+}
 sub getPkgDetailsByPVID
+{
-    my ($pv_id) = @_;
+    my (@plist) = @_;
     my (@row);
+    #
     #   Only do once
+    #
-    return if ( exists $Packages{$pv_id}{name} );
+#    return if ( exists $Packages{$pv_id}{name} );
     # if we are not or cannot connect then return 0 as we have not found anything
     connectRM(\$RM_DB) unless $RM_DB;
     my $m_sqlstr = "SELECT" .
                         " pv.PV_ID, ".                                          #[0]
                         " pv.PKG_ID," .                                         #[5]
                         " pv.MODIFIED_STAMP, ".                                 #[6]
                         " pv.CREATOR_ID  ".                                     #[7]
                    " FROM RELEASE_MANAGER.PACKAGE_VERSIONS pv,".
                    "      RELEASE_MANAGER.PACKAGES pkg ".
-                   " WHERE pv.PV_ID = \'$pv_id\' ".
+                   " WHERE pv.PKG_ID = pkg.PKG_ID ".
-                   "   AND pv.PKG_ID = pkg.PKG_ID" ;
+                   "   AND pv.PV_ID in ( " . join(',', @plist) . "  )" ;
     my $sth = $RM_DB->prepare($m_sqlstr);
     if ( defined($sth) )
+    {
         if ( $sth->execute( ) )
+        {
                 while ( @row = $sth->fetchrow_array )
+                {
                     print join (',',@row), "\n" if ($opt_verbose);
                     my $pvid = $row[0];
+                    my $pkgid = $row[5];
                     $Packages{$pvid}{name} = $row[1];
+                    next if ( exists $Packages{$pvid}{version} );
                     $Packages{$pvid}{version} = $row[2];
                     $Packages{$pvid}{locked} = $row[3];
                     $row[4] =~ tr~\\/~/~;
                     $Packages{$pvid}{vcstag} = $row[4];
-                    $Packages{$pvid}{pkgid} = $row[5];
+                    $Packages{$pvid}{pkgid} = $pkgid;
 #                    $Packages{$pvid}{tlp} = 1;
                     ($Packages{$pvid}{suffix}, $Packages{$pvid}{fullVersion},$Packages{$pvid}{isaRipple} ) = massageVersion( $Packages{$pvid}{version}, $Packages{$pvid}{name} );
                     $Suffixes{$Packages{$pvid}{suffix}}++;
                     $Packages{$pvid}{Age} = ($now - str2time( $row[6] )) / (60 * 60 * 24);
                     $Packages{$pvid}{Creator} = $row[7];
+                    $AllPackages{$pkgid}{essential} = 1;
+                }
+            }
 #            print "--- Finish\n";
             $sth->finish();
+        }
             if ( $sth->rows )
+            {
 #                print "--- Execute ROWS\n";
                 while ( @row = $sth->fetchrow_array )
+                {
+                    $count++;
                     print join (',',@row), "\n" if ($opt_verbose);
                     my $pvid = $row[0];
                     unless ( exists $Packages{$pvid}{name} )
+                    {
+                        my $pkgid = $row[5];
                         $Packages{$pvid}{name} = $row[1];
                         $Packages{$pvid}{version} = $row[2];
                         $Packages{$pvid}{locked} = $row[3];
                         $row[4] =~ tr~\\/~/~;
                         $Packages{$pvid}{vcstag} = $row[4];
-                        $Packages{$pvid}{pkgid} = $row[5];
+                        $Packages{$pvid}{pkgid} = $pkgid;
                         $Packages{$pvid}{tlp} = 1;
                         ($Packages{$pvid}{suffix}, $Packages{$pvid}{fullVersion},$Packages{$pvid}{isaRipple} ) = massageVersion( $Packages{$pvid}{version}, $Packages{$pvid}{name} );
                         $Suffixes{$Packages{$pvid}{suffix}}++;
+                        $AllPackages{$pkgid}{essential} = 1;
                         push @StrayPackages, $pvid;
+                    }
                     my $rtag_id = $row[6];
+                    }
                     if ( $opt_limit )
+                    {
-                        last if ( $count++ > $opt_limit );
+                        last if ( $count > $opt_limit );
+                    }
+                }
+            }
 #            print "--- Finish\n";
             $sth->finish();
+    }
     else
+    {
         Error("Prepare failure" );
+    }
+    Message ("Extract toplevel dependencies: $count rows");
+}
 #-------------------------------------------------------------------------------
 # Function        : GetDepends
+#
 # Description     :
+#
-# Inputs          : $pvid
+# Inputs          : @plist          - list of pvid's to process
+#
 # Returns         :
+#
 sub GetDepends
+{
-    my ($pv_id ) = @_;
+    my (@plist) = @_;
+    #
     #   Ensure we have package information
+    #
-    getPkgDetailsByPVID( $pv_id );
+    getPkgDetailsByPVID( @plist );
-    return if ( $Packages{$pv_id}{depend} );
+#    return if ( $Packages{$pv_id}{depend} );
-    $Packages{$pv_id}{depend} = 1;
+#    $Packages{$pv_id}{depend} = 1;
+    #
     #   Now extract the package dependacies
     #   There may not be any
+    #
     my $m_sqlstr = "SELECT ".
                     " pd.PV_ID, ".
                     " pd.DPV_ID " .
                   " FROM    RELEASE_MANAGER.PACKAGE_DEPENDENCIES pd ".
-                  " WHERE pd.PV_ID = \'$pv_id\'";
+                  " WHERE pd.PV_ID in ( " . join(',', @plist) . " )";
     my $sth = $RM_DB->prepare($m_sqlstr);
     if ( defined($sth) )
+    {
         if ( $sth->execute( ) )
+        {
                 while ( my @row = $sth->fetchrow_array )
+                {
                     my $id = $row[0];
                     my $name = $row[1];
                     next unless ( $id );
-                    $AllPackages{$id} = $name;
+                    $AllPackages{$id}{name} = $name;
+                }
+            }
             $sth->finish();
+        }
         else
+        {
-        Error("GetAllPackageNames:Execute failure" );
+            Error("GetAllPackageNames:Execute failure: $m_sqlstr", $sth->errstr() );
+        }
+    }
     else
+    {
         Error("GetAllPackageNames:Prepare failure" );
+    }
+}
+#-------------------------------------------------------------------------------
+# Function        : getSourcePaths
+#
+# Description     : Get source paths for all packages
+#                   Really only want those not in SVN
+#
+# Inputs          : Globals
+#
+# Returns         : Globals
+#
+sub getSourcePaths
+{
+    Message("Get source paths for all packages");
+    # if we are not or cannot connect then return 0 as we have not found anything
+    connectRM(\$RM_DB) unless $RM_DB;
+    #
+    #   Process on blocks for speed
+    #
+    my @needed;
+    foreach my $pkgid ( keys %AllPackages, 0 )
+    {
+        if ( $pkgid )
+        {
+            next unless ( exists $AllPackages{$pkgid}{essential} );
+            next if ( $AllPackages{$pkgid}{inSvn} );
+            $AllPackages{$pkgid}{essential} = 2;
+            push @needed, $pkgid;
+#print "= $pkgid, $AllPackages{$pkgid}{name}\n";
+        }
+        #
+        #   Have enough for a query
+        #
+        if ( scalar @needed > 100 || (! $pkgid && scalar @needed > 0) )
+        {
+            #
+            #   Now extract all the source paths
+            #
+            my $m_sqlstr = "SELECT UNIQUE" .
+                        " pv.PKG_ID, ".                                     #[0]
+                        " pv.SRC_PATH, ".                                   #[1]
+                        " pv.VCS_TYPE_ID ".                                 #[2]
+                        " FROM RELEASE_MANAGER.PACKAGE_VERSIONS pv".
+                        " WHERE pv.PKG_ID in (" . join(',', @needed) . ")";
+            my $sth = $RM_DB->prepare($m_sqlstr);
+            if ( defined($sth) )
+            {
+                if ( $sth->execute( ) )
+                {
+                    if ( $sth->rows )
+                    {
+                        while ( my @row = $sth->fetchrow_array )
+                        {
+#print "--- @row\n";
+                            my $pkgid = $row[0];
+                            my $type = $row[2] || '2';
+                            if ( $row[1] )
+                            {
+                                #
+                                #   Trivial cleanup
+                                #
+                                my $path = '/' . $row[1];   # Force initial /
+                                $path =~ tr~\\/~/~s;        # Force / and single /
+                                $path =~ s~/+$~~;           # Remove Trailing /
+                                $AllPackages{$pkgid}{srcPath}{$path}++;
+                            }
+                        }
+                    }
+                    $sth->finish();
+                }
+                else
+                {
+                    Error("getSourcePaths:Execute failure: $m_sqlstr", $sth->errstr() );
+                }
+            }
+            else
+            {
+                Error("getSourcePaths:Prepare failure" );
+            }
+            #
+            #   Reset the list
+            #
+            @needed = ();
+        }
+    }
+}
 #-------------------------------------------------------------------------------
 # Function        : massageVersion
+#
 # Description     : Process a version number and return usful bits
 #-------------------------------------------------------------------------------
 # Function        : LocateStrays
+#
-# Description     :
+# Description     : Locate stray packages
+#                   Try to do several at a time to speed up processing
+#
 # Inputs          :
+#
 # Returns         :
+#
 sub LocateStrays
+{
     Message ("Locate indirectly referenced packages");
     while ( $#StrayPackages >= 0 )
+    {
-        my $pv_id = pop @StrayPackages;
+#print "Strays Remaining: ", scalar @StrayPackages ,"\n";
+        my @plist;
+        while ( $#plist <= 200 && @StrayPackages )
+        {
+            my $pv_id = pop @StrayPackages;
-        next if ( exists $Packages{$pv_id}{done} );
+            next if ( exists $Packages{$pv_id}{done} );
-#print "... ",$#StrayPackages,"\n";
+            push @plist, $pv_id;
+        }
-        GetDepends( $pv_id);
+        GetDepends(@plist) if @plist;
+        foreach ( @plist)
+        {
-        $Packages{$pv_id}{done} = 1;
+            $Packages{$_}{done} = 1;
+        }
+    }
+}
 #-------------------------------------------------------------------------------
 # Function        : countPackages

Subversion Repositories DevTools

(root)/core_devl/trunk/TOOLS/LOCAL/cc2svn_gendata.pl – Rev 2478 → 2764