WebSVN – DevTools – Diff – /buildtool/branches/planMetrics/src/main/java/com/erggroup/buildtool/ripple/RippleEngine.java

      *  This is a stand alone method to contain the work
+     *
      * @throws Exception
      * @throws SQLException
      */
-    private void planBuildOrder() throws Exception, SQLException {
+    private void planBuildOrder() throws Exception, SQLException
+    {
-//TODO - Planing is not working well
+//TODO - Planning is not working well
 /**
  * Current status
  * The selection of the package to build next is deeply flawed
- * Currently it will select a ripple over a directly planned package - thus the unit tests are not working
+ * Currently it will select a directly planned package over a ripple - (perhaps that is good)
+ *
+ * Currently on the buildPlan of the 'selected' package is included in the BuildOrder
+ *
  * The current algorithm is based on time - but all packages have zero build time ( in UTF )
  * Even if this is fixed the algorithm is broken as we will select the complete build with the shortest time
  * which will be the ripple of a leaf package ( perhaps that is good )
+ *
  * Also:
- *      Need to include all packages that we can now build into the final build plan, not just the first one and its build tree
+ *      [Done] Need to include all packages that we can now build into the final build plan, not just the first one and its build tree
+ *
- *      Need to generate a single package set - so that WIPS and others replace those in the final build set
+ *      [Done-ish]Need to generate a single package set - so that WIPS and others replace those in the final build set
- *      Remove duplicates from the complete package set
+ *      [Done]Remove duplicates from the complete package set
+ *
  *      Need more test cases
+ *
  *      Need to report all build options, not just the selected one
+ *
  *      Cleanup the ANT build file - don't list all the release dependencies, just those needed to build the current package
+ *
+ *      If a WIP on a package that has circular dependencies is added, then the WIP will be excluded. This prevents the problem being fixed.
+ *      Should not exclude a WIP of a package with a circular dependency
  */
         // Process remaining packages which are need to be reproduced for this baseline.
         //    Determine the build file for each package
         //    For daemon builds:
         if ( mDaemon )
+        {
             //
             //  Daemon Mode:
-            //  Locate all packages that we can build now
-            //      Packages that have a build requirement and have no reason we cannot build them
             //
+            //  Create a list of build candidates
+            //
+            //  Add in the WIPs, Tests and Ripple Requests so that we can process them as one collections
+            //  Now have a collection of packages that we would like to build right now
+            //  See if any of them can be excluded due to issues with creating a new package version
             ArrayList<Package> buildCandidates = new ArrayList<Package>();
+            buildCandidates.addAll(mPackageCollectionWip);
+            buildCandidates.addAll(mPackageCollectionTest);
+            buildCandidates.addAll(mPackageCollectionRipple);
+            //  Locate all packages that we can build now
+            //      Packages that have a build requirement and have no reason we cannot build them
             for (Iterator<Package> it = mBuildOrder.iterator(); it.hasNext(); )
+            {
                 Package p = it.next();
                 if ( p.mProcessed && p.mBuildFile > 0 )
+                {
                         mLogger.info("planRelease Candidate {}",p.mAlias);
+                    }
+                }
+            }
-            //  Add in the WIPs, Tests and Ripple Requests so that we can process them as one collections
+            //  Examine the build candidates and verify that a new version number can be calculated for each
-            //  Now have a collection of packages that we would like to build right now
+            //  version that we need to build.
-            //  See if any of them can be excluded due to issues with creating a new package version
-            buildCandidates.addAll(mPackageCollectionWip);
+            //
-            buildCandidates.addAll(mPackageCollectionTest);
+//TODO If we find WIP/TEST/RIPPLE with invalid nextVersion, then we could email the submitter and treat it as a fail.
-            buildCandidates.addAll(mPackageCollectionRipple);
             for (Iterator<Package> it = buildCandidates.iterator(); it.hasNext(); )
+            {
                 Package p = it.next();
                         else
+                        {
                             //  Calculate the time to build the package AND all of the packages that depend on it
                             //  Calculate the build plan for the package
                             //
-                            calcBuildPlan(p);
+                            calcBuildPlan(p, mPackageCollection, true);
+                        }
+                    }
+                }
+            }
-            //
-            //  Determine the NON-test package the we will build next
-            //
-            Package build = ReleaseManager.NULL_PACKAGE;
-            for (Iterator<Package> it = buildCandidates.iterator(); it.hasNext(); )
-            {
-                Package p = it.next();
-                if ( p.mBuildFile >= 0 )
-                {
-                    if (p.mTestBuildInstruction == 0)
-                    {
-                        //  Not a Test Build
-                        if (build == ReleaseManager.NULL_PACKAGE || p.mRippleTime < build.mRippleTime)
-                        {
-                            build = p;
-                        }
-                    }
-                }
-            }
             //  Determine the first test build package
             //  Assume the collection is in build instruction id order,
             //      So that the first request will be built first
             //  Add all buildable test-builds to the mBuildOrder collection
             Package testBuild = ReleaseManager.NULL_PACKAGE;
+                    }
+                }
+            }
             //
-            //  Extend mBuildOrder with the selected non-test build package and its build plan
+            //  Determine the NON-test packages the we will build next
-            //  Really only for display purposes
+            //  This will be the one with the lowest ripple time
             //
-            if (build != ReleaseManager.NULL_PACKAGE)
+            Package build = ReleaseManager.NULL_PACKAGE;
+            ArrayList<Package> buildNext = new ArrayList<Package>();
+            //  Extract candidates that are left
+            for (Iterator<Package> it = buildCandidates.iterator(); it.hasNext(); )
+            {
+                Package p = it.next();
+                if ( p.mBuildFile >= 0  &&p.mTestBuildInstruction == 0)
+                {
+                    buildNext.add(p);
+                }
+            }
+            if ( !buildNext.isEmpty() )
+            {
+                //  Sort on mRippleTime so that
+                //      We can get the lowest first
+                //      We can generate the complete plan in a nice order
+                //
+                Collections.sort(buildNext, Package.RippleTimeComparator);
+                //  At the moment extract the first one - this is the fastedt
+                //  TODO - could build another candidate as long as it does't extend the time by (say) 10%
+                //         Would need to rejigger the collection - for the next step
+                //
+                build = buildNext.get(0);
+                //  Determine the amount of time we would allow the 'best' plan to be extended
+                //      Allow for a 20% extension in time
+                //      Min of 10 minutes
+                int baseTime = build.mRippleTime;
+                baseTime = ( baseTime * 1200) / 1000;
+                if ( baseTime < 10 * 60) {
+                    baseTime = (10 * 60);
+                }
+                mLogger.error("Scan for package to build: {} {}", build.mRippleTime, baseTime);
+                //  Find last package that does not exceed the extended ripple time
+                for (Iterator<Package> it = buildNext.iterator(); it.hasNext(); )
+                {
+                    Package p = it.next();
+                    if (p.mRippleTime > baseTime)
+                    {
+                        break;
+                    }
+                    build = p;
+                }
+                //
+                //  Extend mBuildOrder with the selected non-test build package and its build plan
+                //  Really only for display purposes
+                //
+                Package.resetProcessed(buildNext);
                 mBuildOrder.add(build);
+                updateBuildList(build.mRipplePlan, buildNext);
                 mBuildOrder.addAll(build.mRipplePlan);
+                //  Add in other build candidates
+                for (Iterator<Package> it = buildNext.iterator(); it.hasNext(); )
+                {
+                    Package p = it.next();
+                    if ( p.mBuildFile >= 0 )
+                    {
+                        if ( p != build && ! p.mIsProcessed)
+                        {
+                            //  Not our selected package and not already processed
+                            mBuildOrder.add(p);
+                            updateBuildList(p.mRipplePlan, buildNext);
+                            mBuildOrder.addAll(p.mRipplePlan);
+                        }
+                    }
+                }
+            }
             //
             //  Mark the first package in the build order as the one to be built
             //  This will give test builds priority, while indicating what will be built after that
                     build.mVersion = build.mNextVersion;
+                }
+            }
             //
+            //  Massage the package collection
+            //      Replace WIP/TEST/RIPPLE so that the mPackageCollection is a collection of packages we would like in the release
+            //      Don't replace those that we can't build due to errors
+            //
+            for (Iterator<Package> it = buildCandidates.iterator(); it.hasNext(); )
+            {
+                Package p = it.next();
+                if ( p.mBuildFile >= 0 )
+                {
+                    Package pReleased = findPackage(p.mAlias);
+                    if ( pReleased != ReleaseManager.NULL_PACKAGE)
+                    {
+                        int index = mReleaseManager.findPackageLastIndex;
+                        mPackageCollection.set(index, p);
+                        p.mIsNotReleased = false;
+                    }
+                }
+            }
+            //
             //  To fit in with the old algorithm ( ie: could be improved )
             //  Insert marks into all packages
             //  Not sure exactly why - Its used in the generation of the ant build file
-            //  Want to set mNoBuildReason, mBuildFile
+            //                     Want to set mNoBuildReason, mBuildFile
             //
             //      Package we have selected to build: 0     , 1
-            //      Package we could have built      : 2     , 3
+            //      Package we could have built      : 0     , 2
             //      Packages we can't build          : reason, 3
             //      Packages that are OK             : 3     , 3
+            //      ????                             : 0     , 3
             for (Iterator<Package> it = mPackageCollectionAll.iterator(); it.hasNext(); )
+            {
                 Package p = it.next();
                 if (p == build ) {
                     p.mNoBuildReason = p.mBuildFile;
                     p.mBuildFile = 3;
+                }
                 else if (p.mBuildReason != null)
+                {
-                    p.mNoBuildReason = 2;
+                    p.mNoBuildReason = 0;
-                    p.mBuildFile = 3;
+                    p.mBuildFile = 2;
+                }
                 else
+                {
                     p.mNoBuildReason = 0;
                     p.mBuildFile = 3;
+                }
+            }
+        }
+    }
+    /** Update a collection of packages such that the collection contains packages from the WIP/RIPPLE collection
+     *  in preference to the package in the main collection
+     *
+     * @param packageList   - List to process and update
+     * @param repList - List of replacement candidates. Assumes valid packages
+     */
+    private void updateBuildList(ArrayList<Package> packageList, ArrayList<Package> repList ) {
+        int baseIndex = 0;
+        for (Iterator<Package> it = packageList.iterator(); it.hasNext(); baseIndex++)
+        {
+            Package basePkg = it.next();
+            for (Iterator<Package> repit = repList.iterator(); repit.hasNext(); )
+            {
+                Package repPkg = repit.next();
+                if ( basePkg.mAlias.compareTo( repPkg.mAlias ) == 0 )
+                {
+                    //  Replacement package has been found
+                    //  Only looking for non-test packages that can be built
+                    //  Flag that the package is now a part of the official release set
+                    //  Flag that the package has been processed to prevent it being included in a future plan
+                    packageList.set(baseIndex, repPkg);
+                    repPkg.mIsNotReleased = false;
+                    repPkg.mIsProcessed = true;
+                    break;
+                }
+            }
+        }
+    }
     /**
      * Calculate a build plan for the specified package
      * Calculates : mRippleTime - time to build the package and the ripple effect
      * Calculates : mBuildPlan - collection of packages in the plan that can be built at the moment
+     *
      * Needs to determine all packages that consume the package - and all packages that consume them
      *  Note: The package may not be in the mPackageCollection ( it may be a WIP/TEST/RIPPLE )
+     *
+     * Note: The rippleTime of PackageA is not the sum of the rippleTimes of the packages that consume PackageA.
+     *
      * Need to:
      *      Only include a package once.
      *      Only include packages that can/need to be built at the moment
+     *
-     * @param p     - Package to process
+     * @param p                 - Package to process
-     */
+     * @param pkgCollection     - Package Collection to scan
+     * @param processAll        - True: Order all packages discovered
-    private void calcBuildPlan(Package p) {
+     */
-        p.mRipplePlan = usedByPackages(p);
+    private void calcBuildPlan(Package p, ArrayList<Package> pkgCollection, boolean processAll) {
+        mLogger.error("calcBuildPlan Add {}", p.mAlias);
+        p.mRipplePlan = usedByPackages(p, pkgCollection);
+        p.mRippleTime = p.mBuildTime;
         //  Use a ListIterator so that we can add elements to the end of the list while processing
+        //  Sum the buildTimes of the packages that we add to the list
         for (ListIterator<Package> it = p.mRipplePlan.listIterator(); it.hasNext(); )
+        {
             Package pkg = it.next();
+            mLogger.error("calc process {}", pkg.mAlias);
+            pkg.mRippleOrder++;
+            p.mRippleTime += pkg.mBuildTime;
-            ArrayList<Package> usedBy = usedByPackages(pkg);
+            ArrayList<Package> usedBy = usedByPackages(pkg, pkgCollection);
             for (Iterator<Package> it1 = usedBy.iterator(); it1.hasNext(); )
+            {
                 Package uPkg = it1.next();
+                uPkg.mRippleOrder = pkg.mRippleOrder + 1;
                 if ( ! p.mRipplePlan.contains( uPkg ))
+                {
                     it.add(uPkg);
+                    it.previous();
+                    p.mRippleTime += uPkg.mBuildTime;
+                    mLogger.error("calc Add {}", uPkg.mAlias);
+                }
+            }
+        }
         //
+        //  Need to determine the rippleTime for each package in the plan
+        //      Have a much smaller set of packages to process
-        //  Now have a plan on what will be affected when we build the package
+        //  Don't care about the mRipplePlan item in the packages
+        //  Don't modify the mRippleOrder
+        //
+        for (ListIterator<Package> it = p.mRipplePlan.listIterator(); it.hasNext(); )
+        {
+            Package pkg = it.next();
+            pkg.mRipplePlan = usedByPackages(pkg, pkgCollection);
+            pkg.mRippleTime = pkg.mBuildTime;
+            //  Use a ListIterator so that we can add elements to the end of the list while processing
-        //  Calculate the time it will take - include my time
+            //  Sum the buildTimes of the packages that we add to the list
+            for (ListIterator<Package> it0 = pkg.mRipplePlan.listIterator(); it0.hasNext(); )
+            {
+                Package pkg1 = it0.next();
+                pkg.mRippleTime += pkg1.mBuildTime;
+                ArrayList<Package> usedBy = usedByPackages(pkg1, pkgCollection);
+                for (Iterator<Package> it1 = usedBy.iterator(); it1.hasNext(); )
+                {
+                    Package uPkg = it1.next();
+                    if ( ! pkg.mRipplePlan.contains( uPkg ))
+                    {
+                        it0.add(uPkg);
+                        it0.previous();
+                        pkg.mRippleTime += uPkg.mBuildTime;
+                    }
+                }
+            }
+        }
         //
+        //  Sort the ripplePlan so that we build the fastest elements first
+        //  Only care about the ripplePlan for the main Package
+        //
+        Collections.sort(p.mRipplePlan, Package.PlanComparator);
-        p.mRippleTime = p.mBuildTime;
+        //  Debugging
+        mLogger.error("calcBuildPlan. Pkg:{} pvid:{}, Order:{} RippleT:{} BuildT:{}", p.mAlias, p.mId, p.mRippleOrder, p.mRippleTime, p.mBuildTime);
         for (ListIterator<Package> it = p.mRipplePlan.listIterator(); it.hasNext(); )
+        {
             Package pkg = it.next();
-            p.mRippleTime += pkg.mBuildTime;
+            mLogger.error("calcBuildPlan. Plan: Pkg:{} pvid:{}, Order:{} RippleT:{} BuildT:{}", pkg.mAlias, pkg.mId, pkg.mRippleOrder, pkg.mRippleTime, pkg.mBuildTime);
+        }
+    }
     /**
      *  Calculate a collection of packages that actively use the named package
      *      If the consumer is an SDK or is Pegged
      *      If the consumer is marked as advisoryRipple
      *      If the consumer cannot be built
+     *
      * @param pkg - Package to process
+     * @param pkgCollection - collection of packages to scan
      * @return A collection of packages that actively 'use' the specified package
      */
-    private ArrayList<Package> usedByPackages(Package pkg) {
+    private ArrayList<Package> usedByPackages(Package pkg, ArrayList<Package> pkgCollection) {
         ArrayList<Package> usedBy = new ArrayList<Package>();
-        for (Iterator<Package> it = mPackageCollection.iterator(); it.hasNext(); )
+        for (Iterator<Package> it = pkgCollection.iterator(); it.hasNext(); )
+        {
             Package p = it.next();
             //  Is this package 'actively used' in the current build
             if (p.mBuildFile >= 0)
+            {
                 for (Iterator<String> it2 = p.mDependencyCollection.iterator(); it2.hasNext(); )
+                {
                     String alias = it2.next();
-                    if ( alias == pkg.mAlias) {
+                    if (  pkg.mAlias.compareTo( alias ) == 0  ) {
                         //  Have found a consumer of 'pkg'
                         usedBy.add(p);
                         break;
+                    }
+                }
+        }
+    }
     /**
      * Find Package by package alias
-     * Searches the complete package collection
+     * Searches the released package collection
      * @param   alias               - alias of package to locate
      * @return  Package with the matching mAlias or NULL_PACKAGE if no package has the mAlias
      */
     public Package findPackage(String alias)
+    {
         Package retVal = mReleaseManager.findPackage(alias, mPackageCollection);
         mLogger.info("findPackage returned {}", retVal.mName);
         return retVal;
+    }
     /**
      * Sets the mBuildFile to -5 (indirectly dependent on package versions which are not reproducible)
      * for the package and all dependent packages that are a part of the release set.
+     *
      * @param p             The package being excluded
             //	Generate properties for each package in this build level or lower build levels
             //	The properties link the packageAlias to the PackageName and PackageVersion
             //
             //	[DEVI 54816] In escrow mode all unreproducible packages are included
-            for (Iterator<Package> it = mPackageCollectionAll.iterator(); it.hasNext(); )
+            for (Iterator<Package> it = mPackageCollection.iterator(); it.hasNext(); )
+            {
                 Package p = it.next();
                 if ( ( ( p.mBuildFile > 0 ) && ( p.mBuildFile <= buildFile ) ) || ( !mDaemon && p.mBuildFile == -2 ) )
+                {
             if ( ! mReleaseManager.mUseDatabase )
+            {
                 // UTF Support
                 // Insert per-package planning information
                 //
-                xml.addComment("mPackageCollectionAll");
+                xml.addComment("mPackageCollection");
-                for (Iterator<Package> it = mPackageCollectionAll.iterator(); it.hasNext(); )
+                for (Iterator<Package> it = mPackageCollection.iterator(); it.hasNext(); )
+                {
                     Package p = it.next();
                     generatePackageInfo(xml, p);
+                }
+                xml.addComment("mPackageCollectionWip");
+                for (Iterator<Package> it = mPackageCollectionWip.iterator(); it.hasNext(); )
+                {
+                    Package p = it.next();
+                    if (p.mIsNotReleased )
+                        generatePackageInfo(xml, p);
+                }
+                xml.addComment("mPackageCollectionTest");
+                for (Iterator<Package> it = mPackageCollectionTest.iterator(); it.hasNext(); )
+                {
+                    Package p = it.next();
+                    if (p.mIsNotReleased )
+                        generatePackageInfo(xml, p);
+                }
+                xml.addComment("mPackageCollectionTestRipple");
+                for (Iterator<Package> it = mPackageCollectionRipple.iterator(); it.hasNext(); )
+                {
+                    Package p = it.next();
+                    if (p.mIsNotReleased )
+                        generatePackageInfo(xml, p);
+                }
                 // UTF Support
                 // Insert build exclusion information
                 xml.addComment("mBuildExclusionCollection");
                 for (Iterator<BuildExclusion> it = mBuildExclusionCollection.iterator(); it.hasNext(); )
+                {
+                    {
                         Package p = it.next();
                         String comment =
                                 "pvid="+ p.mId +
                                 " order=" + (++order) +
+                                " Rorder=" + (p.mRippleOrder) +
+                                " time=" + p.mRippleTime +
                                 " name=\"" + p.mAlias + "\"";
                         xml.addComment(comment);
+                    }
+                }
+            }

Subversion Repositories DevTools

(root)/buildtool/branches/planMetrics/src/main/java/com/erggroup/buildtool/ripple/RippleEngine.java – Rev 7052 → 7070