Subversion Repositories DevTools

import json, sys, datetime, time

from urllib.request import urlopen, Request

CRUCIBLE_URL_TEMPLATE = 'http://cloudasvn03:8060/rest-service/reviews-v1'

RESULTS_KEY = 'results'

GEOMETRY_KEY = 'geometry'

LOCATION_KEY = 'location'

LAT_KEY = 'lat'

LNG_KEY = 'lng'

def trace(*objs) :

    print(*objs,file=sys.stderr)

    sys.stderr.flush()

    pass

class CrucibleExtractor :

    def __init__(self) :

        pass

        self.selected_reviews = {}

        self.runtime_defects = {}

        self.maintenance_defects = {}

        self.security_defects = {}

        self.unclassified_defect_reviews = [ ]

        self.total_defects = 0

        self.verbose = False

    def url_to_object(self, url, top=None) :

        trace(url)

        req = Request(url)

        req.add_header('Accept','application/json')

        req.add_header('Authorization', 'Basic dGxpdHRsZWY6MCRtb3JQRVRI')

        resp = urlopen(req).read()

        retval = json.loads(str(resp,'utf-8'))

        if top : 

            retval = retval[top]

        return retval

    def get_selected_review_ids(self,min_create_date, max_create_date) :

        date_min = datetime.datetime.strptime( min_create_date.replace("-00",""), "%Y-%m" )

        # epoch_min will be sent to Crucible to filter the reviews we want to look at.

        # We back off one day to account for timezones (the records returned will be

        # filtered on create date anyway)

        epoch_min = date_min.timestamp()-24*60*60

        url = CRUCIBLE_URL_TEMPLATE + "/filter?fromDate=%d"%(epoch_min*1000,)

        reviewData = self.url_to_object(url,"reviewData")

        for r in reviewData :

            review_id = r["permaId"]["id"] 

            review_create_date = r["createDate"]

            if review_create_date>=min_create_date and review_create_date<=max_create_date :

                self.selected_reviews[review_id] = [ r['projectKey'], None, ]

            else :

                trace("Excluding review ",review_id)

                pass

        return self.selected_reviews.keys()

    def get_review_summary(self,review_id) :

        url = CRUCIBLE_URL_TEMPLATE + "/" + review_id + "/comments"

        projectKey = str(self.selected_reviews[review_id][0])

        reviewDetails = self.url_to_object(url,'comments')

        retval = [ 

            projectKey,

            0, # non-defect comments

            0, # runtime defects

            0, # maintenance defects

            0, # security defects

            0, # unclassified defects

        ]

        for c in reviewDetails:

            metrics_keys = list(c['metrics'].keys())

            if c['defectRaised'] is False :

                retval[1] += 1

            elif len(metrics_keys) != 1 : # unclassified - either none of the drop downs or more than one selected

                retval[5] += 1 

                self.unclassified_defect_reviews += [ review_id ]

                self.total_defects += 1

            else :

                defect_category_code =  metrics_keys[0]

                defect_type = c['metrics'][defect_category_code]['value']

                trace(defect_category_code, defect_type)

                category_defect_map = None

                if defect_category_code == 'metric-0' : # maintenance

                    retval[3] += 1

                    category_defect_map = self.maintenance_defects

                elif defect_category_code == 'metric-1' : # runtime

                    retval[2] += 1

                    category_defect_map = self.runtime_defects

                elif defect_category_code == 'metric-97' : # security

                    retval[4] += 1

                    category_defect_map = self.security_defects

                else :

                    trace("Unexpected defect category %s (defect type is %s)" % ( defect_category_code, defect_type))

                    retval[5] += 1 

                    self.unclassified_defect_reviews += [ review_id ]

                    self.total_defects += 1

                if category_defect_map is None :

                    pass

                elif defect_type in category_defect_map.keys() :

                    category_defect_map[defect_type] += 1

                else :

                    category_defect_map[defect_type] = 1

                self.total_defects += 1

        self.selected_reviews[review_id][1] = retval[2:]    

        return retval

    def summarize_by_project(self, month_prefix) :

        start_date = month_prefix + "-00"

        end_date = month_prefix + "-99"

        if self.verbose : 

            trace("Getting review ids")

        review_ids = self.get_selected_review_ids(start_date, end_date)

        project_summaries = {}

        overall_summary = [ 0, ] * 6

        for review_id in review_ids :

            if True :

                review_summary = self.get_review_summary(review_id)

                if self.verbose : 

                    trace("%-12s: %s" % ( review_id, self.selected_reviews[review_id][1], ))

                project_id = review_summary[0]

                if  project_id in project_summaries.keys() :

                    project_summary = project_summaries[project_id]

                else :

                    project_summary = [ 0,] * 6

                project_summary[0] += 1

                overall_summary[0] += 1

                for i in range(1,6) :

                    project_summary[i] += review_summary[i]

                    overall_summary[i] += review_summary[i]

                project_summaries[project_id] = project_summary

        REPORT_PRINTF_FORMAT = "%-12s %4s %4s %4s %4s %4s %4s"

        REPORT_FORMAT_SEPARATOR = ( '-------','---','---','---','---','---','---')

        print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )

        print ( REPORT_PRINTF_FORMAT % ( "Project", "rev", "com", "run", "mnt", "sec", "oth" ) )

        print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR ) 

        for project_id in sorted(project_summaries.keys()) :

            ps = project_summaries[project_id]

            print ( REPORT_PRINTF_FORMAT % ( project_id, ps[0], ps[1], ps[2], ps[3], ps[4], ps[5] ) )

        print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )

        ps = overall_summary

        print ( REPORT_PRINTF_FORMAT % ( "TOTAL", ps[0], ps[1], ps[2], ps[3], ps[4], ps[5] ) )

    def summarize_for_category(self,category_name, category_types) :

        REPORT_PRINTF_FORMAT = "%-75s %6s %6s"

        REPORT_FORMAT_SEPARATOR = ( '-----------------------','------','------')

        print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )

        print ( REPORT_PRINTF_FORMAT % ( "%s defects" % (category_name),"count","%") )

        print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )

        count_for_category = 0

        percent_for_category = 0.0

        for d in sorted(category_types.keys()) :

            count_for_type = category_types[d]

            percent_for_type = int( (1000.0 * count_for_type)/self.total_defects) * 0.1

            count_for_category += count_for_type

            percent_for_category += percent_for_type

            d = d.replace("\u2013","-")

            print ( REPORT_PRINTF_FORMAT % ( 

                d, 

                "%6d" % (count_for_type,),

                "%6.1f" % (percent_for_type,), 

            ) )

        print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )

        print ( REPORT_PRINTF_FORMAT % ( 

            "Total %s" % (category_name,), 

            "%6d" % (count_for_category,),

            "%6.1f" % (percent_for_category,),

        ) )

        print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR ) 

    def summarize_by_defect_type(self) :

        self.summarize_for_category("maintenance",self.maintenance_defects)

        self.summarize_for_category("runtime",self.runtime_defects)

        self.summarize_for_category("security",self.security_defects)

    def report_on_unclassified_defects(self) : 

        trace ( "Reviews with unclassified defects: %s" % (self.unclassified_defect_reviews,) )

if __name__ == "__main__" :

    extractor = CrucibleExtractor()

    if len(sys.argv)>=2 :

        if sys.argv[1] == "--usage" :

            trace ( "crucible_reporting.py YYYY-MM" )

            trace ( "   report stats for specified month" )

            sys.exit(0)

        elif sys.argv[1] == "--verbose" :

            extractor.verbose = True

            sys.argv = sys.argv[1:]

    if len(sys.argv)<2 :

        today = datetime.date.today()

        if today.month!= 1 :

            month_prefix = "%04d-%02d" % (today.year, today.month-1)

        else :

            month_prefix = "%04d-%02d" % (today.year-1, 12)

        trace ( "Generating report for previous month (" + month_prefix + ")" )

    else :

        month_prefix = sys.argv[1]

        trace ( "Generating report for " + month_prefix )

    trace("Starting at",time.time())

    sys.stdout = open("%s.txt"%(month_prefix,),"w")

    extractor.summarize_by_project(month_prefix)

    extractor.summarize_by_defect_type()

    extractor.report_on_unclassified_defects()

    trace("Finished at",time.time())