Rev 5298 | Blame | Compare with Previous | Last modification | View Log | RSS feed
import json, sys, datetime, timefrom urllib.request import urlopen, RequestCRUCIBLE_URL_TEMPLATE = 'http://cloudasvn03:8060/rest-service/reviews-v1'RESULTS_KEY = 'results'GEOMETRY_KEY = 'geometry'LOCATION_KEY = 'location'LAT_KEY = 'lat'LNG_KEY = 'lng'def trace(*objs) :print(*objs,file=sys.stderr)sys.stderr.flush()passclass CrucibleExtractor :def __init__(self) :passself.selected_reviews = {}self.runtime_defects = {}self.maintenance_defects = {}self.security_defects = {}self.unclassified_defect_reviews = [ ]self.total_defects = 0self.verbose = Falsedef url_to_object(self, url, top=None) :trace(url)req = Request(url)req.add_header('Accept','application/json')req.add_header('Authorization', 'Basic dGxpdHRsZWY6MCRtb3JQRVRI')resp = urlopen(req).read()retval = json.loads(str(resp,'utf-8'))if top :retval = retval[top]return retvaldef get_selected_review_ids(self,min_create_date, max_create_date) :date_min = datetime.datetime.strptime( min_create_date.replace("-00",""), "%Y-%m" )# epoch_min will be sent to Crucible to filter the reviews we want to look at.# We back off one day to account for timezones (the records returned will be# filtered on create date anyway)epoch_min = date_min.timestamp()-24*60*60url = CRUCIBLE_URL_TEMPLATE + "/filter?fromDate=%d"%(epoch_min*1000,)reviewData = self.url_to_object(url,"reviewData")for r in reviewData :review_id = r["permaId"]["id"]review_create_date = r["createDate"]if review_create_date>=min_create_date and review_create_date<=max_create_date :self.selected_reviews[review_id] = [ r['projectKey'], None, ]else :trace("Excluding review ",review_id)passreturn self.selected_reviews.keys()def get_review_summary(self,review_id) :url = CRUCIBLE_URL_TEMPLATE + "/" + review_id + "/comments"projectKey = str(self.selected_reviews[review_id][0])reviewDetails = self.url_to_object(url,'comments')retval = [projectKey,0, # non-defect comments0, # runtime defects0, # maintenance defects0, # security defects0, # unclassified defects]for c in reviewDetails:metrics_keys = list(c['metrics'].keys())if c['defectRaised'] is False :retval[1] += 1elif len(metrics_keys) != 1 : # unclassified - either none of the drop downs or more than one selectedretval[5] += 1self.unclassified_defect_reviews += [ review_id ]self.total_defects += 1else :defect_category_code = metrics_keys[0]defect_type = c['metrics'][defect_category_code]['value']trace(defect_category_code, defect_type)category_defect_map = Noneif defect_category_code == 'metric-0' : # maintenanceretval[3] += 1category_defect_map = self.maintenance_defectselif defect_category_code == 'metric-1' : # runtimeretval[2] += 1category_defect_map = self.runtime_defectselif defect_category_code == 'metric-97' : # securityretval[4] += 1category_defect_map = self.security_defectselse :trace("Unexpected defect category %s (defect type is %s)" % ( defect_category_code, defect_type))retval[5] += 1self.unclassified_defect_reviews += [ review_id ]self.total_defects += 1if category_defect_map is None :passelif defect_type in category_defect_map.keys() :category_defect_map[defect_type] += 1else :category_defect_map[defect_type] = 1self.total_defects += 1self.selected_reviews[review_id][1] = retval[2:]return retvaldef summarize_by_project(self, month_prefix) :start_date = month_prefix + "-00"end_date = month_prefix + "-99"if self.verbose :trace("Getting review ids")review_ids = self.get_selected_review_ids(start_date, end_date)project_summaries = {}overall_summary = [ 0, ] * 6for review_id in review_ids :if True :review_summary = self.get_review_summary(review_id)if self.verbose :trace("%-12s: %s" % ( review_id, self.selected_reviews[review_id][1], ))project_id = review_summary[0]if project_id in project_summaries.keys() :project_summary = project_summaries[project_id]else :project_summary = [ 0,] * 6project_summary[0] += 1overall_summary[0] += 1for i in range(1,6) :project_summary[i] += review_summary[i]overall_summary[i] += review_summary[i]project_summaries[project_id] = project_summaryREPORT_PRINTF_FORMAT = "%-12s %4s %4s %4s %4s %4s %4s"REPORT_FORMAT_SEPARATOR = ( '-------','---','---','---','---','---','---')print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )print ( REPORT_PRINTF_FORMAT % ( "Project", "rev", "com", "run", "mnt", "sec", "oth" ) )print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )for project_id in sorted(project_summaries.keys()) :ps = project_summaries[project_id]print ( REPORT_PRINTF_FORMAT % ( project_id, ps[0], ps[1], ps[2], ps[3], ps[4], ps[5] ) )print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )ps = overall_summaryprint ( REPORT_PRINTF_FORMAT % ( "TOTAL", ps[0], ps[1], ps[2], ps[3], ps[4], ps[5] ) )def summarize_for_category(self,category_name, category_types) :REPORT_PRINTF_FORMAT = "%-75s %6s %6s"REPORT_FORMAT_SEPARATOR = ( '-----------------------','------','------')print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )print ( REPORT_PRINTF_FORMAT % ( "%s defects" % (category_name),"count","%") )print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )count_for_category = 0percent_for_category = 0.0for d in sorted(category_types.keys()) :count_for_type = category_types[d]percent_for_type = int( (1000.0 * count_for_type)/self.total_defects) * 0.1count_for_category += count_for_typepercent_for_category += percent_for_typed = d.replace("\u2013","-")print ( REPORT_PRINTF_FORMAT % (d,"%6d" % (count_for_type,),"%6.1f" % (percent_for_type,),) )print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )print ( REPORT_PRINTF_FORMAT % ("Total %s" % (category_name,),"%6d" % (count_for_category,),"%6.1f" % (percent_for_category,),) )print ( REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR )def summarize_by_defect_type(self) :self.summarize_for_category("maintenance",self.maintenance_defects)self.summarize_for_category("runtime",self.runtime_defects)self.summarize_for_category("security",self.security_defects)def report_on_unclassified_defects(self) :trace ( "Reviews with unclassified defects: %s" % (self.unclassified_defect_reviews,) )if __name__ == "__main__" :extractor = CrucibleExtractor()if len(sys.argv)>=2 :if sys.argv[1] == "--usage" :trace ( "crucible_reporting.py YYYY-MM" )trace ( " report stats for specified month" )sys.exit(0)elif sys.argv[1] == "--verbose" :extractor.verbose = Truesys.argv = sys.argv[1:]if len(sys.argv)<2 :today = datetime.date.today()if today.month!= 1 :month_prefix = "%04d-%02d" % (today.year, today.month-1)else :month_prefix = "%04d-%02d" % (today.year-1, 12)trace ( "Generating report for previous month (" + month_prefix + ")" )else :month_prefix = sys.argv[1]trace ( "Generating report for " + month_prefix )trace("Starting at",time.time())sys.stdout = open("%s.txt"%(month_prefix,),"w")extractor.summarize_by_project(month_prefix)extractor.summarize_by_defect_type()extractor.report_on_unclassified_defects()trace("Finished at",time.time())