| 4381 |
tlittlef |
1 |
import json, urllib2, sys, datetime
|
| 4380 |
tlittlef |
2 |
|
|
|
3 |
CRUCIBLE_URL_TEMPLATE = 'http://cds:8060/rest-service/reviews-v1'
|
|
|
4 |
RESULTS_KEY = u'results'
|
|
|
5 |
GEOMETRY_KEY = u'geometry'
|
|
|
6 |
LOCATION_KEY = u'location'
|
|
|
7 |
LAT_KEY = u'lat'
|
|
|
8 |
LNG_KEY = u'lng'
|
|
|
9 |
|
|
|
10 |
|
|
|
11 |
class CrucibleExtractor :
|
|
|
12 |
def __init__(self) :
|
|
|
13 |
pass
|
|
|
14 |
self.selected_reviews = {}
|
|
|
15 |
self.runtime_defects = {}
|
|
|
16 |
self.maintenance_defects = {}
|
|
|
17 |
self.security_defects = {}
|
|
|
18 |
self.unclassified_defect_reviews = [ ]
|
|
|
19 |
self.total_defects = 0
|
|
|
20 |
def url_to_object(self, url, top=None) :
|
|
|
21 |
req = urllib2.Request(url)
|
|
|
22 |
req.add_header('Accept','application/json')
|
|
|
23 |
req.add_header('Authorization', 'Basic dGxpdHRsZWY6MCRtb3JQRVRI')
|
|
|
24 |
resp = urllib2.urlopen(req).read()
|
|
|
25 |
retval = json.loads(resp)
|
|
|
26 |
if top :
|
|
|
27 |
retval = retval[top]
|
|
|
28 |
return retval
|
|
|
29 |
def get_selected_review_ids(self,min_create_date, max_create_date) :
|
|
|
30 |
url = CRUCIBLE_URL_TEMPLATE
|
|
|
31 |
reviewData = self.url_to_object(url, "reviewData")
|
|
|
32 |
for r in reviewData :
|
|
|
33 |
review_id = r["permaId"]["id"]
|
|
|
34 |
review_create_date = r["createDate"]
|
|
|
35 |
if review_create_date>=min_create_date and review_create_date<=max_create_date :
|
|
|
36 |
self.selected_reviews[review_id] = [ r[u'projectKey'], None, ]
|
|
|
37 |
else :
|
|
|
38 |
pass
|
|
|
39 |
return self.selected_reviews.keys()
|
|
|
40 |
def get_review_summary(self,review_id) :
|
|
|
41 |
url = CRUCIBLE_URL_TEMPLATE + "/" + review_id + "/comments"
|
|
|
42 |
projectKey = str(self.selected_reviews[review_id][0])
|
|
|
43 |
reviewDetails = self.url_to_object(url,u'comments')
|
|
|
44 |
retval = [
|
|
|
45 |
projectKey,
|
|
|
46 |
0, # non-defect comments
|
|
|
47 |
0, # runtime defects
|
|
|
48 |
0, # maintenance defects
|
|
|
49 |
0, # security defects
|
|
|
50 |
0, # unclassified defects
|
|
|
51 |
]
|
|
|
52 |
for c in reviewDetails:
|
|
|
53 |
if c[u'defectRaised'] is False :
|
|
|
54 |
retval[1] += 1
|
|
|
55 |
elif len(c[u'metrics'].keys()) != 1 : # unclassified - either none of the drop downs or more than one selected
|
|
|
56 |
retval[5] += 1
|
|
|
57 |
self.unclassified_defect_reviews += [ review_id ]
|
|
|
58 |
self.total_defects += 1
|
|
|
59 |
else :
|
|
|
60 |
defect_category_code = c[u'metrics'].keys()[0]
|
|
|
61 |
defect_type = c[u'metrics'][defect_category_code][u'value']
|
|
|
62 |
category_defect_map = None
|
|
|
63 |
if defect_category_code == u'metric-57' : # maintenance
|
|
|
64 |
retval[3] += 1
|
|
|
65 |
category_defect_map = self.maintenance_defects
|
|
|
66 |
elif defect_category_code == u'metric-95' : # runtime
|
|
|
67 |
retval[2] += 1
|
|
|
68 |
category_defect_map = self.runtime_defects
|
|
|
69 |
else :
|
|
|
70 |
print c[u'metrics'].keys()[0] # security - haven't seen one yet so want to know what the metric- code is
|
|
|
71 |
retval[4] += 1
|
|
|
72 |
category_defect_map = self.security_defects
|
|
|
73 |
if defect_type in category_defect_map.keys() :
|
|
|
74 |
category_defect_map[defect_type] += 1
|
|
|
75 |
else :
|
|
|
76 |
category_defect_map[defect_type] = 1
|
|
|
77 |
|
|
|
78 |
self.total_defects += 1
|
|
|
79 |
self.selected_reviews[review_id][1] = retval[2:]
|
|
|
80 |
return retval
|
| 4381 |
tlittlef |
81 |
def summarize_by_project(self, month_prefix) :
|
|
|
82 |
start_date = month_prefix + "-00"
|
|
|
83 |
end_date = month_prefix + "-99"
|
| 4380 |
tlittlef |
84 |
print "Getting review ids"
|
|
|
85 |
review_ids = self.get_selected_review_ids(start_date, end_date)
|
|
|
86 |
project_summaries = {}
|
|
|
87 |
overall_summary = [ 0, ] * 6
|
|
|
88 |
for review_id in sorted(review_ids) :
|
|
|
89 |
if True :
|
|
|
90 |
review_summary = self.get_review_summary(review_id)
|
|
|
91 |
print "%-12s: %s" % ( review_id, self.selected_reviews[review_id][1] )
|
|
|
92 |
project_id = review_summary[0]
|
|
|
93 |
if project_id in project_summaries.keys() :
|
|
|
94 |
project_summary = project_summaries[project_id]
|
|
|
95 |
else :
|
|
|
96 |
project_summary = [ 0,] * 6
|
|
|
97 |
project_summary[0] += 1
|
|
|
98 |
overall_summary[0] += 1
|
|
|
99 |
for i in range(1,6) :
|
|
|
100 |
project_summary[i] += review_summary[i]
|
|
|
101 |
overall_summary[i] += review_summary[i]
|
|
|
102 |
project_summaries[project_id] = project_summary
|
|
|
103 |
REPORT_PRINTF_FORMAT = "%-12s %4s %4s %4s %4s %4s %4s"
|
|
|
104 |
REPORT_FORMAT_SEPARATOR = ( '-------','---','---','---','---','---','---')
|
|
|
105 |
print REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR
|
|
|
106 |
print REPORT_PRINTF_FORMAT % ( "Project", "rev", "com", "run", "mnt", "sec", "oth" )
|
|
|
107 |
print REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR
|
|
|
108 |
for project_id in sorted(project_summaries.keys()) :
|
|
|
109 |
ps = project_summaries[project_id]
|
|
|
110 |
print REPORT_PRINTF_FORMAT % ( project_id, ps[0], ps[1], ps[2], ps[3], ps[4], ps[5] )
|
|
|
111 |
print REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR
|
|
|
112 |
ps = overall_summary
|
|
|
113 |
print REPORT_PRINTF_FORMAT % ( "TOTAL", ps[0], ps[1], ps[2], ps[3], ps[4], ps[5] )
|
|
|
114 |
def summarize_for_category(self,category_name, category_types) :
|
|
|
115 |
REPORT_PRINTF_FORMAT = "%-75s %6s %6s"
|
|
|
116 |
REPORT_FORMAT_SEPARATOR = ( '-----------------------','------','------')
|
|
|
117 |
print REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR
|
|
|
118 |
print REPORT_PRINTF_FORMAT % ( "%s defects" % (category_name),"count","%")
|
|
|
119 |
print REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR
|
|
|
120 |
count_for_category = 0
|
|
|
121 |
percent_for_category = 0.0
|
|
|
122 |
for d in sorted(category_types.keys()) :
|
|
|
123 |
count_for_type = category_types[d]
|
|
|
124 |
percent_for_type = int( (1000.0 * count_for_type)/self.total_defects) * 0.1
|
|
|
125 |
count_for_category += count_for_type
|
|
|
126 |
percent_for_category += percent_for_type
|
|
|
127 |
d = d.replace(u"\u2013",u"-")
|
|
|
128 |
print REPORT_PRINTF_FORMAT % ( d, count_for_type, percent_for_type )
|
|
|
129 |
print REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR
|
|
|
130 |
print REPORT_PRINTF_FORMAT % ( "Total %s" % (category_name,), count_for_category, percent_for_category )
|
|
|
131 |
print REPORT_PRINTF_FORMAT % REPORT_FORMAT_SEPARATOR
|
|
|
132 |
def summarize_by_defect_type(self) :
|
|
|
133 |
self.summarize_for_category("maintenance",self.maintenance_defects)
|
|
|
134 |
self.summarize_for_category("runtime",self.runtime_defects)
|
|
|
135 |
self.summarize_for_category("security",self.security_defects)
|
|
|
136 |
def report_on_unclassified_defects(self) :
|
|
|
137 |
print "Reviews with unclassified defects: %s" % (self.unclassified_defect_reviews,)
|
|
|
138 |
|
|
|
139 |
if __name__ == "__main__" :
|
|
|
140 |
extractor = CrucibleExtractor()
|
| 4381 |
tlittlef |
141 |
if len(sys.argv) < 2 :
|
|
|
142 |
today = datetime.date.today()
|
|
|
143 |
if today.month!= 1 :
|
|
|
144 |
month_prefix = "%04d-%02d" % (today.year, today.month-1)
|
|
|
145 |
else :
|
|
|
146 |
month_prefix = "%04d-%02d" % (today.year-1, 12)
|
|
|
147 |
print "Generating report for previous month (" + month_prefix + ")"
|
|
|
148 |
elif sys.argv[1] == "--usage" :
|
|
|
149 |
print "crucible_reporting.py YYYY-MM"
|
|
|
150 |
print " report stats for specified month"
|
|
|
151 |
print "crucible_reporting.py"
|
|
|
152 |
print " report stats for previous month"
|
|
|
153 |
sys.exit(0)
|
|
|
154 |
else :
|
|
|
155 |
month_prefix = sys.argv[1]
|
|
|
156 |
|
|
|
157 |
extractor.summarize_by_project(month_prefix)
|
| 4380 |
tlittlef |
158 |
extractor.summarize_by_defect_type()
|
|
|
159 |
extractor.report_on_unclassified_defects()
|
|
|
160 |
|
|
|
161 |
|