source: rtems-tools/tester/rt/coverage.py @ b762312

Last change on this file since b762312 was b762312, checked in by Vijay Kumar Banerjee <vijaykumar9597@…>, on Jun 4, 2018 at 8:44:43 PM

tester: Add script to generate html coverage report from covoar output

Add support in tester to run covoar and generate an html report to display
the summary of the coverage reports generated from covoar.

Co-authored-by : Cillian O'Donnell <cpodonnell8@…>

  • Property mode set to 100644
File size: 15.6 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2014 Krzysztof Miesowicz (krzysztof.miesowicz@gmail.com)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from rtemstoolkit import error
32from rtemstoolkit import path
33from rtemstoolkit import log
34from rtemstoolkit import execute
35from rtemstoolkit import macros
36
37from datetime import datetime
38
39from . import options
40
41import shutil
42import os
43
44try:
45    import configparser
46except:
47    import ConfigParser as configparser
48
49class summary:
50    def __init__(self, p_summary_dir):
51        self.summary_file_path = path.join(p_summary_dir, 'summary.txt')
52        self.index_file_path = path.join(p_summary_dir, 'index.html')
53        self.bytes_analyzed = 0
54        self.bytes_not_executed = 0
55        self.percentage_executed = 0.0
56        self.percentage_not_executed = 100.0
57        self.ranges_uncovered = 0
58        self.branches_uncovered = 0
59        self.branches_total = 0
60        self.branches_always_taken = 0
61        self.branches_never_taken = 0
62        self.percentage_branches_covered = 0.0
63        self.is_failure = False
64
65    def parse(self):
66        if(not path.exists(self.summary_file_path)):
67            log.notice('summary file %s does not exist!' % (self.summary_file_path))
68            self.is_failure = True
69            return
70
71        with open(self.summary_file_path,'r') as summary_file:
72           self.bytes_analyzed = self._get_next_with_colon(summary_file)
73           self.bytes_not_executed = self._get_next_with_colon(summary_file)
74           self.percentage_executed = self._get_next_with_colon(summary_file)
75           self.percentage_not_executed = self._get_next_with_colon(summary_file)
76           self.ranges_uncovered = self._get_next_with_colon(summary_file)
77           self.branches_total = self._get_next_with_colon(summary_file)
78           self.branches_uncovered = self._get_next_with_colon(summary_file)
79           self.branches_always_taken = self._get_next_without_colon(summary_file)
80           self.branches_never_taken = self._get_next_without_colon(summary_file)
81        if len(self.branches_uncovered) > 0 and len(self.branches_total) > 0:
82            self.percentage_branches_covered = \
83            1 - (float(self.branches_uncovered) / float(self.branches_total))
84        else:
85            self.percentage_branches_covered = 0.0
86        return
87
88    def _get_next_with_colon(self, summary_file):
89        line = summary_file.readline()
90        if ':' in line:
91            return line.split(':')[1].strip()
92        else:
93            return ''
94
95    def _get_next_without_colon(self, summary_file):
96        line = summary_file.readline()
97        return line.strip().split(' ')[0]
98
99class report_gen_html:
100    def __init__(self, p_symbol_sets_list, build_dir, rtdir, bsp):
101        self.symbol_sets_list = ['score']
102        self.build_dir = build_dir
103        self.partial_reports_files = list(["index.html", "summary.txt"])
104        self.number_of_columns = 1
105        self.covoar_src_path = path.join(rtdir, 'covoar')
106        self.bsp = bsp
107
108    def _find_partial_reports(self):
109        partial_reports = {}
110        for symbol_set in self.symbol_sets_list:
111            set_summary = summary(path.join(self.build_dir,
112                                  self.bsp + "-coverage",
113                                  symbol_set))
114            set_summary.parse()
115            partial_reports[symbol_set] = set_summary
116        return partial_reports
117
118    def _prepare_head_section(self):
119        head_section = '''
120        <head>
121        <title>RTEMS coverage report</title>
122        <style type="text/css">
123            progress[value] {
124              -webkit-appearance: none;
125               appearance: none;
126
127              width: 150px;
128              height: 15px;
129            }
130        </style>
131        </head>'''
132        return head_section
133
134    def _prepare_index_content(self, partial_reports):
135        header = "<h1> RTEMS coverage analysis report </h1>"
136        header += "<h3>Coverage reports by symbols sets:</h3>"
137        table = "<table>"
138        table += self._header_row()
139        for symbol_set in partial_reports:
140            table += self._row(symbol_set, partial_reports[symbol_set])
141        table += "</table> </br>"
142        timestamp = "Analysis performed on " + datetime.now().ctime()
143        return "<body>\n" + header + table + timestamp + "\n</body>"
144
145    def _row(self, symbol_set, summary):
146        row = "<tr>"
147        row += "<td>" + symbol_set + "</td>"
148        if summary.is_failure:
149            row += ' <td colspan="' + str(self.number_of_columns-1) \
150            + '" style="background-color:red">FAILURE</td>'
151        else:
152            row += " <td>" + self._link(summary.index_file_path,"Index") \
153            + "</td>"
154            row += " <td>" + self._link(summary.summary_file_path,"Summary") \
155            + "</td>"
156            row += " <td>" + summary.bytes_analyzed + "</td>"
157            row += " <td>" + summary.bytes_not_executed + "</td>"
158            row += " <td>" + summary.ranges_uncovered + "</td>"
159            row += " <td>" + summary.percentage_executed + "%</td>"
160            row += " <td>" + summary.percentage_not_executed + "%</td>"
161            row += ' <td><progress value="' + summary.percentage_executed \
162            + '" max="100"></progress></td>'
163            row += " <td>" + summary.branches_uncovered + "</td>"
164            row += " <td>" + summary.branches_total + "</td>"
165            row += " <td> {:.3%} </td>".format(summary.percentage_branches_covered)
166            row += ' <td><progress value="{:.3}" max="100"></progress></td>'.format(100*summary.percentage_branches_covered)
167            row += "</tr>\n"
168        return row
169
170    def _header_row(self):
171        row = "<tr>"
172        row += "<th> Symbols set name </th>"
173        row += "<th> Index file </th>"
174        row += "<th> Summary file </th>"
175        row += "<th> Bytes analyzed </th>"
176        row += "<th> Bytes not executed </th>"
177        row += "<th> Uncovered ranges </th>"
178        row += "<th> Percentage covered </th>"
179        row += "<th> Percentage uncovered </th>"
180        row += "<th> Instruction coverage </th>"
181        row += "<th> Branches uncovered </th>"
182        row += "<th> Branches total </th>"
183        row += "<th> Branches covered percentage </th>"
184        row += "<th> Branches coverage </th>"
185        row += "</tr>\n"
186        self.number_of_columns = row.count('<th>')
187        return row
188
189    def _link(self, address, text):
190        return '<a href="' + address + '">' + text + '</a>'
191
192    def _create_index_file(self, head_section, content):
193        with open(path.join(self.build_dir,
194                            self.bsp + "-report.html"),'w') as f:
195            f.write(head_section)
196            f.write(content)
197
198    def generate(self):
199        partial_reports = self._find_partial_reports()
200        head_section = self._prepare_head_section()
201        index_content = self._prepare_index_content(partial_reports)
202        self._create_index_file(head_section,index_content)
203
204    def add_covoar_src_path(self):
205        table_js_path = path.join(self.covoar_src_path, 'table.js')
206        covoar_css_path = path.join(self.covoar_src_path, 'covoar.css')
207        for symbol_set in self.symbol_sets_list:
208            symbol_set_dir = path.join(self.build_dir,
209                                       self.bsp + '-coverage', symbol_set)
210            html_files = os.listdir(symbol_set_dir)
211            for html_file in html_files:
212                html_file = path.join(symbol_set_dir, html_file)
213                if path.exists(html_file) and 'html' in html_file:
214                    with open(html_file, 'r') as f:
215                        file_data = f.read()
216                    file_data = file_data.replace('table.js', table_js_path)
217                    file_data = file_data.replace('covoar.css',
218                                                  covoar_css_path)
219                    with open(html_file, 'w') as f:
220                        f.write(file_data)
221
222class build_path_generator(object):
223    '''
224    Generates the build path from the path to executables
225    '''
226    def __init__(self, executables, target):
227        self.executables = executables
228        self.target = target
229    def run(self):
230        build_path = '/'
231        Path = self.executables[0].split('/')
232        for P in Path:
233            if P == self.target:
234                break;
235            else:
236                build_path = path.join(build_path, P)
237        return build_path
238
239class symbol_parser(object):
240    '''
241    Parse the symbol sets ini and create custom ini file for covoar
242    '''
243    def __init__(self, symbol_config_path,
244                 symbol_select_path, coverage_arg, build_dir):
245        self.symbol_select_file = symbol_select_path
246        self.symbol_file = symbol_config_path
247        self.build_dir = build_dir
248        self.symbol_sets = {}
249        self.cov_arg = coverage_arg
250        self.ssets = []
251
252    def parse(self):
253        config = configparser.ConfigParser()
254        try:
255            config.read(self.symbol_file)
256            if self.cov_arg:
257                self.ssets = self.cov_arg.split(',')
258            else:
259                self.ssets = config.get('symbol-sets', 'sets').split(',')
260                self.ssets = [ sset.encode('utf-8') for sset in self.ssets]
261            for sset in self.ssets:
262                lib = path.join(self.build_dir,
263                                config.get('libraries', sset))
264                self.symbol_sets[sset] = lib.encode('utf-8')
265        except:
266            raise error.general('Symbol set parsing failed')
267
268    def _write_ini(self):
269        config = configparser.ConfigParser()
270        try:
271            sets = ', '.join(self.symbol_sets.keys())
272            config.add_section('symbol-sets')
273            config.set('symbol-sets', 'sets', sets)
274            for key in self.symbol_sets.keys():
275                config.add_section(key)
276                config.set(key, 'libraries', self.symbol_sets[key])
277            with open(self.symbol_select_file, 'w') as conf:
278                config.write(conf)
279        except:
280            raise error.general('write failed')
281
282    def run(self):
283        self.parse()
284        self._write_ini()
285
286class covoar(object):
287    '''
288    Covoar runner
289    '''
290    def __init__(self, base_result_dir, config_dir, executables, explanations_txt):
291        self.base_result_dir = base_result_dir
292        self.config_dir = config_dir
293        self.executables = ' '.join(executables)
294        self.explanations_txt = explanations_txt
295        self.project_name = 'RTEMS-5'
296
297    def run(self, set_name, symbol_file):
298        covoar_result_dir = path.join(self.base_result_dir, set_name)
299        if (not path.exists(covoar_result_dir)):
300            path.mkdir(covoar_result_dir)
301        if (not path.exists(symbol_file)):
302            raise error.general('symbol set file: coverage %s was not created for covoar, skipping %s'% (symbol_file, set_name))
303        command = ('covoar -S ' + symbol_file
304                  + ' -O ' + covoar_result_dir
305                  + ' -E ' + self.explanations_txt
306                  + ' -p ' + self.project_name + ' ' + self.executables)
307        log.notice('Running covoar for %s' % (set_name))
308        print( 'covoar results directory:\n' + covoar_result_dir )
309        executor = execute.execute(verbose = True, output = self.output_handler)
310        exit_code = executor.shell(command, cwd=os.getcwd())
311        if (exit_code[0] != 0):
312            raise error.general('covoar failure exit code: %d' % (exit_code[0]))
313        log.notice('Coverage run for %s finished successfully.' % (set_name))
314        log.notice('-----------------------------------------------')
315
316    def output_handler(self, text):
317        log.notice('%s' % (text))
318
319class coverage_run(object):
320    '''
321    Coverage analysis support for rtems-test
322    '''
323    def __init__(self, p_macros, coverage_arg, executables):
324        '''
325        Constructor
326        '''
327        self.macros = p_macros
328        self.build_dir = self.macros['_cwd']
329        self.explanations_txt = self.macros.expand(self.macros['cov_explanations'])
330        self.test_dir = path.join(self.build_dir, self.macros['bsp'] + '-coverage')
331        if (not path.exists(self.test_dir)):
332            path.mkdir(self.test_dir)
333        self.rtdir = path.abspath(self.macros['_rtdir'])
334        self.rtscripts = self.macros.expand(self.macros['_rtscripts'])
335        self.coverage_config_path = path.join(self.rtscripts, 'coverage')
336        self.symbol_config_path = path.join(self.coverage_config_path,
337                                            'symbol-sets.ini')
338        self.symbol_select_path = path.join(self.coverage_config_path,
339                                            self.macros['bsp'] + '-symbols.ini')
340        self.executables = executables
341        self.symbol_sets = []
342        self.no_clean = int(self.macros['_no_clean'])
343        self.report_format = self.macros['cov_report_format']
344        self.coverage_arg = coverage_arg
345        self.target = self.macros['target']
346
347    def run(self):
348        try:
349            if self.executables is None:
350                raise error.general('no test executables provided.')
351            build_dir = build_path_generator(self.executables, self.target).run()
352            parser = symbol_parser(self.symbol_config_path,
353                                   self.symbol_select_path,
354                                   self.coverage_arg,
355                                   build_dir)
356            parser.run()
357            covoar_runner = covoar(self.test_dir, self.symbol_select_path,
358                                   self.executables, self.explanations_txt)
359            covoar_runner.run('score', self.symbol_select_path)
360            self._generate_reports();
361            self._summarize();
362        finally:
363            self._cleanup();
364
365    def _generate_reports(self):
366        log.notice('Generating reports')
367        if self.report_format == 'html':
368            report = report_gen_html(self.symbol_sets,
369                                     self.build_dir,
370                                     self.rtdir,
371                                     self.macros['bsp'])
372            report.generate()
373            report.add_covoar_src_path()
374
375    def _cleanup(self):
376        if not self.no_clean:
377            log.notice('***Cleaning tempfiles***')
378            for exe in self.executables:
379                trace_file = exe + '.cov'
380                if path.exists(trace_file):
381                    os.remove(trace_file)
382            os.remove(self.symbol_select_path)
383
384    def _summarize(self):
385        log.notice('Coverage analysis finished. You can find results in %s' % (self.build_dir))
Note: See TracBrowser for help on using the repository browser.