source: rtems-tools/tester/rt/coverage.py @ e341a65

5
Last change on this file since e341a65 was e341a65, checked in by Chris Johns <chrisj@…>, on 06/18/18 at 00:13:47

tester: Make the path to covoar absolute to ignore the env PATH.

Using the environment's path to find covoar allow invalid versions
to be used which may vary in subtle ways. Find and use the covoar
that is build with the version of 'rtems-test'.

This patch means you do not need to install the tools before
running improving the development experience.

Closes #3458
  • Property mode set to 100644
File size: 16.3 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2014 Krzysztof Miesowicz (krzysztof.miesowicz@gmail.com)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import datetime
34import shutil
35import os
36import sys
37
38try:
39    import configparser
40except:
41    import ConfigParser as configparser
42
43from rtemstoolkit import error
44from rtemstoolkit import path
45from rtemstoolkit import log
46from rtemstoolkit import execute
47from rtemstoolkit import macros
48
49
50from . import options
51
52class summary:
53    def __init__(self, p_summary_dir):
54        self.summary_file_path = path.join(p_summary_dir, 'summary.txt')
55        self.index_file_path = path.join(p_summary_dir, 'index.html')
56        self.bytes_analyzed = 0
57        self.bytes_not_executed = 0
58        self.percentage_executed = 0.0
59        self.percentage_not_executed = 100.0
60        self.ranges_uncovered = 0
61        self.branches_uncovered = 0
62        self.branches_total = 0
63        self.branches_always_taken = 0
64        self.branches_never_taken = 0
65        self.percentage_branches_covered = 0.0
66        self.is_failure = False
67
68    def parse(self):
69        if not path.exists(self.summary_file_path):
70            log.output('coverage: summary file %s does not exist!' % (self.summary_file_path))
71            self.is_failure = True
72
73        with open(self.summary_file_path, 'r') as summary_file:
74           self.bytes_analyzed = self._get_next_with_colon(summary_file)
75           self.bytes_not_executed = self._get_next_with_colon(summary_file)
76           self.percentage_executed = self._get_next_with_colon(summary_file)
77           self.percentage_not_executed = self._get_next_with_colon(summary_file)
78           self.ranges_uncovered = self._get_next_with_colon(summary_file)
79           self.branches_total = self._get_next_with_colon(summary_file)
80           self.branches_uncovered = self._get_next_with_colon(summary_file)
81           self.branches_always_taken = self._get_next_without_colon(summary_file)
82           self.branches_never_taken = self._get_next_without_colon(summary_file)
83        if len(self.branches_uncovered) > 0 and len(self.branches_total) > 0:
84            self.percentage_branches_covered = \
85                1.0 - (float(self.branches_uncovered) / float(self.branches_total))
86        else:
87            self.percentage_branches_covered = 0.0
88        return
89
90    def _get_next_with_colon(self, summary_file):
91        line = summary_file.readline()
92        if ':' in line:
93            return line.split(':')[1].strip()
94        else:
95            return ''
96
97    def _get_next_without_colon(self, summary_file):
98        line = summary_file.readline()
99        return line.strip().split(' ')[0]
100
101class report_gen_html:
102    def __init__(self, p_symbol_sets_list, build_dir, rtdir, bsp):
103        self.symbol_sets_list = ['score']
104        self.build_dir = build_dir
105        self.partial_reports_files = list(["index.html", "summary.txt"])
106        self.number_of_columns = 1
107        self.covoar_src_path = path.join(rtdir, 'covoar')
108        self.bsp = bsp
109
110    def _find_partial_reports(self):
111        partial_reports = {}
112        for symbol_set in self.symbol_sets_list:
113            set_summary = summary(path.join(self.build_dir,
114                                  self.bsp + "-coverage",
115                                  symbol_set))
116            set_summary.parse()
117            partial_reports[symbol_set] = set_summary
118        return partial_reports
119
120    def _prepare_head_section(self):
121        head_section = '''
122        <head>
123        <title>RTEMS coverage report</title>
124        <style type="text/css">
125            progress[value] {
126              -webkit-appearance: none;
127               appearance: none;
128
129              width: 150px;
130              height: 15px;
131            }
132        </style>
133        </head>'''
134        return head_section
135
136    def _prepare_index_content(self, partial_reports):
137        header = "<h1> RTEMS coverage analysis report </h1>"
138        header += "<h3>Coverage reports by symbols sets:</h3>"
139        table = "<table>"
140        table += self._header_row()
141        for symbol_set in partial_reports:
142            table += self._row(symbol_set, partial_reports[symbol_set])
143        table += "</table> </br>"
144        timestamp = "Analysis performed on " + datetime.datetime.now().ctime()
145        return "<body>\n" + header + table + timestamp + "\n</body>"
146
147    def _row(self, symbol_set, summary):
148        row = "<tr>"
149        row += "<td>" + symbol_set + "</td>"
150        if summary.is_failure:
151            row += ' <td colspan="' + str(self.number_of_columns-1) \
152            + '" style="background-color:red">FAILURE</td>'
153        else:
154            row += " <td>" + self._link(summary.index_file_path,"Index") \
155            + "</td>"
156            row += " <td>" + self._link(summary.summary_file_path,"Summary") \
157            + "</td>"
158            row += " <td>" + summary.bytes_analyzed + "</td>"
159            row += " <td>" + summary.bytes_not_executed + "</td>"
160            row += " <td>" + summary.ranges_uncovered + "</td>"
161            row += " <td>" + summary.percentage_executed + "%</td>"
162            row += " <td>" + summary.percentage_not_executed + "%</td>"
163            row += ' <td><progress value="' + summary.percentage_executed \
164            + '" max="100"></progress></td>'
165            row += " <td>" + summary.branches_uncovered + "</td>"
166            row += " <td>" + summary.branches_total + "</td>"
167            row += " <td> {:.3%} </td>".format(summary.percentage_branches_covered)
168            spbc = 100 * summary.percentage_branches_covered
169            row += ' <td><progress value="{:.3}" max="100"></progress></td>'.format(spbc)
170            row += "</tr>\n"
171        return row
172
173    def _header_row(self):
174        row = "<tr>"
175        row += "<th> Symbols set name </th>"
176        row += "<th> Index file </th>"
177        row += "<th> Summary file </th>"
178        row += "<th> Bytes analyzed </th>"
179        row += "<th> Bytes not executed </th>"
180        row += "<th> Uncovered ranges </th>"
181        row += "<th> Percentage covered </th>"
182        row += "<th> Percentage uncovered </th>"
183        row += "<th> Instruction coverage </th>"
184        row += "<th> Branches uncovered </th>"
185        row += "<th> Branches total </th>"
186        row += "<th> Branches covered percentage </th>"
187        row += "<th> Branches coverage </th>"
188        row += "</tr>\n"
189        self.number_of_columns = row.count('<th>')
190        return row
191
192    def _link(self, address, text):
193        return '<a href="' + address + '">' + text + '</a>'
194
195    def _create_index_file(self, head_section, content):
196        name = path.join(self.build_dir, self.bsp + "-report.html")
197        with open(name, 'w') as f:
198            f.write(head_section)
199            f.write(content)
200
201    def generate(self):
202        partial_reports = self._find_partial_reports()
203        head_section = self._prepare_head_section()
204        index_content = self._prepare_index_content(partial_reports)
205        self._create_index_file(head_section,index_content)
206
207    def add_covoar_src_path(self):
208        table_js_path = path.join(self.covoar_src_path, 'table.js')
209        covoar_css_path = path.join(self.covoar_src_path, 'covoar.css')
210        for symbol_set in self.symbol_sets_list:
211            symbol_set_dir = path.join(self.build_dir,
212                                       self.bsp + '-coverage', symbol_set)
213            html_files = os.listdir(symbol_set_dir)
214            for html_file in html_files:
215                html_file = path.join(symbol_set_dir, html_file)
216                if path.exists(html_file) and 'html' in html_file:
217                    with open(html_file, 'r') as f:
218                        file_data = f.read()
219                    file_data = file_data.replace('table.js', table_js_path)
220                    file_data = file_data.replace('covoar.css',
221                                                  covoar_css_path)
222                    with open(html_file, 'w') as f:
223                        f.write(file_data)
224
225class build_path_generator(object):
226    '''
227    Generates the build path from the path to executables
228    '''
229    def __init__(self, executables, target):
230        self.executables = executables
231        self.target = target
232    def run(self):
233        build_path = '/'
234        path_ = self.executables[0].split('/')
235        for p in path_:
236            if p == self.target:
237                break
238            else:
239                build_path = path.join(build_path, p)
240        return build_path
241
242class symbol_parser(object):
243    '''
244    Parse the symbol sets ini and create custom ini file for covoar
245    '''
246    def __init__(self,
247                 symbol_config_path,
248                 symbol_select_path,
249                 symbol_set,
250                 build_dir):
251        self.symbol_select_file = symbol_select_path
252        self.symbol_file = symbol_config_path
253        self.build_dir = build_dir
254        self.symbol_sets = {}
255        self.symbol_set = symbol_set
256        self.ssets = []
257
258    def parse(self):
259        config = configparser.ConfigParser()
260        try:
261            config.read(self.symbol_file)
262            if self.symbol_set is not None:
263                self.ssets = self.symbol_set.split(',')
264            else:
265                self.ssets = config.get('symbol-sets', 'sets').split(',')
266                self.ssets = [sset.encode('utf-8') for sset in self.ssets]
267            for sset in self.ssets:
268                lib = path.join(self.build_dir, config.get('libraries', sset))
269                self.symbol_sets[sset] = lib.encode('utf-8')
270        except:
271            raise error.general('Symbol set parsing failed')
272
273    def _write_ini(self):
274        config = configparser.ConfigParser()
275        try:
276            sets = ', '.join(self.symbol_sets.keys())
277            config.add_section('symbol-sets')
278            config.set('symbol-sets', 'sets', sets)
279            for key in self.symbol_sets.keys():
280                config.add_section(key)
281                config.set(key, 'libraries', self.symbol_sets[key])
282            with open(self.symbol_select_file, 'w') as conf:
283                config.write(conf)
284        except:
285            raise error.general('symbol parser write failed')
286
287    def run(self):
288        self.parse()
289        self._write_ini()
290
291class covoar(object):
292    '''
293    Covoar runner
294    '''
295    def __init__(self, base_result_dir, config_dir, executables, explanations_txt, trace):
296        self.base_result_dir = base_result_dir
297        self.config_dir = config_dir
298        self.executables = ' '.join(executables)
299        self.explanations_txt = explanations_txt
300        self.project_name = 'RTEMS-5'
301        self.trace = trace
302
303    def _find_covoar(self):
304        covoar_exe = 'covoar'
305        tester_dir = path.dirname(path.abspath(sys.argv[0]))
306        base = path.dirname(tester_dir)
307        exe = path.join(base, 'bin', covoar_exe)
308        if path.isfile(exe):
309            return exe
310        exe = path.join(base, 'build', 'tester', 'covoar', covoar_exe)
311        if path.isfile(exe):
312            return exe
313        raise error.general('coverage: %s not found'% (covoar_exe))
314
315    def run(self, set_name, symbol_file):
316        covoar_result_dir = path.join(self.base_result_dir, set_name)
317        if not path.exists(covoar_result_dir):
318            path.mkdir(covoar_result_dir)
319        if not path.exists(symbol_file):
320            raise error.general('coverage: no symbol set file: %s'% (symbol_file))
321        exe = self._find_covoar()
322        command = exe + ' -S ' + symbol_file + \
323                  ' -O ' + covoar_result_dir + \
324                  ' -E ' + self.explanations_txt + \
325                  ' -p ' + self.project_name + ' ' + self.executables
326        log.notice()
327        log.notice('Running coverage analysis: %s (%s)' % (set_name, covoar_result_dir))
328        start_time = datetime.datetime.now()
329        executor = execute.execute(verbose = self.trace, output = self.output_handler)
330        exit_code = executor.shell(command, cwd=os.getcwd())
331        if exit_code[0] != 0:
332            raise error.general('coverage: covoar failure:: %d' % (exit_code[0]))
333        end_time = datetime.datetime.now()
334        log.notice('Coverage time: %s' % (str(end_time - start_time)))
335
336    def output_handler(self, text):
337        log.output('%s' % (text))
338
339class coverage_run(object):
340    '''
341    Coverage analysis support for rtems-test
342    '''
343    def __init__(self, macros_, executables, symbol_set = None, trace = False):
344        '''
345        Constructor
346        '''
347        self.trace = trace
348        self.macros = macros_
349        self.build_dir = self.macros['_cwd']
350        self.explanations_txt = self.macros.expand(self.macros['cov_explanations'])
351        self.test_dir = path.join(self.build_dir, self.macros['bsp'] + '-coverage')
352        if not path.exists(self.test_dir):
353            path.mkdir(self.test_dir)
354        self.rtdir = path.abspath(self.macros['_rtdir'])
355        self.rtscripts = self.macros.expand(self.macros['_rtscripts'])
356        self.coverage_config_path = path.join(self.rtscripts, 'coverage')
357        self.symbol_config_path = path.join(self.coverage_config_path,
358                                            'symbol-sets.ini')
359        self.symbol_select_path = path.join(self.coverage_config_path,
360                                            self.macros['bsp'] + '-symbols.ini')
361        self.executables = executables
362        self.symbol_sets = []
363        self.no_clean = int(self.macros['_no_clean'])
364        self.report_format = self.macros['cov_report_format']
365        self.symbol_set = symbol_set
366        self.target = self.macros['target']
367
368    def run(self):
369        try:
370            if self.executables is None:
371                raise error.general('no test executables provided.')
372            build_dir = build_path_generator(self.executables, self.target).run()
373            parser = symbol_parser(self.symbol_config_path,
374                                   self.symbol_select_path,
375                                   self.symbol_set,
376                                   build_dir)
377            parser.run()
378            covoar_runner = covoar(self.test_dir, self.symbol_select_path,
379                                   self.executables, self.explanations_txt,
380                                   self.trace)
381            covoar_runner.run('score', self.symbol_select_path)
382            self._generate_reports();
383            self._summarize();
384        finally:
385            self._cleanup();
386
387    def _generate_reports(self):
388        log.notice('Coverage generating reports')
389        if self.report_format == 'html':
390            report = report_gen_html(self.symbol_sets,
391                                     self.build_dir,
392                                     self.rtdir,
393                                     self.macros['bsp'])
394            report.generate()
395            report.add_covoar_src_path()
396
397    def _cleanup(self):
398        if not self.no_clean:
399            if self.trace:
400                log.output('Coverage cleaning tempfiles')
401            for exe in self.executables:
402                trace_file = exe + '.cov'
403                if path.exists(trace_file):
404                    os.remove(trace_file)
405            os.remove(self.symbol_select_path)
406
407    def _summarize(self):
408        log.notice('Coverage analysis finished: %s' % (self.build_dir))
Note: See TracBrowser for help on using the repository browser.