source: rtems-tools/tester/rt/coverage.py @ e058db0

Last change on this file since e058db0 was e058db0, checked in by Chris Johns <chrisj@…>, on Nov 7, 2018 at 3:55:20 AM

python: Provide support to select a valid python version.

  • Update imports after wrapping the code.
  • Fix python3 issues.
  • Fix config path issues for in repo and install runs.

Closes #3537

  • Property mode set to 100644
File size: 17.7 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2014 Krzysztof Miesowicz (krzysztof.miesowicz@gmail.com)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import datetime
34import shutil
35import os
36import sys
37
38try:
39    import configparser
40except:
41    import ConfigParser as configparser
42
43from rtemstoolkit import error
44from rtemstoolkit import path
45from rtemstoolkit import log
46from rtemstoolkit import execute
47from rtemstoolkit import macros
48
49
50import options
51
52class summary:
53    def __init__(self, p_summary_dir):
54        self.summary_file_path = path.join(p_summary_dir, 'summary.txt')
55        self.index_file_path = path.join(p_summary_dir, 'index.html')
56        self.bytes_analyzed = 0
57        self.bytes_not_executed = 0
58        self.percentage_executed = 0.0
59        self.percentage_not_executed = 100.0
60        self.ranges_uncovered = 0
61        self.branches_uncovered = 0
62        self.branches_total = 0
63        self.branches_always_taken = 0
64        self.branches_never_taken = 0
65        self.percentage_branches_covered = 0.0
66        self.is_failure = False
67
68    def parse(self):
69        if not path.exists(self.summary_file_path):
70            log.output('coverage: summary file %s does not exist!' % (self.summary_file_path))
71            self.is_failure = True
72
73        with open(self.summary_file_path, 'r') as summary_file:
74           self.bytes_analyzed = self._get_next_with_colon(summary_file)
75           self.bytes_not_executed = self._get_next_with_colon(summary_file)
76           self.percentage_executed = self._get_next_with_colon(summary_file)
77           self.percentage_not_executed = self._get_next_with_colon(summary_file)
78           self.ranges_uncovered = self._get_next_with_colon(summary_file)
79           self.branches_total = self._get_next_with_colon(summary_file)
80           self.branches_uncovered = self._get_next_with_colon(summary_file)
81           self.branches_always_taken = self._get_next_without_colon(summary_file)
82           self.branches_never_taken = self._get_next_without_colon(summary_file)
83        if len(self.branches_uncovered) > 0 and len(self.branches_total) > 0:
84            self.percentage_branches_covered = \
85                1.0 - (float(self.branches_uncovered) / float(self.branches_total))
86        else:
87            self.percentage_branches_covered = 0.0
88        return
89
90    def _get_next_with_colon(self, summary_file):
91        line = summary_file.readline()
92        if ':' in line:
93            return line.split(':')[1].strip()
94        else:
95            return ''
96
97    def _get_next_without_colon(self, summary_file):
98        line = summary_file.readline()
99        return line.strip().split(' ')[0]
100
101class report_gen_html:
102    def __init__(self, symbol_sets, build_dir, rtdir, bsp):
103        self.symbol_sets = symbol_sets
104        self.build_dir = build_dir
105        self.partial_reports_files = list(['index.html', 'summary.txt'])
106        self.number_of_columns = 1
107        self.covoar_src_path = path.join(rtdir, 'covoar')
108        self.bsp = bsp
109
110    def _find_partial_reports(self):
111        partial_reports = {}
112        for symbol_set in self.symbol_sets:
113            set_summary = summary(path.join(self.bsp + "-coverage",
114                                            symbol_set))
115            set_summary.parse()
116            partial_reports[symbol_set] = set_summary
117        return partial_reports
118
119    def _prepare_head_section(self):
120        head_section = '<head>' + os.linesep
121        head_section += ' <title>RTEMS coverage report</title>' + os.linesep
122        head_section += ' <style type="text/css">' + os.linesep
123        head_section += '  progress[value] {' + os.linesep
124        head_section += '    -webkit-appearance: none;' + os.linesep
125        head_section += '    appearance: none;' + os.linesep
126        head_section += '    width: 150px;' + os.linesep
127        head_section += '    height: 15px;' + os.linesep
128        head_section += '  }' + os.linesep
129        head_section += ' </style>' + os.linesep
130        head_section += '</head>' + os.linesep
131        return head_section
132
133    def _prepare_index_content(self, partial_reports):
134        header = "<h1> RTEMS coverage analysis report </h1>" + os.linesep
135        header += "<h3>Coverage reports by symbols sets:</h3>" + os.linesep
136        table = "<table>" + os.linesep
137        table += self._header_row()
138        for symbol_set in partial_reports:
139            table += self._row(symbol_set, partial_reports[symbol_set])
140        table += "</table> </br>"
141        timestamp = "Analysis performed on " + datetime.datetime.now().ctime()
142        return "<body>\n" + header + table + timestamp + "\n</body>"
143
144    def _row(self, symbol_set, summary):
145        row = "<tr>" + os.linesep
146        row += "<td>" + symbol_set + "</td>" + os.linesep
147        if summary.is_failure:
148            row += ' <td colspan="' + str(self.number_of_columns-1) \
149                        + '" style="background-color:red">FAILURE</td>' + os.linesep
150        else:
151            row += ' <td>' + self._link(summary.index_file_path, 'Index') \
152                   + '</td>' + os.linesep
153            row += ' <td>' + self._link(summary.summary_file_path, 'Summary') \
154                   + '</td>' + os.linesep
155            row += ' <td>' + summary.bytes_analyzed + '</td>' + os.linesep
156            row += ' <td>' + summary.bytes_not_executed + '</td>' + os.linesep
157            row += ' <td>' + summary.ranges_uncovered + '</td>' + os.linesep
158            row += ' <td>' + summary.percentage_executed + '%</td>' + os.linesep
159            row += ' <td>' + summary.percentage_not_executed + '%</td>' + os.linesep
160            row += ' <td><progress value="' + summary.percentage_executed \
161                    + '" max="100"></progress></td>' + os.linesep
162            row += ' <td>' + summary.branches_uncovered + '</td>' + os.linesep
163            row += ' <td>' + summary.branches_total + '</td>' + os.linesep
164            row += ' <td> {:.3%} </td>'.format(summary.percentage_branches_covered)
165            spbc = 100 * summary.percentage_branches_covered
166            row += ' <td><progress value="{:.3}" max="100"></progress></td>'.format(spbc)
167            row += '</tr>' + os.linesep
168        return row
169
170    def _header_row(self):
171        row = "<tr>" + os.linesep
172        row += " <th> Symbols set name </th>" + os.linesep
173        row += " <th> Index file </th>" + os.linesep
174        row += " <th> Summary file </th>" + os.linesep
175        row += " <th> Bytes analyzed </th>" + os.linesep
176        row += " <th> Bytes not executed </th>" + os.linesep
177        row += " <th> Uncovered ranges </th>" + os.linesep
178        row += " <th> Percentage covered </th>" + os.linesep
179        row += " <th> Percentage uncovered </th>" + os.linesep
180        row += " <th> Instruction coverage </th>" + os.linesep
181        row += " <th> Branches uncovered </th>" + os.linesep
182        row += " <th> Branches total </th>" + os.linesep
183        row += " <th> Branches covered percentage </th>" + os.linesep
184        row += " <th> Branches coverage </th>" + os.linesep
185        row += "</tr>"
186        self.number_of_columns = row.count('<th>')
187        return row
188
189    def _link(self, address, text):
190        return '<a href="' + address + '">' + text + '</a>'
191
192    def _create_index_file(self, head_section, content):
193        name = path.join(self.build_dir, self.bsp + "-report.html")
194        with open(name, 'w') as f:
195            f.write(head_section)
196            f.write(content)
197
198    def generate(self):
199        partial_reports = self._find_partial_reports()
200        head_section = self._prepare_head_section()
201        index_content = self._prepare_index_content(partial_reports)
202        self._create_index_file(head_section,index_content)
203
204    def add_covoar_src_path(self):
205        table_js_path = path.join(self.covoar_src_path, 'table.js')
206        covoar_css_path = path.join(self.covoar_src_path, 'covoar.css')
207        for symbol_set in self.symbol_sets:
208            symbol_set_dir = path.join(self.build_dir,
209                                       self.bsp + '-coverage', symbol_set)
210            html_files = os.listdir(symbol_set_dir)
211            for html_file in html_files:
212                html_file = path.join(symbol_set_dir, html_file)
213                if path.exists(html_file) and 'html' in html_file:
214                    with open(html_file, 'r') as f:
215                        file_data = f.read()
216                    file_data = file_data.replace('table.js', table_js_path)
217                    file_data = file_data.replace('covoar.css',
218                                                  covoar_css_path)
219                    with open(html_file, 'w') as f:
220                        f.write(file_data)
221
222class build_path_generator(object):
223    '''
224    Generates the build path from the path to executables
225    '''
226    def __init__(self, executables, target):
227        self.executables = executables
228        self.target = target
229    def run(self):
230        build_path = '/'
231        path_ = self.executables[0].split('/')
232        for p in path_:
233            if p == self.target:
234                break
235            else:
236                build_path = path.join(build_path, p)
237        return build_path
238
239class symbol_parser(object):
240    '''
241    Parse the symbol sets ini and create custom ini file for covoar
242    '''
243    def __init__(self,
244                 symbol_config_path,
245                 symbol_select_path,
246                 symbol_set,
247                 build_dir,
248                 bsp_name,
249                 target):
250        self.symbol_select_file = symbol_select_path
251        self.symbol_file = symbol_config_path
252        self.build_dir = build_dir
253        self.symbol_sets = {}
254        self.symbol_set = symbol_set
255        self.ssets = []
256        self.bsp_name = bsp_name
257        self.target = target
258
259    def parse(self):
260        config = configparser.ConfigParser()
261        try:
262            config.read(self.symbol_file)
263            if self.symbol_set is not None:
264                self.ssets = self.symbol_set.split(',')
265            else:
266                self.ssets = config.get('symbol-sets', 'sets').split(',')
267                self.ssets = [sset.encode('utf-8') for sset in self.ssets]
268            for sset in self.ssets:
269                lib = path.join(self.build_dir, config.get('libraries', sset))
270                self.symbol_sets[sset] = lib.encode('utf-8')
271                ss = self.symbol_sets[sset]
272                ss = ss.replace('@BSP@', self.bsp_name)
273                ss = ss.replace('@BUILD-TARGET@', self.target)
274                self.symbol_sets[sset] = ss
275            return self.ssets
276        except:
277            raise error.general('Symbol set parsing failed for %s' % (sset))
278
279    def write_ini(self, symbol_set):
280        config = configparser.ConfigParser()
281        try:
282            sset = symbol_set
283            config.add_section('symbol-sets')
284            config.set('symbol-sets', 'sets', sset)
285            config.add_section(sset)
286            object_files = [o for o in os.listdir(self.symbol_sets[sset]) if o[-1] == 'o']
287            object_paths = []
288            for o in object_files:
289                object_paths.append(path.join(self.symbol_sets[sset], o))
290            config.set(sset, 'libraries', ','.join(object_paths))
291            with open(self.symbol_select_file, 'w') as conf:
292                    config.write(conf)
293        except:
294            raise error.general('symbol parser write failed for %s' % (sset))
295
296class covoar(object):
297    '''
298    Covoar runner
299    '''
300    def __init__(self, base_result_dir, config_dir, executables, explanations_txt, trace):
301        self.base_result_dir = base_result_dir
302        self.config_dir = config_dir
303        self.executables = ' '.join(executables)
304        self.explanations_txt = explanations_txt
305        self.project_name = 'RTEMS-5'
306        self.trace = trace
307
308    def _find_covoar(self):
309        covoar_exe = 'covoar'
310        tester_dir = path.dirname(path.abspath(sys.argv[0]))
311        base = path.dirname(tester_dir)
312        exe = path.join(base, 'bin', covoar_exe)
313        if path.isfile(exe):
314            return exe
315        exe = path.join(base, 'build', 'tester', 'covoar', covoar_exe)
316        if path.isfile(exe):
317            return exe
318        raise error.general('coverage: %s not found'% (covoar_exe))
319
320    def run(self, set_name, symbol_file):
321        covoar_result_dir = path.join(self.base_result_dir, set_name)
322        if not path.exists(covoar_result_dir):
323            path.mkdir(covoar_result_dir)
324        if not path.exists(symbol_file):
325            raise error.general('coverage: no symbol set file: %s'% (symbol_file))
326        exe = self._find_covoar()
327        command = exe + ' -S ' + symbol_file + \
328                  ' -O ' + covoar_result_dir + \
329                  ' -E ' + self.explanations_txt + \
330                  ' -p ' + self.project_name + ' ' + self.executables
331        log.notice()
332        log.notice('Running coverage analysis: %s (%s)' % (set_name, covoar_result_dir))
333        start_time = datetime.datetime.now()
334        executor = execute.execute(verbose = self.trace, output = self.output_handler)
335        exit_code = executor.shell(command, cwd=os.getcwd())
336        if exit_code[0] != 0:
337            raise error.general('coverage: covoar failure:: %d' % (exit_code[0]))
338        end_time = datetime.datetime.now()
339        log.notice('Coverage time: %s' % (str(end_time - start_time)))
340
341    def output_handler(self, text):
342        log.output('%s' % (text))
343
344class coverage_run(object):
345    '''
346    Coverage analysis support for rtems-test
347    '''
348    def __init__(self, macros_, executables, symbol_set = None, trace = False):
349        '''
350        Constructor
351        '''
352        self.trace = trace
353        self.macros = macros_
354        self.build_dir = self.macros['_cwd']
355        self.explanations_txt = self.macros.expand(self.macros['cov_explanations'])
356        self.test_dir = path.join(self.build_dir, self.macros['bsp'] + '-coverage')
357        if not path.exists(self.test_dir):
358            path.mkdir(self.test_dir)
359        self.rtdir = path.abspath(self.macros['_rtdir'])
360        self.rtscripts = self.macros.expand(self.macros['_rtscripts'])
361        self.coverage_config_path = path.join(self.rtscripts, 'coverage')
362        self.symbol_config_path = path.join(self.coverage_config_path,
363                                            'symbol-sets.ini')
364        self.symbol_select_path = path.join(self.coverage_config_path,
365                                            self.macros['bsp'] + '-symbols.ini')
366        self.executables = executables
367        self.symbol_sets = []
368        self.no_clean = int(self.macros['_no_clean'])
369        self.report_format = self.macros['cov_report_format']
370        self.symbol_set = symbol_set
371        self.target = self.macros['target']
372        self.bsp_name = self.macros['bsp'].split('-')[0]
373
374    def run(self):
375        try:
376            if self.executables is None:
377                raise error.general('no test executables provided.')
378            build_dir = build_path_generator(self.executables, self.target).run()
379            parser = symbol_parser(self.symbol_config_path,
380                                   self.symbol_select_path,
381                                   self.symbol_set,
382                                   build_dir,
383                                   self.bsp_name,
384                                   self.target)
385            symbol_sets = parser.parse()
386            for sset in symbol_sets:
387                parser.write_ini(sset)
388                covoar_runner = covoar(self.test_dir, self.symbol_select_path,
389                                   self.executables, self.explanations_txt,
390                                   self.trace)
391                covoar_runner.run(sset, self.symbol_select_path)
392            self._generate_reports(symbol_sets);
393            self._summarize();
394        finally:
395            self._cleanup();
396
397    def _generate_reports(self, symbol_sets):
398        log.notice('Coverage generating reports')
399        if self.report_format == 'html':
400            report = report_gen_html(symbol_sets,
401                                     self.build_dir,
402                                     self.rtdir,
403                                     self.macros['bsp'])
404            report.generate()
405            report.add_covoar_src_path()
406
407    def _cleanup(self):
408        if not self.no_clean:
409            if self.trace:
410                log.output('Coverage cleaning tempfiles')
411            for exe in self.executables:
412                trace_file = exe + '.cov'
413                if path.exists(trace_file):
414                    os.remove(trace_file)
415            os.remove(self.symbol_select_path)
416
417    def _summarize(self):
418        log.notice('Coverage analysis finished: %s' % (self.build_dir))
Note: See TracBrowser for help on using the repository browser.