source: rtems-tools/tester/rt/coverage.py @ 8dcfc0e

5
Last change on this file since 8dcfc0e was 8dcfc0e, checked in by Vijay Kumar Banerjee <vijaykumar9597@…>, on 03/22/19 at 15:05:12

coverage: remove utf-8 encoding in symbolset names
  • Property mode set to 100644
File size: 17.4 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2014 Krzysztof Miesowicz (krzysztof.miesowicz@gmail.com)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import datetime
34import shutil
35import os
36import sys
37
38try:
39    import configparser
40except:
41    import ConfigParser as configparser
42
43from rtemstoolkit import error
44from rtemstoolkit import path
45from rtemstoolkit import log
46from rtemstoolkit import execute
47from rtemstoolkit import macros
48from rtemstoolkit import version
49
50
51import options
52
53class summary:
54    def __init__(self, p_summary_dir):
55        self.summary_file_path = path.join(p_summary_dir, 'summary.txt')
56        self.index_file_path = path.join(p_summary_dir, 'index.html')
57        self.bytes_analyzed = 0
58        self.bytes_not_executed = 0
59        self.percentage_executed = 0.0
60        self.percentage_not_executed = 100.0
61        self.ranges_uncovered = 0
62        self.branches_uncovered = 0
63        self.branches_total = 0
64        self.branches_always_taken = 0
65        self.branches_never_taken = 0
66        self.percentage_branches_covered = 0.0
67        self.is_failure = False
68
69    def parse(self):
70        if not path.exists(self.summary_file_path):
71            log.output('coverage: summary file %s does not exist!' % (self.summary_file_path))
72            self.is_failure = True
73
74        with open(self.summary_file_path, 'r') as summary_file:
75           self.bytes_analyzed = self._get_next_with_colon(summary_file)
76           self.bytes_not_executed = self._get_next_with_colon(summary_file)
77           self.percentage_executed = self._get_next_with_colon(summary_file)
78           self.percentage_not_executed = self._get_next_with_colon(summary_file)
79           self.ranges_uncovered = self._get_next_with_colon(summary_file)
80           self.branches_total = self._get_next_with_colon(summary_file)
81           self.branches_uncovered = self._get_next_with_colon(summary_file)
82           self.branches_always_taken = self._get_next_without_colon(summary_file)
83           self.branches_never_taken = self._get_next_without_colon(summary_file)
84        if len(self.branches_uncovered) > 0 and len(self.branches_total) > 0:
85            self.percentage_branches_covered = \
86                1.0 - (float(self.branches_uncovered) / float(self.branches_total))
87        else:
88            self.percentage_branches_covered = 0.0
89        return
90
91    def _get_next_with_colon(self, summary_file):
92        line = summary_file.readline()
93        if ':' in line:
94            return line.split(':')[1].strip()
95        else:
96            return ''
97
98    def _get_next_without_colon(self, summary_file):
99        line = summary_file.readline()
100        return line.strip().split(' ')[0]
101
102class report_gen_html:
103    def __init__(self, symbol_sets, build_dir, rtdir, bsp):
104        self.symbol_sets = symbol_sets
105        self.build_dir = build_dir
106        self.partial_reports_files = list(['index.html', 'summary.txt'])
107        self.number_of_columns = 1
108        self.covoar_src_path = path.join(rtdir, 'covoar')
109        self.bsp = bsp
110
111    def _find_partial_reports(self):
112        partial_reports = {}
113        for symbol_set in self.symbol_sets:
114            set_summary = summary(path.join(self.bsp + "-coverage",
115                                            symbol_set))
116            set_summary.parse()
117            partial_reports[symbol_set] = set_summary
118        return partial_reports
119
120    def _prepare_head_section(self):
121        head_section = '<head>' + os.linesep
122        head_section += ' <title>RTEMS coverage report</title>' + os.linesep
123        head_section += ' <style type="text/css">' + os.linesep
124        head_section += '  progress[value] {' + os.linesep
125        head_section += '    -webkit-appearance: none;' + os.linesep
126        head_section += '    appearance: none;' + os.linesep
127        head_section += '    width: 150px;' + os.linesep
128        head_section += '    height: 15px;' + os.linesep
129        head_section += '  }' + os.linesep
130        head_section += ' </style>' + os.linesep
131        head_section += '</head>' + os.linesep
132        return head_section
133
134    def _prepare_index_content(self, partial_reports):
135        header = "<h1> RTEMS coverage analysis report </h1>" + os.linesep
136        header += "<h3>Coverage reports by symbols sets:</h3>" + os.linesep
137        table = "<table>" + os.linesep
138        table += self._header_row()
139        for symbol_set in partial_reports:
140            table += self._row(symbol_set, partial_reports[symbol_set])
141        table += "</table> </br>"
142        timestamp = "Analysis performed on " + datetime.datetime.now().ctime()
143        return "<body>\n" + header + table + timestamp + "\n</body>"
144
145    def _row(self, symbol_set, summary):
146        row = "<tr>" + os.linesep
147        row += "<td>" + symbol_set + "</td>" + os.linesep
148        if summary.is_failure:
149            row += ' <td colspan="' + str(self.number_of_columns-1) \
150                        + '" style="background-color:red">FAILURE</td>' + os.linesep
151        else:
152            row += ' <td>' + self._link(summary.index_file_path, 'Index') \
153                   + '</td>' + os.linesep
154            row += ' <td>' + self._link(summary.summary_file_path, 'Summary') \
155                   + '</td>' + os.linesep
156            row += ' <td>' + summary.bytes_analyzed + '</td>' + os.linesep
157            row += ' <td>' + summary.bytes_not_executed + '</td>' + os.linesep
158            row += ' <td>' + summary.ranges_uncovered + '</td>' + os.linesep
159            row += ' <td>' + summary.percentage_executed + '%</td>' + os.linesep
160            row += ' <td>' + summary.percentage_not_executed + '%</td>' + os.linesep
161            row += ' <td><progress value="' + summary.percentage_executed \
162                    + '" max="100"></progress></td>' + os.linesep
163            row += ' <td>' + summary.branches_uncovered + '</td>' + os.linesep
164            row += ' <td>' + summary.branches_total + '</td>' + os.linesep
165            row += ' <td> {:.3%} </td>'.format(summary.percentage_branches_covered)
166            spbc = 100 * summary.percentage_branches_covered
167            row += ' <td><progress value="{:.3}" max="100"></progress></td>'.format(spbc)
168            row += '</tr>' + os.linesep
169        return row
170
171    def _header_row(self):
172        row = "<tr>" + os.linesep
173        row += " <th> Symbols set name </th>" + os.linesep
174        row += " <th> Index file </th>" + os.linesep
175        row += " <th> Summary file </th>" + os.linesep
176        row += " <th> Bytes analyzed </th>" + os.linesep
177        row += " <th> Bytes not executed </th>" + os.linesep
178        row += " <th> Uncovered ranges </th>" + os.linesep
179        row += " <th> Percentage covered </th>" + os.linesep
180        row += " <th> Percentage uncovered </th>" + os.linesep
181        row += " <th> Instruction coverage </th>" + os.linesep
182        row += " <th> Branches uncovered </th>" + os.linesep
183        row += " <th> Branches total </th>" + os.linesep
184        row += " <th> Branches covered percentage </th>" + os.linesep
185        row += " <th> Branches coverage </th>" + os.linesep
186        row += "</tr>"
187        self.number_of_columns = row.count('<th>')
188        return row
189
190    def _link(self, address, text):
191        return '<a href="' + address + '">' + text + '</a>'
192
193    def _create_index_file(self, head_section, content):
194        name = path.join(self.build_dir, self.bsp + "-report.html")
195        with open(name, 'w') as f:
196            f.write(head_section)
197            f.write(content)
198
199    def generate(self):
200        partial_reports = self._find_partial_reports()
201        head_section = self._prepare_head_section()
202        index_content = self._prepare_index_content(partial_reports)
203        self._create_index_file(head_section,index_content)
204
205    def add_covoar_css(self):
206        table_js_path = path.join(self.covoar_src_path, 'table.js')
207        covoar_css_path = path.join(self.covoar_src_path, 'covoar.css')
208        for symbol_set in self.symbol_sets:
209            symbol_set_dir = path.join(self.build_dir,
210                                       self.bsp + '-coverage', symbol_set)
211            path.copy_tree(covoar_css_path, symbol_set_dir)
212            path.copy_tree(table_js_path, symbol_set_dir)
213
214class build_path_generator(object):
215    '''
216    Generates the build path from the path to executables
217    '''
218    def __init__(self, executables, target):
219        self.executables = executables
220        self.target = target
221    def run(self):
222        build_path = '/'
223        path_ = self.executables[0].split('/')
224        for p in path_:
225            if p == self.target:
226                break
227            else:
228                build_path = path.join(build_path, p)
229        return build_path
230
231class symbol_parser(object):
232    '''
233    Parse the symbol sets ini and create custom ini file for covoar
234    '''
235    def __init__(self,
236                 symbol_config_path,
237                 symbol_select_path,
238                 symbol_set,
239                 build_dir,
240                 bsp_name,
241                 target):
242        self.symbol_select_file = symbol_select_path
243        self.symbol_file = symbol_config_path
244        self.build_dir = build_dir
245        self.symbol_sets = {}
246        self.symbol_set = symbol_set
247        self.ssets = []
248        self.bsp_name = bsp_name
249        self.target = target
250
251    def parse(self):
252        config = configparser.ConfigParser()
253        try:
254            config.read(self.symbol_file)
255            if self.symbol_set is not None:
256                self.ssets = self.symbol_set.split(',')
257            else:
258                self.ssets = config.get('symbol-sets', 'sets').split(',')
259            for sset in self.ssets:
260                lib = path.join(self.build_dir, config.get('libraries', sset))
261                self.symbol_sets[sset] = lib
262                ss = self.symbol_sets[sset]
263                ss = ss.replace('@BSP@', self.bsp_name)
264                ss = ss.replace('@BUILD-TARGET@', self.target)
265                self.symbol_sets[sset] = ss
266            return self.ssets
267        except:
268            raise error.general('Symbol set parsing failed for %s' % (sset))
269
270    def write_ini(self, symbol_set):
271        config = configparser.ConfigParser()
272        try:
273            sset = symbol_set
274            config.add_section('symbol-sets')
275            config.set('symbol-sets', 'sets', sset)
276            config.add_section(sset)
277            object_files = [o for o in os.listdir(self.symbol_sets[sset]) if o[-1] == 'o']
278            object_paths = []
279            for o in object_files:
280                object_paths.append(path.join(self.symbol_sets[sset], o))
281            config.set(sset, 'libraries', ','.join(object_paths))
282            with open(self.symbol_select_file, 'w') as conf:
283                    config.write(conf)
284        except:
285            raise error.general('symbol parser write failed for %s' % (sset))
286
287class covoar(object):
288    '''
289    Covoar runner
290    '''
291    def __init__(self,
292                 base_result_dir,
293                 config_dir,
294                 executables,
295                 trace,
296                 prefix,
297                 covoar_cmd):
298        self.base_result_dir = base_result_dir
299        self.config_dir = config_dir
300        self.executables = ' '.join(executables)
301        self.project_name = 'RTEMS-' + str(version.version())
302        self.trace = trace
303        self.prefix = prefix
304        self.covoar_cmd = covoar_cmd
305
306    def _find_covoar(self):
307        covoar_exe = 'covoar'
308        tester_dir = path.dirname(path.abspath(sys.argv[0]))
309        base = path.dirname(tester_dir)
310        exe = path.join(self.prefix, 'share', 'rtems', 'tester', 'bin', covoar_exe)
311        if path.isfile(exe):
312            return exe
313        exe = path.join(base, 'build', 'tester', 'covoar', covoar_exe)
314        if path.isfile(exe):
315            return exe
316        raise error.general('coverage: %s not found'% (covoar_exe))
317
318    def run(self, set_name, symbol_file):
319        covoar_result_dir = path.join(self.base_result_dir, set_name)
320        if not path.exists(covoar_result_dir):
321            path.mkdir(covoar_result_dir)
322        if not path.exists(symbol_file):
323            raise error.general('coverage: no symbol set file: %s'% (symbol_file))
324        exe = self._find_covoar()
325        command = exe + ' -O ' + covoar_result_dir + \
326                  ' -p ' + self.project_name + \
327                  ' ' + self.executables + ' '
328        command += self.covoar_cmd
329
330        log.notice()
331        log.notice('Running coverage analysis: %s (%s)' % (set_name, covoar_result_dir))
332        start_time = datetime.datetime.now()
333        executor = execute.execute(verbose = self.trace, output = self.output_handler)
334        exit_code = executor.shell(command, cwd=os.getcwd())
335        if exit_code[0] != 0:
336            raise error.general('coverage: covoar failure:: %d' % (exit_code[0]))
337        end_time = datetime.datetime.now()
338        log.notice('Coverage time: %s' % (str(end_time - start_time)))
339
340    def output_handler(self, text):
341        log.output('%s' % (text))
342
343class coverage_run(object):
344    '''
345    Coverage analysis support for rtems-test
346    '''
347    def __init__(self, macros_, executables, prefix, symbol_set = None, trace = False):
348        '''
349        Constructor
350        '''
351        self.trace = trace
352        self.macros = macros_
353        self.build_dir = self.macros['_cwd']
354        self.test_dir = path.join(self.build_dir, self.macros['bsp'] + '-coverage')
355        if not path.exists(self.test_dir):
356            path.mkdir(self.test_dir)
357        self.rtdir = path.abspath(self.macros['_rtdir'])
358        self.rtscripts = self.macros.expand(self.macros['_rtscripts'])
359        self.coverage_config_path = path.join(self.rtscripts, 'coverage')
360        self.symbol_config_path = path.join(self.coverage_config_path,
361                                            'symbol-sets.ini')
362        self.symbol_select_path = self.macros.expand(self.macros['bsp_symbol_path'])
363        self.executables = executables
364        self.symbol_sets = []
365        self.no_clean = int(self.macros['_no_clean'])
366        self.report_format = self.macros['cov_report_format']
367        self.symbol_set = symbol_set
368        self.target = self.macros['target']
369        self.bsp_name = self.macros['bsp'].split('-')[0]
370        self.prefix = prefix
371        self.macros.define('coverage')
372        self.covoar_cmd = self.macros.expand(self.macros['bsp_covoar_cmd'])
373
374    def run(self):
375        try:
376            if self.executables is None:
377                raise error.general('no test executables provided.')
378            build_dir = build_path_generator(self.executables, self.target).run()
379            parser = symbol_parser(self.symbol_config_path,
380                                   self.symbol_select_path,
381                                   self.symbol_set,
382                                   build_dir,
383                                   self.bsp_name,
384                                   self.target)
385            symbol_sets = parser.parse()
386            for sset in symbol_sets:
387                parser.write_ini(sset)
388                covoar_runner = covoar(self.test_dir,
389                                       self.symbol_select_path,
390                                       self.executables,
391                                       self.trace,
392                                       self.prefix,
393                                       self.covoar_cmd)
394                covoar_runner.run(sset, self.symbol_select_path)
395            self._generate_reports(symbol_sets);
396            self._summarize();
397        finally:
398            self._cleanup();
399
400    def _generate_reports(self, symbol_sets):
401        log.notice('Coverage generating reports')
402        if self.report_format == 'html':
403            report = report_gen_html(symbol_sets,
404                                     self.build_dir,
405                                     self.rtdir,
406                                     self.macros['bsp'])
407            report.generate()
408            report.add_covoar_css()
409
410    def _cleanup(self):
411        if not self.no_clean:
412            if self.trace:
413                log.output('Coverage cleaning tempfiles')
414            for exe in self.executables:
415                trace_file = exe + '.cov'
416                if path.exists(trace_file):
417                    os.remove(trace_file)
418            os.remove(self.symbol_select_path)
419
420    def _summarize(self):
421        log.notice('Coverage analysis finished: %s' % (self.build_dir))
Note: See TracBrowser for help on using the repository browser.