[b762312] | 1 | # |
---|
| 2 | # RTEMS Tools Project (http://www.rtems.org/) |
---|
| 3 | # Copyright 2014 Krzysztof Miesowicz (krzysztof.miesowicz@gmail.com) |
---|
| 4 | # All rights reserved. |
---|
| 5 | # |
---|
| 6 | # This file is part of the RTEMS Tools package in 'rtems-tools'. |
---|
| 7 | # |
---|
| 8 | # Redistribution and use in source and binary forms, with or without |
---|
| 9 | # modification, are permitted provided that the following conditions are met: |
---|
| 10 | # |
---|
| 11 | # 1. Redistributions of source code must retain the above copyright notice, |
---|
| 12 | # this list of conditions and the following disclaimer. |
---|
| 13 | # |
---|
| 14 | # 2. Redistributions in binary form must reproduce the above copyright notice, |
---|
| 15 | # this list of conditions and the following disclaimer in the documentation |
---|
| 16 | # and/or other materials provided with the distribution. |
---|
| 17 | # |
---|
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' |
---|
| 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
---|
| 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
---|
| 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
---|
| 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
---|
| 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
---|
| 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
---|
| 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
---|
| 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
---|
| 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
---|
| 28 | # POSSIBILITY OF SUCH DAMAGE. |
---|
| 29 | # |
---|
| 30 | |
---|
| 31 | from rtemstoolkit import error |
---|
| 32 | from rtemstoolkit import path |
---|
| 33 | from rtemstoolkit import log |
---|
| 34 | from rtemstoolkit import execute |
---|
| 35 | from rtemstoolkit import macros |
---|
| 36 | |
---|
| 37 | from datetime import datetime |
---|
| 38 | |
---|
| 39 | from . import options |
---|
| 40 | |
---|
| 41 | import shutil |
---|
| 42 | import os |
---|
| 43 | |
---|
| 44 | try: |
---|
| 45 | import configparser |
---|
| 46 | except: |
---|
| 47 | import ConfigParser as configparser |
---|
| 48 | |
---|
| 49 | class summary: |
---|
| 50 | def __init__(self, p_summary_dir): |
---|
| 51 | self.summary_file_path = path.join(p_summary_dir, 'summary.txt') |
---|
| 52 | self.index_file_path = path.join(p_summary_dir, 'index.html') |
---|
| 53 | self.bytes_analyzed = 0 |
---|
| 54 | self.bytes_not_executed = 0 |
---|
| 55 | self.percentage_executed = 0.0 |
---|
| 56 | self.percentage_not_executed = 100.0 |
---|
| 57 | self.ranges_uncovered = 0 |
---|
| 58 | self.branches_uncovered = 0 |
---|
| 59 | self.branches_total = 0 |
---|
| 60 | self.branches_always_taken = 0 |
---|
| 61 | self.branches_never_taken = 0 |
---|
| 62 | self.percentage_branches_covered = 0.0 |
---|
| 63 | self.is_failure = False |
---|
| 64 | |
---|
| 65 | def parse(self): |
---|
| 66 | if(not path.exists(self.summary_file_path)): |
---|
| 67 | log.notice('summary file %s does not exist!' % (self.summary_file_path)) |
---|
| 68 | self.is_failure = True |
---|
| 69 | return |
---|
| 70 | |
---|
| 71 | with open(self.summary_file_path,'r') as summary_file: |
---|
| 72 | self.bytes_analyzed = self._get_next_with_colon(summary_file) |
---|
| 73 | self.bytes_not_executed = self._get_next_with_colon(summary_file) |
---|
| 74 | self.percentage_executed = self._get_next_with_colon(summary_file) |
---|
| 75 | self.percentage_not_executed = self._get_next_with_colon(summary_file) |
---|
| 76 | self.ranges_uncovered = self._get_next_with_colon(summary_file) |
---|
| 77 | self.branches_total = self._get_next_with_colon(summary_file) |
---|
| 78 | self.branches_uncovered = self._get_next_with_colon(summary_file) |
---|
| 79 | self.branches_always_taken = self._get_next_without_colon(summary_file) |
---|
| 80 | self.branches_never_taken = self._get_next_without_colon(summary_file) |
---|
| 81 | if len(self.branches_uncovered) > 0 and len(self.branches_total) > 0: |
---|
| 82 | self.percentage_branches_covered = \ |
---|
| 83 | 1 - (float(self.branches_uncovered) / float(self.branches_total)) |
---|
| 84 | else: |
---|
| 85 | self.percentage_branches_covered = 0.0 |
---|
| 86 | return |
---|
| 87 | |
---|
| 88 | def _get_next_with_colon(self, summary_file): |
---|
| 89 | line = summary_file.readline() |
---|
| 90 | if ':' in line: |
---|
| 91 | return line.split(':')[1].strip() |
---|
| 92 | else: |
---|
| 93 | return '' |
---|
| 94 | |
---|
| 95 | def _get_next_without_colon(self, summary_file): |
---|
| 96 | line = summary_file.readline() |
---|
| 97 | return line.strip().split(' ')[0] |
---|
| 98 | |
---|
| 99 | class report_gen_html: |
---|
| 100 | def __init__(self, p_symbol_sets_list, build_dir, rtdir, bsp): |
---|
| 101 | self.symbol_sets_list = ['score'] |
---|
| 102 | self.build_dir = build_dir |
---|
| 103 | self.partial_reports_files = list(["index.html", "summary.txt"]) |
---|
| 104 | self.number_of_columns = 1 |
---|
| 105 | self.covoar_src_path = path.join(rtdir, 'covoar') |
---|
| 106 | self.bsp = bsp |
---|
| 107 | |
---|
| 108 | def _find_partial_reports(self): |
---|
| 109 | partial_reports = {} |
---|
| 110 | for symbol_set in self.symbol_sets_list: |
---|
| 111 | set_summary = summary(path.join(self.build_dir, |
---|
| 112 | self.bsp + "-coverage", |
---|
| 113 | symbol_set)) |
---|
| 114 | set_summary.parse() |
---|
| 115 | partial_reports[symbol_set] = set_summary |
---|
| 116 | return partial_reports |
---|
| 117 | |
---|
| 118 | def _prepare_head_section(self): |
---|
| 119 | head_section = ''' |
---|
| 120 | <head> |
---|
| 121 | <title>RTEMS coverage report</title> |
---|
| 122 | <style type="text/css"> |
---|
| 123 | progress[value] { |
---|
| 124 | -webkit-appearance: none; |
---|
| 125 | appearance: none; |
---|
| 126 | |
---|
| 127 | width: 150px; |
---|
| 128 | height: 15px; |
---|
| 129 | } |
---|
| 130 | </style> |
---|
| 131 | </head>''' |
---|
| 132 | return head_section |
---|
| 133 | |
---|
| 134 | def _prepare_index_content(self, partial_reports): |
---|
| 135 | header = "<h1> RTEMS coverage analysis report </h1>" |
---|
| 136 | header += "<h3>Coverage reports by symbols sets:</h3>" |
---|
| 137 | table = "<table>" |
---|
| 138 | table += self._header_row() |
---|
| 139 | for symbol_set in partial_reports: |
---|
| 140 | table += self._row(symbol_set, partial_reports[symbol_set]) |
---|
| 141 | table += "</table> </br>" |
---|
| 142 | timestamp = "Analysis performed on " + datetime.now().ctime() |
---|
| 143 | return "<body>\n" + header + table + timestamp + "\n</body>" |
---|
| 144 | |
---|
| 145 | def _row(self, symbol_set, summary): |
---|
| 146 | row = "<tr>" |
---|
| 147 | row += "<td>" + symbol_set + "</td>" |
---|
| 148 | if summary.is_failure: |
---|
| 149 | row += ' <td colspan="' + str(self.number_of_columns-1) \ |
---|
| 150 | + '" style="background-color:red">FAILURE</td>' |
---|
| 151 | else: |
---|
| 152 | row += " <td>" + self._link(summary.index_file_path,"Index") \ |
---|
| 153 | + "</td>" |
---|
| 154 | row += " <td>" + self._link(summary.summary_file_path,"Summary") \ |
---|
| 155 | + "</td>" |
---|
| 156 | row += " <td>" + summary.bytes_analyzed + "</td>" |
---|
| 157 | row += " <td>" + summary.bytes_not_executed + "</td>" |
---|
| 158 | row += " <td>" + summary.ranges_uncovered + "</td>" |
---|
| 159 | row += " <td>" + summary.percentage_executed + "%</td>" |
---|
| 160 | row += " <td>" + summary.percentage_not_executed + "%</td>" |
---|
| 161 | row += ' <td><progress value="' + summary.percentage_executed \ |
---|
| 162 | + '" max="100"></progress></td>' |
---|
| 163 | row += " <td>" + summary.branches_uncovered + "</td>" |
---|
| 164 | row += " <td>" + summary.branches_total + "</td>" |
---|
| 165 | row += " <td> {:.3%} </td>".format(summary.percentage_branches_covered) |
---|
| 166 | row += ' <td><progress value="{:.3}" max="100"></progress></td>'.format(100*summary.percentage_branches_covered) |
---|
| 167 | row += "</tr>\n" |
---|
| 168 | return row |
---|
| 169 | |
---|
| 170 | def _header_row(self): |
---|
| 171 | row = "<tr>" |
---|
| 172 | row += "<th> Symbols set name </th>" |
---|
| 173 | row += "<th> Index file </th>" |
---|
| 174 | row += "<th> Summary file </th>" |
---|
| 175 | row += "<th> Bytes analyzed </th>" |
---|
| 176 | row += "<th> Bytes not executed </th>" |
---|
| 177 | row += "<th> Uncovered ranges </th>" |
---|
| 178 | row += "<th> Percentage covered </th>" |
---|
| 179 | row += "<th> Percentage uncovered </th>" |
---|
| 180 | row += "<th> Instruction coverage </th>" |
---|
| 181 | row += "<th> Branches uncovered </th>" |
---|
| 182 | row += "<th> Branches total </th>" |
---|
| 183 | row += "<th> Branches covered percentage </th>" |
---|
| 184 | row += "<th> Branches coverage </th>" |
---|
| 185 | row += "</tr>\n" |
---|
| 186 | self.number_of_columns = row.count('<th>') |
---|
| 187 | return row |
---|
| 188 | |
---|
| 189 | def _link(self, address, text): |
---|
| 190 | return '<a href="' + address + '">' + text + '</a>' |
---|
| 191 | |
---|
| 192 | def _create_index_file(self, head_section, content): |
---|
| 193 | with open(path.join(self.build_dir, |
---|
| 194 | self.bsp + "-report.html"),'w') as f: |
---|
| 195 | f.write(head_section) |
---|
| 196 | f.write(content) |
---|
| 197 | |
---|
| 198 | def generate(self): |
---|
| 199 | partial_reports = self._find_partial_reports() |
---|
| 200 | head_section = self._prepare_head_section() |
---|
| 201 | index_content = self._prepare_index_content(partial_reports) |
---|
| 202 | self._create_index_file(head_section,index_content) |
---|
| 203 | |
---|
| 204 | def add_covoar_src_path(self): |
---|
| 205 | table_js_path = path.join(self.covoar_src_path, 'table.js') |
---|
| 206 | covoar_css_path = path.join(self.covoar_src_path, 'covoar.css') |
---|
| 207 | for symbol_set in self.symbol_sets_list: |
---|
| 208 | symbol_set_dir = path.join(self.build_dir, |
---|
| 209 | self.bsp + '-coverage', symbol_set) |
---|
| 210 | html_files = os.listdir(symbol_set_dir) |
---|
| 211 | for html_file in html_files: |
---|
| 212 | html_file = path.join(symbol_set_dir, html_file) |
---|
| 213 | if path.exists(html_file) and 'html' in html_file: |
---|
| 214 | with open(html_file, 'r') as f: |
---|
| 215 | file_data = f.read() |
---|
| 216 | file_data = file_data.replace('table.js', table_js_path) |
---|
| 217 | file_data = file_data.replace('covoar.css', |
---|
| 218 | covoar_css_path) |
---|
| 219 | with open(html_file, 'w') as f: |
---|
| 220 | f.write(file_data) |
---|
| 221 | |
---|
| 222 | class build_path_generator(object): |
---|
| 223 | ''' |
---|
| 224 | Generates the build path from the path to executables |
---|
| 225 | ''' |
---|
| 226 | def __init__(self, executables, target): |
---|
| 227 | self.executables = executables |
---|
| 228 | self.target = target |
---|
| 229 | def run(self): |
---|
| 230 | build_path = '/' |
---|
| 231 | Path = self.executables[0].split('/') |
---|
| 232 | for P in Path: |
---|
| 233 | if P == self.target: |
---|
| 234 | break; |
---|
| 235 | else: |
---|
| 236 | build_path = path.join(build_path, P) |
---|
| 237 | return build_path |
---|
| 238 | |
---|
| 239 | class symbol_parser(object): |
---|
| 240 | ''' |
---|
| 241 | Parse the symbol sets ini and create custom ini file for covoar |
---|
| 242 | ''' |
---|
| 243 | def __init__(self, symbol_config_path, |
---|
| 244 | symbol_select_path, coverage_arg, build_dir): |
---|
| 245 | self.symbol_select_file = symbol_select_path |
---|
| 246 | self.symbol_file = symbol_config_path |
---|
| 247 | self.build_dir = build_dir |
---|
| 248 | self.symbol_sets = {} |
---|
| 249 | self.cov_arg = coverage_arg |
---|
| 250 | self.ssets = [] |
---|
| 251 | |
---|
| 252 | def parse(self): |
---|
| 253 | config = configparser.ConfigParser() |
---|
| 254 | try: |
---|
| 255 | config.read(self.symbol_file) |
---|
| 256 | if self.cov_arg: |
---|
| 257 | self.ssets = self.cov_arg.split(',') |
---|
| 258 | else: |
---|
| 259 | self.ssets = config.get('symbol-sets', 'sets').split(',') |
---|
| 260 | self.ssets = [ sset.encode('utf-8') for sset in self.ssets] |
---|
| 261 | for sset in self.ssets: |
---|
| 262 | lib = path.join(self.build_dir, |
---|
| 263 | config.get('libraries', sset)) |
---|
| 264 | self.symbol_sets[sset] = lib.encode('utf-8') |
---|
| 265 | except: |
---|
| 266 | raise error.general('Symbol set parsing failed') |
---|
| 267 | |
---|
| 268 | def _write_ini(self): |
---|
| 269 | config = configparser.ConfigParser() |
---|
| 270 | try: |
---|
| 271 | sets = ', '.join(self.symbol_sets.keys()) |
---|
| 272 | config.add_section('symbol-sets') |
---|
| 273 | config.set('symbol-sets', 'sets', sets) |
---|
| 274 | for key in self.symbol_sets.keys(): |
---|
| 275 | config.add_section(key) |
---|
| 276 | config.set(key, 'libraries', self.symbol_sets[key]) |
---|
| 277 | with open(self.symbol_select_file, 'w') as conf: |
---|
| 278 | config.write(conf) |
---|
| 279 | except: |
---|
| 280 | raise error.general('write failed') |
---|
| 281 | |
---|
| 282 | def run(self): |
---|
| 283 | self.parse() |
---|
| 284 | self._write_ini() |
---|
| 285 | |
---|
| 286 | class covoar(object): |
---|
| 287 | ''' |
---|
| 288 | Covoar runner |
---|
| 289 | ''' |
---|
| 290 | def __init__(self, base_result_dir, config_dir, executables, explanations_txt): |
---|
| 291 | self.base_result_dir = base_result_dir |
---|
| 292 | self.config_dir = config_dir |
---|
| 293 | self.executables = ' '.join(executables) |
---|
| 294 | self.explanations_txt = explanations_txt |
---|
| 295 | self.project_name = 'RTEMS-5' |
---|
| 296 | |
---|
| 297 | def run(self, set_name, symbol_file): |
---|
| 298 | covoar_result_dir = path.join(self.base_result_dir, set_name) |
---|
| 299 | if (not path.exists(covoar_result_dir)): |
---|
| 300 | path.mkdir(covoar_result_dir) |
---|
| 301 | if (not path.exists(symbol_file)): |
---|
| 302 | raise error.general('symbol set file: coverage %s was not created for covoar, skipping %s'% (symbol_file, set_name)) |
---|
| 303 | command = ('covoar -S ' + symbol_file |
---|
| 304 | + ' -O ' + covoar_result_dir |
---|
| 305 | + ' -E ' + self.explanations_txt |
---|
| 306 | + ' -p ' + self.project_name + ' ' + self.executables) |
---|
| 307 | log.notice('Running covoar for %s' % (set_name)) |
---|
| 308 | print( 'covoar results directory:\n' + covoar_result_dir ) |
---|
| 309 | executor = execute.execute(verbose = True, output = self.output_handler) |
---|
| 310 | exit_code = executor.shell(command, cwd=os.getcwd()) |
---|
| 311 | if (exit_code[0] != 0): |
---|
| 312 | raise error.general('covoar failure exit code: %d' % (exit_code[0])) |
---|
| 313 | log.notice('Coverage run for %s finished successfully.' % (set_name)) |
---|
| 314 | log.notice('-----------------------------------------------') |
---|
| 315 | |
---|
| 316 | def output_handler(self, text): |
---|
| 317 | log.notice('%s' % (text)) |
---|
| 318 | |
---|
| 319 | class coverage_run(object): |
---|
| 320 | ''' |
---|
| 321 | Coverage analysis support for rtems-test |
---|
| 322 | ''' |
---|
| 323 | def __init__(self, p_macros, coverage_arg, executables): |
---|
| 324 | ''' |
---|
| 325 | Constructor |
---|
| 326 | ''' |
---|
| 327 | self.macros = p_macros |
---|
| 328 | self.build_dir = self.macros['_cwd'] |
---|
| 329 | self.explanations_txt = self.macros.expand(self.macros['cov_explanations']) |
---|
| 330 | self.test_dir = path.join(self.build_dir, self.macros['bsp'] + '-coverage') |
---|
| 331 | if (not path.exists(self.test_dir)): |
---|
| 332 | path.mkdir(self.test_dir) |
---|
| 333 | self.rtdir = path.abspath(self.macros['_rtdir']) |
---|
| 334 | self.rtscripts = self.macros.expand(self.macros['_rtscripts']) |
---|
| 335 | self.coverage_config_path = path.join(self.rtscripts, 'coverage') |
---|
| 336 | self.symbol_config_path = path.join(self.coverage_config_path, |
---|
| 337 | 'symbol-sets.ini') |
---|
| 338 | self.symbol_select_path = path.join(self.coverage_config_path, |
---|
| 339 | self.macros['bsp'] + '-symbols.ini') |
---|
| 340 | self.executables = executables |
---|
| 341 | self.symbol_sets = [] |
---|
| 342 | self.no_clean = int(self.macros['_no_clean']) |
---|
| 343 | self.report_format = self.macros['cov_report_format'] |
---|
| 344 | self.coverage_arg = coverage_arg |
---|
| 345 | self.target = self.macros['target'] |
---|
| 346 | |
---|
| 347 | def run(self): |
---|
| 348 | try: |
---|
| 349 | if self.executables is None: |
---|
| 350 | raise error.general('no test executables provided.') |
---|
| 351 | build_dir = build_path_generator(self.executables, self.target).run() |
---|
| 352 | parser = symbol_parser(self.symbol_config_path, |
---|
| 353 | self.symbol_select_path, |
---|
| 354 | self.coverage_arg, |
---|
| 355 | build_dir) |
---|
| 356 | parser.run() |
---|
| 357 | covoar_runner = covoar(self.test_dir, self.symbol_select_path, |
---|
| 358 | self.executables, self.explanations_txt) |
---|
| 359 | covoar_runner.run('score', self.symbol_select_path) |
---|
| 360 | self._generate_reports(); |
---|
| 361 | self._summarize(); |
---|
| 362 | finally: |
---|
| 363 | self._cleanup(); |
---|
| 364 | |
---|
| 365 | def _generate_reports(self): |
---|
| 366 | log.notice('Generating reports') |
---|
| 367 | if self.report_format == 'html': |
---|
| 368 | report = report_gen_html(self.symbol_sets, |
---|
| 369 | self.build_dir, |
---|
| 370 | self.rtdir, |
---|
| 371 | self.macros['bsp']) |
---|
| 372 | report.generate() |
---|
| 373 | report.add_covoar_src_path() |
---|
| 374 | |
---|
| 375 | def _cleanup(self): |
---|
| 376 | if not self.no_clean: |
---|
| 377 | log.notice('***Cleaning tempfiles***') |
---|
| 378 | for exe in self.executables: |
---|
| 379 | trace_file = exe + '.cov' |
---|
| 380 | if path.exists(trace_file): |
---|
| 381 | os.remove(trace_file) |
---|
| 382 | os.remove(self.symbol_select_path) |
---|
| 383 | |
---|
| 384 | def _summarize(self): |
---|
| 385 | log.notice('Coverage analysis finished. You can find results in %s' % (self.build_dir)) |
---|