[50fdf12] | 1 | # |
---|
| 2 | # RTEMS Tools Project (http://www.rtems.org/) |
---|
[b7c4753] | 3 | # Copyright 2013-2018 Chris Johns (chrisj@rtems.org) |
---|
[50fdf12] | 4 | # All rights reserved. |
---|
| 5 | # |
---|
| 6 | # This file is part of the RTEMS Tools package in 'rtems-tools'. |
---|
| 7 | # |
---|
| 8 | # Redistribution and use in source and binary forms, with or without |
---|
| 9 | # modification, are permitted provided that the following conditions are met: |
---|
| 10 | # |
---|
| 11 | # 1. Redistributions of source code must retain the above copyright notice, |
---|
| 12 | # this list of conditions and the following disclaimer. |
---|
| 13 | # |
---|
| 14 | # 2. Redistributions in binary form must reproduce the above copyright notice, |
---|
| 15 | # this list of conditions and the following disclaimer in the documentation |
---|
| 16 | # and/or other materials provided with the distribution. |
---|
| 17 | # |
---|
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
---|
| 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
---|
| 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
---|
| 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
---|
| 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
---|
| 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
---|
| 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
---|
| 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
---|
| 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
---|
| 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
---|
| 28 | # POSSIBILITY OF SUCH DAMAGE. |
---|
| 29 | # |
---|
| 30 | |
---|
[b0fa2ae] | 31 | from __future__ import print_function |
---|
| 32 | |
---|
[50fdf12] | 33 | import copy |
---|
| 34 | import datetime |
---|
[b0fa2ae] | 35 | import fnmatch |
---|
[50fdf12] | 36 | import os |
---|
[cec5878] | 37 | import re |
---|
[50fdf12] | 38 | import sys |
---|
| 39 | import threading |
---|
| 40 | import time |
---|
| 41 | |
---|
[bf58911] | 42 | from rtemstoolkit import configuration |
---|
[50fdf12] | 43 | from rtemstoolkit import error |
---|
[c68beb8] | 44 | from rtemstoolkit import host |
---|
[50fdf12] | 45 | from rtemstoolkit import log |
---|
| 46 | from rtemstoolkit import path |
---|
[89e8c2a] | 47 | from rtemstoolkit import mailer |
---|
[3a867a4] | 48 | from rtemstoolkit import reraise |
---|
[c04a849] | 49 | from rtemstoolkit import stacktraces |
---|
[efc4f09] | 50 | from rtemstoolkit import version |
---|
[b762312] | 51 | from rtemstoolkit import check |
---|
[50fdf12] | 52 | |
---|
[b0fa2ae] | 53 | from . import bsps |
---|
| 54 | from . import config |
---|
| 55 | from . import console |
---|
| 56 | from . import options |
---|
| 57 | from . import report |
---|
[b762312] | 58 | from . import coverage |
---|
[50fdf12] | 59 | |
---|
[89e8c2a] | 60 | class log_capture(object): |
---|
| 61 | def __init__(self): |
---|
| 62 | self.log = [] |
---|
| 63 | log.capture = self.capture |
---|
| 64 | |
---|
| 65 | def __str__(self): |
---|
| 66 | return os.linesep.join(self.log) |
---|
| 67 | |
---|
| 68 | def capture(self, text): |
---|
| 69 | self.log += [l for l in text.replace(chr(13), '').splitlines()] |
---|
| 70 | |
---|
| 71 | def get(self): |
---|
[cec5878] | 72 | s = [] |
---|
| 73 | status = [] |
---|
| 74 | status_regx = re.compile('^\[\s*\d+/\s*\d+\] p:.+') |
---|
| 75 | for l in self.log: |
---|
| 76 | if status_regx.match(l): |
---|
| 77 | status += [l] |
---|
| 78 | else: |
---|
| 79 | if len(status) == 1: |
---|
| 80 | s += [status[0]] |
---|
| 81 | status = [] |
---|
| 82 | elif len(status) > 1: |
---|
| 83 | s += [status[0]] |
---|
| 84 | if len(status) > 2: |
---|
[bb218b6] | 85 | s += [' <<skipping passes>>'] |
---|
[cec5878] | 86 | s += [status[-1]] |
---|
| 87 | status = [] |
---|
| 88 | s += [l] |
---|
| 89 | return s |
---|
[89e8c2a] | 90 | |
---|
[50fdf12] | 91 | class test(object): |
---|
| 92 | def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts): |
---|
| 93 | self.index = index |
---|
| 94 | self.total = total |
---|
| 95 | self.report = report |
---|
| 96 | self.bsp = bsp |
---|
| 97 | self.bsp_config = bsp_config |
---|
| 98 | self.opts = copy.copy(opts) |
---|
| 99 | self.opts.defaults['test_index'] = str(index) |
---|
| 100 | self.opts.defaults['test_total'] = str(total) |
---|
| 101 | self.opts.defaults['bsp'] = bsp |
---|
[bf58911] | 102 | self.opts.defaults['bsp_arch'] = '%{arch}' |
---|
[50fdf12] | 103 | if not path.isfile(executable): |
---|
| 104 | raise error.general('cannot find executable: %s' % (executable)) |
---|
| 105 | self.opts.defaults['test_executable'] = executable |
---|
| 106 | if rtems_tools: |
---|
[9600c39] | 107 | rtems_tools_bin = path.join(self.opts.defaults.expand(rtems_tools), 'bin') |
---|
[50fdf12] | 108 | if not path.isdir(rtems_tools_bin): |
---|
| 109 | raise error.general('cannot find RTEMS tools path: %s' % (rtems_tools_bin)) |
---|
| 110 | self.opts.defaults['rtems_tools'] = rtems_tools_bin |
---|
[662e1f7] | 111 | self.config = config.file(index, total, self.report, self.bsp_config, self.opts) |
---|
[c04a849] | 112 | |
---|
| 113 | def run(self): |
---|
| 114 | if self.config: |
---|
| 115 | self.config.run() |
---|
| 116 | |
---|
| 117 | def kill(self): |
---|
| 118 | if self.config: |
---|
| 119 | self.config.kill() |
---|
[50fdf12] | 120 | |
---|
| 121 | class test_run(object): |
---|
| 122 | def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts): |
---|
| 123 | self.test = None |
---|
| 124 | self.result = None |
---|
| 125 | self.start_time = None |
---|
| 126 | self.end_time = None |
---|
| 127 | self.index = copy.copy(index) |
---|
| 128 | self.total = total |
---|
| 129 | self.report = report |
---|
| 130 | self.executable = copy.copy(executable) |
---|
| 131 | self.rtems_tools = rtems_tools |
---|
| 132 | self.bsp = bsp |
---|
| 133 | self.bsp_config = bsp_config |
---|
| 134 | self.opts = opts |
---|
| 135 | |
---|
| 136 | def runner(self): |
---|
| 137 | self.start_time = datetime.datetime.now() |
---|
| 138 | try: |
---|
| 139 | self.test = test(self.index, self.total, self.report, |
---|
| 140 | self.executable, self.rtems_tools, |
---|
| 141 | self.bsp, self.bsp_config, |
---|
| 142 | self.opts) |
---|
[c04a849] | 143 | self.test.run() |
---|
[50fdf12] | 144 | except KeyboardInterrupt: |
---|
| 145 | pass |
---|
| 146 | except: |
---|
| 147 | self.result = sys.exc_info() |
---|
| 148 | self.end_time = datetime.datetime.now() |
---|
| 149 | |
---|
| 150 | def run(self): |
---|
| 151 | self.thread = threading.Thread(target = self.runner, |
---|
[b762312] | 152 | name = 'test[%s]' % path.basename(self.executable)) |
---|
[50fdf12] | 153 | self.thread.start() |
---|
| 154 | |
---|
| 155 | def is_alive(self): |
---|
| 156 | return self.thread and self.thread.is_alive() |
---|
| 157 | |
---|
| 158 | def reraise(self): |
---|
| 159 | if self.result is not None: |
---|
[3a867a4] | 160 | reraise.reraise(*self.result) |
---|
[50fdf12] | 161 | |
---|
[c04a849] | 162 | def kill(self): |
---|
| 163 | if self.test: |
---|
| 164 | self.test.kill() |
---|
| 165 | |
---|
[803b63d] | 166 | def find_executables(paths, glob): |
---|
[50fdf12] | 167 | executables = [] |
---|
| 168 | for p in paths: |
---|
| 169 | if path.isfile(p): |
---|
| 170 | executables += [p] |
---|
| 171 | elif path.isdir(p): |
---|
| 172 | for root, dirs, files in os.walk(p, followlinks = True): |
---|
| 173 | for f in files: |
---|
[803b63d] | 174 | if fnmatch.fnmatch(f.lower(), glob): |
---|
[50fdf12] | 175 | executables += [path.join(root, f)] |
---|
[b7c4753] | 176 | norun = re.compile('.*\.norun.*') |
---|
| 177 | executables = [e for e in executables if not norun.match(e)] |
---|
[50fdf12] | 178 | return sorted(executables) |
---|
| 179 | |
---|
| 180 | def report_finished(reports, report_mode, reporting, finished, job_trace): |
---|
| 181 | processing = True |
---|
| 182 | while processing: |
---|
| 183 | processing = False |
---|
| 184 | reported = [] |
---|
| 185 | for tst in finished: |
---|
| 186 | if tst not in reported and \ |
---|
| 187 | (reporting < 0 or tst.index == reporting): |
---|
| 188 | if job_trace: |
---|
| 189 | log.notice('}} %*d: %s: %s (%d)' % (len(str(tst.total)), tst.index, |
---|
| 190 | path.basename(tst.executable), |
---|
| 191 | 'reporting', |
---|
| 192 | reporting)) |
---|
| 193 | processing = True |
---|
| 194 | reports.log(tst.executable, report_mode) |
---|
| 195 | reported += [tst] |
---|
| 196 | reporting += 1 |
---|
| 197 | finished[:] = [t for t in finished if t not in reported] |
---|
| 198 | if len(reported): |
---|
| 199 | del reported[:] |
---|
| 200 | if job_trace: |
---|
[04a5204] | 201 | print('}} threading:', threading.active_count()) |
---|
[50fdf12] | 202 | for t in threading.enumerate(): |
---|
[04a5204] | 203 | print('}} ', t.name) |
---|
[50fdf12] | 204 | return reporting |
---|
| 205 | |
---|
| 206 | def _job_trace(tst, msg, total, exe, active, reporting): |
---|
| 207 | s = '' |
---|
| 208 | for a in active: |
---|
| 209 | s += ' %d:%s' % (a.index, path.basename(a.executable)) |
---|
| 210 | log.notice('}} %*d: %s: %s (%d %d %d%s)' % (len(str(tst.total)), tst.index, |
---|
| 211 | path.basename(tst.executable), |
---|
| 212 | msg, |
---|
| 213 | reporting, total, exe, s)) |
---|
| 214 | |
---|
[c04a849] | 215 | def killall(tests): |
---|
| 216 | for test in tests: |
---|
| 217 | test.kill() |
---|
| 218 | |
---|
[50fdf12] | 219 | def run(command_path = None): |
---|
| 220 | import sys |
---|
[c04a849] | 221 | tests = [] |
---|
[50fdf12] | 222 | stdtty = console.save() |
---|
| 223 | opts = None |
---|
[803b63d] | 224 | default_exefilter = '*.exe' |
---|
[50fdf12] | 225 | try: |
---|
[b762312] | 226 | optargs = { '--rtems-tools': 'The path to the RTEMS tools', |
---|
| 227 | '--rtems-bsp': 'The RTEMS BSP to run the test on', |
---|
| 228 | '--user-config': 'Path to your local user configuration INI file', |
---|
| 229 | '--report-mode': 'Reporting modes, failures (default),all,none', |
---|
| 230 | '--list-bsps': 'List the supported BSPs', |
---|
| 231 | '--debug-trace': 'Debug trace based on specific flags', |
---|
| 232 | '--filter': 'Glob that executables must match to run (default: ' + |
---|
[803b63d] | 233 | default_exefilter + ')', |
---|
[b762312] | 234 | '--stacktrace': 'Dump a stack trace on a user termination (^C)', |
---|
| 235 | '--coverage': 'Perform coverage analysis of test executables.'} |
---|
[89e8c2a] | 236 | mailer.append_options(optargs) |
---|
[50fdf12] | 237 | opts = options.load(sys.argv, |
---|
| 238 | optargs = optargs, |
---|
| 239 | command_path = command_path) |
---|
[89e8c2a] | 240 | mail = None |
---|
| 241 | output = None |
---|
| 242 | if opts.find_arg('--mail'): |
---|
| 243 | mail = mailer.mail(opts) |
---|
[c73e030] | 244 | # Request these now to generate any errors. |
---|
| 245 | from_addr = mail.from_address() |
---|
| 246 | smtp_host = mail.smtp_host() |
---|
| 247 | to_addr = opts.find_arg('--mail-to') |
---|
| 248 | if to_addr: |
---|
| 249 | to_addr = to_addr[1] |
---|
| 250 | else: |
---|
| 251 | to_addr = 'build@rtems.org' |
---|
[89e8c2a] | 252 | output = log_capture() |
---|
[efc4f09] | 253 | log.notice('RTEMS Testing - Tester, %s' % (version.str())) |
---|
[50fdf12] | 254 | if opts.find_arg('--list-bsps'): |
---|
[c04a849] | 255 | bsps.list(opts) |
---|
[803b63d] | 256 | exe_filter = opts.find_arg('--filter') |
---|
| 257 | if exe_filter: |
---|
| 258 | exe_filter = exe_filter[1] |
---|
| 259 | else: |
---|
| 260 | exe_filter = default_exefilter |
---|
[50fdf12] | 261 | opts.log_info() |
---|
[c68beb8] | 262 | log.output('Host: ' + host.label(mode = 'all')) |
---|
[b762312] | 263 | executables = find_executables(opts.params(), exe_filter) |
---|
[50fdf12] | 264 | debug_trace = opts.find_arg('--debug-trace') |
---|
| 265 | if debug_trace: |
---|
[6f8b07c] | 266 | if len(debug_trace) != 1: |
---|
| 267 | debug_trace = debug_trace[1] |
---|
| 268 | else: |
---|
| 269 | raise error.general('no debug flags, can be: console,gdb,output') |
---|
[50fdf12] | 270 | else: |
---|
| 271 | debug_trace = '' |
---|
[c68beb8] | 272 | opts.defaults['exe_trace'] = debug_trace |
---|
[50fdf12] | 273 | job_trace = 'jobs' in debug_trace.split(',') |
---|
| 274 | rtems_tools = opts.find_arg('--rtems-tools') |
---|
| 275 | if rtems_tools: |
---|
[9600c39] | 276 | if len(rtems_tools) != 2: |
---|
| 277 | raise error.general('invalid RTEMS tools option') |
---|
[50fdf12] | 278 | rtems_tools = rtems_tools[1] |
---|
[9600c39] | 279 | else: |
---|
| 280 | rtems_tools = '%{_prefix}' |
---|
[50fdf12] | 281 | bsp = opts.find_arg('--rtems-bsp') |
---|
[9600c39] | 282 | if bsp is None or len(bsp) != 2: |
---|
[bf58911] | 283 | raise error.general('RTEMS BSP not provided or an invalid option') |
---|
[42e39d2] | 284 | bsp = config.load(bsp[1], opts) |
---|
[bf58911] | 285 | bsp_config = opts.defaults.expand(opts.defaults['tester']) |
---|
[b762312] | 286 | coverage_enabled = opts.find_arg('--coverage') |
---|
| 287 | if coverage_enabled: |
---|
| 288 | if len(coverage_enabled) == 2: |
---|
| 289 | coverage_runner = coverage.coverage_run(opts.defaults, |
---|
| 290 | coverage_enabled[1], |
---|
| 291 | executables) |
---|
| 292 | else: |
---|
| 293 | coverage_runner = coverage.coverage_run(opts.defaults, 0, |
---|
| 294 | executables) |
---|
[50fdf12] | 295 | report_mode = opts.find_arg('--report-mode') |
---|
| 296 | if report_mode: |
---|
| 297 | if report_mode[1] != 'failures' and \ |
---|
| 298 | report_mode[1] != 'all' and \ |
---|
| 299 | report_mode[1] != 'none': |
---|
| 300 | raise error.general('invalid report mode') |
---|
| 301 | report_mode = report_mode[1] |
---|
| 302 | else: |
---|
| 303 | report_mode = 'failures' |
---|
| 304 | if len(executables) == 0: |
---|
[e4014f0] | 305 | raise error.general('no executables supplied') |
---|
[50fdf12] | 306 | start_time = datetime.datetime.now() |
---|
| 307 | total = len(executables) |
---|
| 308 | reports = report.report(total) |
---|
| 309 | reporting = 1 |
---|
| 310 | jobs = int(opts.jobs(opts.defaults['_ncpus'])) |
---|
| 311 | exe = 0 |
---|
| 312 | finished = [] |
---|
| 313 | if jobs > len(executables): |
---|
| 314 | jobs = len(executables) |
---|
| 315 | while exe < total or len(tests) > 0: |
---|
| 316 | if exe < total and len(tests) < jobs: |
---|
| 317 | tst = test_run(exe + 1, total, reports, |
---|
| 318 | executables[exe], |
---|
| 319 | rtems_tools, bsp, bsp_config, |
---|
| 320 | opts) |
---|
| 321 | exe += 1 |
---|
| 322 | tests += [tst] |
---|
| 323 | if job_trace: |
---|
| 324 | _job_trace(tst, 'create', |
---|
| 325 | total, exe, tests, reporting) |
---|
| 326 | tst.run() |
---|
| 327 | else: |
---|
| 328 | dead = [t for t in tests if not t.is_alive()] |
---|
| 329 | tests[:] = [t for t in tests if t not in dead] |
---|
| 330 | for tst in dead: |
---|
| 331 | if job_trace: |
---|
| 332 | _job_trace(tst, 'dead', |
---|
| 333 | total, exe, tests, reporting) |
---|
| 334 | finished += [tst] |
---|
| 335 | tst.reraise() |
---|
| 336 | del dead |
---|
| 337 | if len(tests) >= jobs or exe >= total: |
---|
| 338 | time.sleep(0.250) |
---|
| 339 | if len(finished): |
---|
| 340 | reporting = report_finished(reports, |
---|
| 341 | report_mode, |
---|
| 342 | reporting, |
---|
| 343 | finished, |
---|
| 344 | job_trace) |
---|
| 345 | finished_time = datetime.datetime.now() |
---|
| 346 | reporting = report_finished(reports, report_mode, |
---|
| 347 | reporting, finished, job_trace) |
---|
| 348 | if reporting < total: |
---|
| 349 | log.warning('finished jobs does match: %d' % (reporting)) |
---|
| 350 | report_finished(reports, report_mode, -1, finished, job_trace) |
---|
| 351 | reports.summary() |
---|
| 352 | end_time = datetime.datetime.now() |
---|
[89e8c2a] | 353 | average_time = 'Average test time: %s' % (str((end_time - start_time) / total)) |
---|
| 354 | total_time = 'Testing time : %s' % (str(end_time - start_time)) |
---|
| 355 | log.notice(average_time) |
---|
| 356 | log.notice(total_time) |
---|
| 357 | if mail is not None and output is not None: |
---|
[24d51f7] | 358 | m_arch = opts.defaults.expand('%{arch}') |
---|
| 359 | m_bsp = opts.defaults.expand('%{bsp}') |
---|
[30218f5] | 360 | build = ' %s:' % (reports.get_config('build', not_found = '')) |
---|
| 361 | subject = '[rtems-test] %s/%s:%s %s' % (m_arch, |
---|
| 362 | m_bsp, |
---|
| 363 | build, |
---|
| 364 | reports.score_card('short')) |
---|
| 365 | np = 'Not present in test' |
---|
| 366 | ver = reports.get_config('version', not_found = np) |
---|
| 367 | build = reports.get_config('build', not_found = np) |
---|
| 368 | tools = reports.get_config('tools', not_found = np) |
---|
[89e8c2a] | 369 | body = [total_time, average_time, |
---|
[c68beb8] | 370 | '', 'Host', '====', host.label(mode = 'all'), |
---|
[30218f5] | 371 | '', 'Configuration', '=============', |
---|
| 372 | 'Version: %s' % (ver), |
---|
| 373 | 'Build : %s' % (build), |
---|
| 374 | 'Tools : %s' % (tools), |
---|
[89e8c2a] | 375 | '', 'Summary', '=======', '', |
---|
| 376 | reports.score_card(), '', |
---|
| 377 | reports.failures(), |
---|
[30218f5] | 378 | 'Log', '===', ''] + output.get() |
---|
[89e8c2a] | 379 | mail.send(to_addr, subject, os.linesep.join(body)) |
---|
[b762312] | 380 | if coverage_enabled: |
---|
| 381 | coverage_runner.run() |
---|
[89e8c2a] | 382 | |
---|
[04a5204] | 383 | except error.general as gerr: |
---|
| 384 | print(gerr) |
---|
[50fdf12] | 385 | sys.exit(1) |
---|
[04a5204] | 386 | except error.internal as ierr: |
---|
| 387 | print(ierr) |
---|
[50fdf12] | 388 | sys.exit(1) |
---|
[04a5204] | 389 | except error.exit: |
---|
[50fdf12] | 390 | sys.exit(2) |
---|
| 391 | except KeyboardInterrupt: |
---|
[b7d48ef] | 392 | if opts is not None and opts.find_arg('--stacktrace'): |
---|
[04a5204] | 393 | print('}} dumping:', threading.active_count()) |
---|
[50fdf12] | 394 | for t in threading.enumerate(): |
---|
[04a5204] | 395 | print('}} ', t.name) |
---|
| 396 | print(stacktraces.trace()) |
---|
[50fdf12] | 397 | log.notice('abort: user terminated') |
---|
[c04a849] | 398 | killall(tests) |
---|
[50fdf12] | 399 | sys.exit(1) |
---|
| 400 | finally: |
---|
| 401 | console.restore(stdtty) |
---|
| 402 | sys.exit(0) |
---|
| 403 | |
---|
| 404 | if __name__ == "__main__": |
---|
| 405 | run() |
---|