source: rtems-tools/tester/rt/test.py @ 5416cfa

5
Last change on this file since 5416cfa was e341a65, checked in by Chris Johns <chrisj@…>, on 06/18/18 at 00:13:47

tester: Make the path to covoar absolute to ignore the env PATH.

Using the environment's path to find covoar allow invalid versions
to be used which may vary in subtle ways. Find and use the covoar
that is build with the version of 'rtems-test'.

This patch means you do not need to install the tools before
running improving the development experience.

Closes #3458
  • Property mode set to 100644
File size: 15.9 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2013-2018 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import copy
34import datetime
35import fnmatch
36import os
37import re
38import sys
39import threading
40import time
41
42from rtemstoolkit import configuration
43from rtemstoolkit import error
44from rtemstoolkit import host
45from rtemstoolkit import log
46from rtemstoolkit import path
47from rtemstoolkit import mailer
48from rtemstoolkit import reraise
49from rtemstoolkit import stacktraces
50from rtemstoolkit import version
51from rtemstoolkit import check
52
53from . import bsps
54from . import config
55from . import console
56from . import options
57from . import report
58from . import coverage
59
60class log_capture(object):
61    def __init__(self):
62        self.log = []
63        log.capture = self.capture
64
65    def __str__(self):
66        return os.linesep.join(self.log)
67
68    def capture(self, text):
69        self.log += [l for l in text.replace(chr(13), '').splitlines()]
70
71    def get(self):
72        s = []
73        status = []
74        status_regx = re.compile('^\[\s*\d+/\s*\d+\] p:.+')
75        for l in self.log:
76            if status_regx.match(l):
77                status += [l]
78            else:
79                if len(status) == 1:
80                    s += [status[0]]
81                    status = []
82                elif len(status) > 1:
83                    s += [status[0]]
84                    if len(status) > 2:
85                        s += [' <<skipping passes>>']
86                    s += [status[-1]]
87                    status = []
88                s += [l]
89        return s
90
91class test(object):
92    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
93        self.index = index
94        self.total = total
95        self.report = report
96        self.bsp = bsp
97        self.bsp_config = bsp_config
98        self.opts = copy.copy(opts)
99        self.opts.defaults['test_index'] = str(index)
100        self.opts.defaults['test_total'] = str(total)
101        self.opts.defaults['bsp'] = bsp
102        self.opts.defaults['bsp_arch'] = '%{arch}'
103        if not path.isfile(executable):
104            raise error.general('cannot find executable: %s' % (executable))
105        self.opts.defaults['test_executable'] = executable
106        if rtems_tools:
107            rtems_tools_bin = path.join(self.opts.defaults.expand(rtems_tools), 'bin')
108            if not path.isdir(rtems_tools_bin):
109                raise error.general('cannot find RTEMS tools path: %s' % (rtems_tools_bin))
110            self.opts.defaults['rtems_tools'] = rtems_tools_bin
111        self.config = config.file(index, total, self.report, self.bsp_config, self.opts)
112
113    def run(self):
114        if self.config:
115            self.config.run()
116
117    def kill(self):
118        if self.config:
119            self.config.kill()
120
121class test_run(object):
122    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
123        self.test = None
124        self.result = None
125        self.start_time = None
126        self.end_time = None
127        self.index = copy.copy(index)
128        self.total = total
129        self.report = report
130        self.executable = copy.copy(executable)
131        self.rtems_tools = rtems_tools
132        self.bsp = bsp
133        self.bsp_config = bsp_config
134        self.opts = opts
135
136    def runner(self):
137        self.start_time = datetime.datetime.now()
138        try:
139            self.test = test(self.index, self.total, self.report,
140                             self.executable, self.rtems_tools,
141                             self.bsp, self.bsp_config,
142                             self.opts)
143            self.test.run()
144        except KeyboardInterrupt:
145            pass
146        except:
147            self.result = sys.exc_info()
148        self.end_time = datetime.datetime.now()
149
150    def run(self):
151        self.thread = threading.Thread(target = self.runner,
152                            name = 'test[%s]' % path.basename(self.executable))
153        self.thread.start()
154
155    def is_alive(self):
156        return self.thread and self.thread.is_alive()
157
158    def reraise(self):
159        if self.result is not None:
160            reraise.reraise(*self.result)
161
162    def kill(self):
163        if self.test:
164            self.test.kill()
165
166def find_executables(paths, glob):
167    executables = []
168    for p in paths:
169        if path.isfile(p):
170            executables += [p]
171        elif path.isdir(p):
172            for root, dirs, files in os.walk(p, followlinks = True):
173                for f in files:
174                    if fnmatch.fnmatch(f.lower(), glob):
175                        executables += [path.join(root, f)]
176    norun = re.compile('.*\.norun.*')
177    executables = [e for e in executables if not norun.match(e)]
178    return sorted(executables)
179
180def report_finished(reports, report_mode, reporting, finished, job_trace):
181    processing = True
182    while processing:
183        processing = False
184        reported = []
185        for tst in finished:
186            if tst not in reported and \
187                    (reporting < 0 or tst.index == reporting):
188                if job_trace:
189                    log.notice('}} %*d: %s: %s (%d)' % (len(str(tst.total)), tst.index,
190                                                        path.basename(tst.executable),
191                                                        'reporting',
192                                                        reporting))
193                processing = True
194                reports.log(tst.executable, report_mode)
195                reported += [tst]
196                reporting += 1
197        finished[:] = [t for t in finished if t not in reported]
198        if len(reported):
199            del reported[:]
200            if job_trace:
201                print('}} threading:', threading.active_count())
202                for t in threading.enumerate():
203                    print('}} ', t.name)
204    return reporting
205
206def _job_trace(tst, msg, total, exe, active, reporting):
207    s = ''
208    for a in active:
209        s += ' %d:%s' % (a.index, path.basename(a.executable))
210    log.notice('}} %*d: %s: %s (%d %d %d%s)' % (len(str(tst.total)), tst.index,
211                                                path.basename(tst.executable),
212                                                msg,
213                                                reporting, total, exe, s))
214
215def killall(tests):
216    for test in tests:
217        test.kill()
218
219def run(command_path = None):
220    import sys
221    tests = []
222    stdtty = console.save()
223    opts = None
224    default_exefilter = '*.exe'
225    try:
226        optargs = { '--rtems-tools':    'The path to the RTEMS tools',
227                    '--rtems-bsp':      'The RTEMS BSP to run the test on',
228                    '--user-config':    'Path to your local user configuration INI file',
229                    '--report-mode':    'Reporting modes, failures (default),all,none',
230                    '--list-bsps':      'List the supported BSPs',
231                    '--debug-trace':    'Debug trace based on specific flags (console,gdb,output,cov)',
232                    '--filter':         'Glob that executables must match to run (default: ' +
233                              default_exefilter + ')',
234                    '--stacktrace':     'Dump a stack trace on a user termination (^C)',
235                    '--coverage':       'Perform coverage analysis of test executables.'}
236        mailer.append_options(optargs)
237        opts = options.load(sys.argv,
238                            optargs = optargs,
239                            command_path = command_path)
240        mail = None
241        output = None
242        if opts.find_arg('--mail'):
243            mail = mailer.mail(opts)
244            # Request these now to generate any errors.
245            from_addr = mail.from_address()
246            smtp_host = mail.smtp_host()
247            to_addr = opts.find_arg('--mail-to')
248            if to_addr:
249                to_addr = to_addr[1]
250            else:
251                to_addr = 'build@rtems.org'
252            output = log_capture()
253        log.notice('RTEMS Testing - Tester, %s' % (version.str()))
254        if opts.find_arg('--list-bsps'):
255            bsps.list(opts)
256        exe_filter = opts.find_arg('--filter')
257        if exe_filter:
258            exe_filter = exe_filter[1]
259        else:
260            exe_filter = default_exefilter
261        opts.log_info()
262        log.output('Host: ' + host.label(mode = 'all'))
263        executables = find_executables(opts.params(), exe_filter)
264        debug_trace = opts.find_arg('--debug-trace')
265        if debug_trace:
266            if len(debug_trace) != 1:
267                debug_trace = debug_trace[1]
268            else:
269                raise error.general('no debug flags, can be: console,gdb,output,cov')
270        else:
271            debug_trace = ''
272        opts.defaults['exe_trace'] = debug_trace
273        job_trace = 'jobs' in debug_trace.split(',')
274        rtems_tools = opts.find_arg('--rtems-tools')
275        if rtems_tools:
276            if len(rtems_tools) != 2:
277                raise error.general('invalid RTEMS tools option')
278            rtems_tools = rtems_tools[1]
279        else:
280            rtems_tools = '%{_prefix}'
281        bsp = opts.find_arg('--rtems-bsp')
282        if bsp is None or len(bsp) != 2:
283            raise error.general('RTEMS BSP not provided or an invalid option')
284        bsp = config.load(bsp[1], opts)
285        bsp_config = opts.defaults.expand(opts.defaults['tester'])
286        coverage_enabled = opts.find_arg('--coverage')
287        if coverage_enabled:
288            cov_trace = 'cov' in debug_trace.split(',')
289            if len(coverage_enabled) == 2:
290                coverage_runner = coverage.coverage_run(opts.defaults,
291                                                        executables,
292                                                        symbol_set = coverage_enabled[1],
293                                                        trace = cov_trace)
294            else:
295                coverage_runner = coverage.coverage_run(opts.defaults,
296                                                        executables,
297                                                        trace = cov_trace)
298        report_mode = opts.find_arg('--report-mode')
299        if report_mode:
300            if report_mode[1] != 'failures' and \
301                    report_mode[1] != 'all' and \
302                    report_mode[1] != 'none':
303                raise error.general('invalid report mode')
304            report_mode = report_mode[1]
305        else:
306            report_mode = 'failures'
307        if len(executables) == 0:
308            raise error.general('no executables supplied')
309        start_time = datetime.datetime.now()
310        total = len(executables)
311        reports = report.report(total)
312        reporting = 1
313        jobs = int(opts.jobs(opts.defaults['_ncpus']))
314        exe = 0
315        finished = []
316        if jobs > len(executables):
317            jobs = len(executables)
318        while exe < total or len(tests) > 0:
319            if exe < total and len(tests) < jobs:
320                tst = test_run(exe + 1, total, reports,
321                               executables[exe],
322                               rtems_tools, bsp, bsp_config,
323                               opts)
324                exe += 1
325                tests += [tst]
326                if job_trace:
327                    _job_trace(tst, 'create',
328                               total, exe, tests, reporting)
329                tst.run()
330            else:
331                dead = [t for t in tests if not t.is_alive()]
332                tests[:] = [t for t in tests if t not in dead]
333                for tst in dead:
334                    if job_trace:
335                        _job_trace(tst, 'dead',
336                                   total, exe, tests, reporting)
337                    finished += [tst]
338                    tst.reraise()
339                del dead
340                if len(tests) >= jobs or exe >= total:
341                    time.sleep(0.250)
342                if len(finished):
343                    reporting = report_finished(reports,
344                                                report_mode,
345                                                reporting,
346                                                finished,
347                                                job_trace)
348        finished_time = datetime.datetime.now()
349        reporting = report_finished(reports, report_mode,
350                                    reporting, finished, job_trace)
351        if reporting < total:
352            log.warning('finished jobs does match: %d' % (reporting))
353            report_finished(reports, report_mode, -1, finished, job_trace)
354        reports.summary()
355        end_time = datetime.datetime.now()
356        average_time = 'Average test time: %s' % (str((end_time - start_time) / total))
357        total_time = 'Testing time     : %s' % (str(end_time - start_time))
358        log.notice(average_time)
359        log.notice(total_time)
360        if mail is not None and output is not None:
361            m_arch = opts.defaults.expand('%{arch}')
362            m_bsp = opts.defaults.expand('%{bsp}')
363            build = ' %s:' % (reports.get_config('build', not_found = ''))
364            subject = '[rtems-test] %s/%s:%s %s' % (m_arch,
365                                                    m_bsp,
366                                                    build,
367                                                    reports.score_card('short'))
368            np = 'Not present in test'
369            ver = reports.get_config('version', not_found = np)
370            build = reports.get_config('build', not_found = np)
371            tools = reports.get_config('tools', not_found = np)
372            body = [total_time, average_time,
373                    '', 'Host', '====', host.label(mode = 'all'),
374                    '', 'Configuration', '=============',
375                    'Version: %s' % (ver),
376                    'Build  : %s' % (build),
377                    'Tools  : %s' % (tools),
378                    '', 'Summary', '=======', '',
379                    reports.score_card(), '',
380                    reports.failures(),
381                    'Log', '===', ''] + output.get()
382            mail.send(to_addr, subject, os.linesep.join(body))
383        if coverage_enabled:
384            coverage_runner.run()
385
386    except error.general as gerr:
387        print(gerr)
388        sys.exit(1)
389    except error.internal as ierr:
390        print(ierr)
391        sys.exit(1)
392    except error.exit:
393        sys.exit(2)
394    except KeyboardInterrupt:
395        if opts is not None and opts.find_arg('--stacktrace'):
396            print('}} dumping:', threading.active_count())
397            for t in threading.enumerate():
398                print('}} ', t.name)
399            print(stacktraces.trace())
400        log.notice('abort: user terminated')
401        killall(tests)
402        sys.exit(1)
403    finally:
404        console.restore(stdtty)
405    sys.exit(0)
406
407if __name__ == "__main__":
408    run()
Note: See TracBrowser for help on using the repository browser.