source: rtems-tools/tester/rt/test.py @ 6f8b07c

4.105
Last change on this file since 6f8b07c was 6f8b07c, checked in by Chris Johns <chrisj@…>, on 03/14/16 at 03:59:11

rtemstoolkit: Fix execute's writer thread to not eval() the input.

The conversion to Python3 added an eval() call which is wrong.

Fix the spelling in execute.

Fix labels in the tester gdb locking.

Check the debug-trace arguments.

Close #2642.

  • Property mode set to 100644
File size: 12.9 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2013-2014 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import copy
34import datetime
35import fnmatch
36import os
37import sys
38import threading
39import time
40
41from rtemstoolkit import error
42from rtemstoolkit import log
43from rtemstoolkit import path
44from rtemstoolkit import stacktraces
45from rtemstoolkit import version
46
47from . import bsps
48from . import config
49from . import console
50from . import options
51from . import report
52
53class test(object):
54    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
55        self.index = index
56        self.total = total
57        self.report = report
58        self.bsp = bsp
59        self.bsp_config = bsp_config
60        self.opts = copy.copy(opts)
61        self.opts.defaults['test_index'] = str(index)
62        self.opts.defaults['test_total'] = str(total)
63        self.opts.defaults['bsp'] = bsp
64        self.opts.defaults['bsp_arch'] = '%%{%s_arch}' % (bsp)
65        self.opts.defaults['bsp_opts'] = '%%{%s_opts}' % (bsp)
66        if not path.isfile(executable):
67            raise error.general('cannot find executable: %s' % (executable))
68        self.opts.defaults['test_executable'] = executable
69        if rtems_tools:
70            rtems_tools_bin = path.join(self.opts.defaults.expand(rtems_tools), 'bin')
71            if not path.isdir(rtems_tools_bin):
72                raise error.general('cannot find RTEMS tools path: %s' % (rtems_tools_bin))
73            self.opts.defaults['rtems_tools'] = rtems_tools_bin
74        self.config = config.file(self.report, self.bsp_config, self.opts)
75
76    def run(self):
77        if self.config:
78            self.config.run()
79
80    def kill(self):
81        if self.config:
82            self.config.kill()
83
84class test_run(object):
85    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
86        self.test = None
87        self.result = None
88        self.start_time = None
89        self.end_time = None
90        self.index = copy.copy(index)
91        self.total = total
92        self.report = report
93        self.executable = copy.copy(executable)
94        self.rtems_tools = rtems_tools
95        self.bsp = bsp
96        self.bsp_config = bsp_config
97        self.opts = opts
98
99    def runner(self):
100        self.start_time = datetime.datetime.now()
101        try:
102            self.test = test(self.index, self.total, self.report,
103                             self.executable, self.rtems_tools,
104                             self.bsp, self.bsp_config,
105                             self.opts)
106            self.test.run()
107        except KeyboardInterrupt:
108            pass
109        except:
110            self.result = sys.exc_info()
111        self.end_time = datetime.datetime.now()
112
113    def run(self):
114        self.thread = threading.Thread(target = self.runner,
115                                       name = 'test[%s]' % path.basename(self.executable))
116        self.thread.start()
117
118    def is_alive(self):
119        return self.thread and self.thread.is_alive()
120
121    def reraise(self):
122        if self.result is not None:
123            with_tb = getattr(self.result[1], 'with_traceback', None)
124            if with_tb:
125                raise self.result[1].with_traceback(self.result[2])
126            raise (self.result[0], self.result[1], self.result[2])
127
128    def kill(self):
129        if self.test:
130            self.test.kill()
131
132def find_executables(paths, glob):
133    executables = []
134    for p in paths:
135        if path.isfile(p):
136            executables += [p]
137        elif path.isdir(p):
138            for root, dirs, files in os.walk(p, followlinks = True):
139                for f in files:
140                    if fnmatch.fnmatch(f.lower(), glob):
141                        executables += [path.join(root, f)]
142    return sorted(executables)
143
144def report_finished(reports, report_mode, reporting, finished, job_trace):
145    processing = True
146    while processing:
147        processing = False
148        reported = []
149        for tst in finished:
150            if tst not in reported and \
151                    (reporting < 0 or tst.index == reporting):
152                if job_trace:
153                    log.notice('}} %*d: %s: %s (%d)' % (len(str(tst.total)), tst.index,
154                                                        path.basename(tst.executable),
155                                                        'reporting',
156                                                        reporting))
157                processing = True
158                reports.log(tst.executable, report_mode)
159                reported += [tst]
160                reporting += 1
161        finished[:] = [t for t in finished if t not in reported]
162        if len(reported):
163            del reported[:]
164            if job_trace:
165                print('}} threading:', threading.active_count())
166                for t in threading.enumerate():
167                    print('}} ', t.name)
168    return reporting
169
170def _job_trace(tst, msg, total, exe, active, reporting):
171    s = ''
172    for a in active:
173        s += ' %d:%s' % (a.index, path.basename(a.executable))
174    log.notice('}} %*d: %s: %s (%d %d %d%s)' % (len(str(tst.total)), tst.index,
175                                                path.basename(tst.executable),
176                                                msg,
177                                                reporting, total, exe, s))
178
179def list_bsps(opts):
180    path_ = opts.defaults.expand('%%{_configdir}/bsps/*.mc')
181    bsps = path.collect_files(path_)
182    log.notice(' BSP List:')
183    for bsp in bsps:
184        log.notice('  %s' % (path.basename(bsp[:-3])))
185    raise error.exit()
186
187def killall(tests):
188    for test in tests:
189        test.kill()
190
191def run(command_path = None):
192    import sys
193    tests = []
194    stdtty = console.save()
195    opts = None
196    default_exefilter = '*.exe'
197    try:
198        optargs = { '--rtems-tools': 'The path to the RTEMS tools',
199                    '--rtems-bsp':   'The RTEMS BSP to run the test on',
200                    '--report-mode': 'Reporting modes, failures (default),all,none',
201                    '--list-bsps':   'List the supported BSPs',
202                    '--debug-trace': 'Debug trace based on specific flags',
203                    '--filter':      'Glob that executables must match to run (default: ' +
204                              default_exefilter + ')',
205                    '--stacktrace':  'Dump a stack trace on a user termination (^C)' }
206        opts = options.load(sys.argv,
207                            optargs = optargs,
208                            command_path = command_path)
209        log.notice('RTEMS Testing - Tester, %s' % (version.str()))
210        if opts.find_arg('--list-bsps'):
211            bsps.list(opts)
212        exe_filter = opts.find_arg('--filter')
213        if exe_filter:
214            exe_filter = exe_filter[1]
215        else:
216            exe_filter = default_exefilter
217        opts.log_info()
218        debug_trace = opts.find_arg('--debug-trace')
219        if debug_trace:
220            if len(debug_trace) != 1:
221                debug_trace = debug_trace[1]
222            else:
223                raise error.general('no debug flags, can be: console,gdb,output')
224        else:
225            debug_trace = ''
226        opts.defaults['debug_trace'] = debug_trace
227        job_trace = 'jobs' in debug_trace.split(',')
228        rtems_tools = opts.find_arg('--rtems-tools')
229        if rtems_tools:
230            if len(rtems_tools) != 2:
231                raise error.general('invalid RTEMS tools option')
232            rtems_tools = rtems_tools[1]
233        else:
234            rtems_tools = '%{_prefix}'
235        bsp = opts.find_arg('--rtems-bsp')
236        if bsp is None or len(bsp) != 2:
237            raise error.general('RTEMS BSP not provided or invalid option')
238        opts.defaults.load('%%{_configdir}/bsps/%s.mc' % (bsp[1]))
239        bsp = opts.defaults.get('%{bsp}')
240        if not bsp:
241            raise error.general('BSP definition (%{bsp}) not found in the global map')
242        bsp = bsp[2]
243        if not opts.defaults.set_read_map(bsp):
244            raise error.general('no BSP map found')
245        bsp_script = opts.defaults.get(bsp)
246        if not bsp_script:
247            raise error.general('BSP script not found: %s' % (bsp))
248        bsp_config = opts.defaults.expand(opts.defaults[bsp])
249        report_mode = opts.find_arg('--report-mode')
250        if report_mode:
251            if report_mode[1] != 'failures' and \
252                    report_mode[1] != 'all' and \
253                    report_mode[1] != 'none':
254                raise error.general('invalid report mode')
255            report_mode = report_mode[1]
256        else:
257            report_mode = 'failures'
258        executables = find_executables(opts.params(), exe_filter)
259        if len(executables) == 0:
260            raise error.general('no executables supplied')
261        start_time = datetime.datetime.now()
262        total = len(executables)
263        reports = report.report(total)
264        invalid_tests = opts.defaults['invalid_tests']
265        if invalid_tests:
266            reports.set_invalid_tests([l.strip() for l in invalid_tests.splitlines()])
267        reporting = 1
268        jobs = int(opts.jobs(opts.defaults['_ncpus']))
269        exe = 0
270        finished = []
271        if jobs > len(executables):
272            jobs = len(executables)
273        while exe < total or len(tests) > 0:
274            if exe < total and len(tests) < jobs:
275                tst = test_run(exe + 1, total, reports,
276                               executables[exe],
277                               rtems_tools, bsp, bsp_config,
278                               opts)
279                exe += 1
280                tests += [tst]
281                if job_trace:
282                    _job_trace(tst, 'create',
283                               total, exe, tests, reporting)
284                tst.run()
285            else:
286                dead = [t for t in tests if not t.is_alive()]
287                tests[:] = [t for t in tests if t not in dead]
288                for tst in dead:
289                    if job_trace:
290                        _job_trace(tst, 'dead',
291                                   total, exe, tests, reporting)
292                    finished += [tst]
293                    tst.reraise()
294                del dead
295                if len(tests) >= jobs or exe >= total:
296                    time.sleep(0.250)
297                if len(finished):
298                    reporting = report_finished(reports,
299                                                report_mode,
300                                                reporting,
301                                                finished,
302                                                job_trace)
303        finished_time = datetime.datetime.now()
304        reporting = report_finished(reports, report_mode,
305                                    reporting, finished, job_trace)
306        if reporting < total:
307            log.warning('finished jobs does match: %d' % (reporting))
308            report_finished(reports, report_mode, -1, finished, job_trace)
309        reports.summary()
310        end_time = datetime.datetime.now()
311        log.notice('Average test time: %s' % (str((end_time - start_time) / total)))
312        log.notice('Testing time     : %s' % (str(end_time - start_time)))
313    except error.general as gerr:
314        print(gerr)
315        sys.exit(1)
316    except error.internal as ierr:
317        print(ierr)
318        sys.exit(1)
319    except error.exit:
320        sys.exit(2)
321    except KeyboardInterrupt:
322        if opts is not None and opts.find_arg('--stacktrace'):
323            print('}} dumping:', threading.active_count())
324            for t in threading.enumerate():
325                print('}} ', t.name)
326            print(stacktraces.trace())
327        log.notice('abort: user terminated')
328        killall(tests)
329        sys.exit(1)
330    finally:
331        console.restore(stdtty)
332    sys.exit(0)
333
334if __name__ == "__main__":
335    run()
Note: See TracBrowser for help on using the repository browser.