source: rtems-tools/tester/rt/test.py @ f632bd8

5
Last change on this file since f632bd8 was f632bd8, checked in by Chris Johns <chrisj@…>, on 11/25/18 at 22:56:13

tester: Minor formatting change.
  • Property mode set to 100644
File size: 15.9 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2013-2018 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import copy
34import datetime
35import fnmatch
36import os
37import re
38import sys
39import threading
40import time
41
42from rtemstoolkit import configuration
43from rtemstoolkit import error
44from rtemstoolkit import host
45from rtemstoolkit import log
46from rtemstoolkit import path
47from rtemstoolkit import mailer
48from rtemstoolkit import reraise
49from rtemstoolkit import stacktraces
50from rtemstoolkit import version
51from rtemstoolkit import check
52
53import bsps
54import config
55import console
56import options
57import report
58import coverage
59
60class log_capture(object):
61    def __init__(self):
62        self.log = []
63        log.capture = self.capture
64
65    def __str__(self):
66        return os.linesep.join(self.log)
67
68    def capture(self, text):
69        self.log += [l for l in text.replace(chr(13), '').splitlines()]
70
71    def get(self):
72        s = []
73        status = []
74        status_regx = re.compile('^\[\s*\d+/\s*\d+\] p:.+')
75        for l in self.log:
76            if status_regx.match(l):
77                status += [l]
78            else:
79                if len(status) == 1:
80                    s += [status[0]]
81                    status = []
82                elif len(status) > 1:
83                    s += [status[0]]
84                    if len(status) > 2:
85                        s += [' <<skipping passes>>']
86                    s += [status[-1]]
87                    status = []
88                s += [l]
89        return s
90
91class test(object):
92    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
93        self.index = index
94        self.total = total
95        self.report = report
96        self.bsp = bsp
97        self.bsp_config = bsp_config
98        self.opts = copy.copy(opts)
99        self.opts.defaults['test_index'] = str(index)
100        self.opts.defaults['test_total'] = str(total)
101        self.opts.defaults['bsp'] = bsp
102        self.opts.defaults['bsp_arch'] = '%{arch}'
103        if not path.isfile(executable):
104            raise error.general('cannot find executable: %s' % (executable))
105        self.opts.defaults['test_executable'] = executable
106        if rtems_tools:
107            rtems_tools_bin = path.join(self.opts.defaults.expand(rtems_tools), 'bin')
108            if not path.isdir(rtems_tools_bin):
109                raise error.general('cannot find RTEMS tools path: %s' % (rtems_tools_bin))
110            self.opts.defaults['rtems_tools'] = rtems_tools_bin
111        self.config = config.file(index, total, self.report, self.bsp_config, self.opts)
112
113    def run(self):
114        if self.config:
115            self.config.run()
116
117    def kill(self):
118        if self.config:
119            self.config.kill()
120
121class test_run(object):
122    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
123        self.test = None
124        self.result = None
125        self.start_time = None
126        self.end_time = None
127        self.index = copy.copy(index)
128        self.total = total
129        self.report = report
130        self.executable = copy.copy(executable)
131        self.rtems_tools = rtems_tools
132        self.bsp = bsp
133        self.bsp_config = bsp_config
134        self.opts = opts
135
136    def runner(self):
137        self.start_time = datetime.datetime.now()
138        try:
139            self.test = test(self.index, self.total, self.report,
140                             self.executable, self.rtems_tools,
141                             self.bsp, self.bsp_config,
142                             self.opts)
143            self.test.run()
144        except KeyboardInterrupt:
145            pass
146        except:
147            self.result = sys.exc_info()
148        self.end_time = datetime.datetime.now()
149
150    def run(self):
151        name = 'test[%s]' % path.basename(self.executable)
152        self.thread = threading.Thread(target = self.runner,
153                                       name = name)
154        self.thread.start()
155
156    def is_alive(self):
157        return self.thread and self.thread.is_alive()
158
159    def reraise(self):
160        if self.result is not None:
161            reraise.reraise(*self.result)
162
163    def kill(self):
164        if self.test:
165            self.test.kill()
166
167def find_executables(paths, glob):
168    executables = []
169    for p in paths:
170        if path.isfile(p):
171            executables += [p]
172        elif path.isdir(p):
173            for root, dirs, files in os.walk(p, followlinks = True):
174                for f in files:
175                    if fnmatch.fnmatch(f.lower(), glob):
176                        executables += [path.join(root, f)]
177    norun = re.compile('.*\.norun.*')
178    executables = [e for e in executables if not norun.match(e)]
179    return sorted(executables)
180
181def report_finished(reports, report_mode, reporting, finished, job_trace):
182    processing = True
183    while processing:
184        processing = False
185        reported = []
186        for tst in finished:
187            if tst not in reported and \
188                    (reporting < 0 or tst.index == reporting):
189                if job_trace:
190                    log.notice('}} %*d: %s: %s (%d)' % (len(str(tst.total)), tst.index,
191                                                        path.basename(tst.executable),
192                                                        'reporting',
193                                                        reporting))
194                processing = True
195                reports.log(tst.executable, report_mode)
196                reported += [tst]
197                reporting += 1
198        finished[:] = [t for t in finished if t not in reported]
199        if len(reported):
200            del reported[:]
201            if job_trace:
202                print('}} threading:', threading.active_count())
203                for t in threading.enumerate():
204                    print('}} ', t.name)
205    return reporting
206
207def _job_trace(tst, msg, total, exe, active, reporting):
208    s = ''
209    for a in active:
210        s += ' %d:%s' % (a.index, path.basename(a.executable))
211    log.notice('}} %*d: %s: %s (%d %d %d%s)' % (len(str(tst.total)), tst.index,
212                                                path.basename(tst.executable),
213                                                msg,
214                                                reporting, total, exe, s))
215
216def killall(tests):
217    for test in tests:
218        test.kill()
219
220def run(args, command_path = None):
221    import sys
222    tests = []
223    stdtty = console.save()
224    opts = None
225    default_exefilter = '*.exe'
226    try:
227        optargs = { '--rtems-tools':    'The path to the RTEMS tools',
228                    '--rtems-bsp':      'The RTEMS BSP to run the test on',
229                    '--user-config':    'Path to your local user configuration INI file',
230                    '--report-mode':    'Reporting modes, failures (default),all,none',
231                    '--list-bsps':      'List the supported BSPs',
232                    '--debug-trace':    'Debug trace based on specific flags (console,gdb,output,cov)',
233                    '--filter':         'Glob that executables must match to run (default: ' +
234                              default_exefilter + ')',
235                    '--stacktrace':     'Dump a stack trace on a user termination (^C)',
236                    '--coverage':       'Perform coverage analysis of test executables.'}
237        mailer.append_options(optargs)
238        opts = options.load(args,
239                            optargs = optargs,
240                            command_path = command_path)
241        mail = None
242        output = None
243        if opts.find_arg('--mail'):
244            mail = mailer.mail(opts)
245            # Request these now to generate any errors.
246            from_addr = mail.from_address()
247            smtp_host = mail.smtp_host()
248            to_addr = opts.find_arg('--mail-to')
249            if to_addr:
250                to_addr = to_addr[1]
251            else:
252                to_addr = 'build@rtems.org'
253            output = log_capture()
254        log.notice('RTEMS Testing - Tester, %s' % (version.string()))
255        if opts.find_arg('--list-bsps'):
256            bsps.list(opts)
257        exe_filter = opts.find_arg('--filter')
258        if exe_filter:
259            exe_filter = exe_filter[1]
260        else:
261            exe_filter = default_exefilter
262        opts.log_info()
263        log.output('Host: ' + host.label(mode = 'all'))
264        executables = find_executables(opts.params(), exe_filter)
265        debug_trace = opts.find_arg('--debug-trace')
266        if debug_trace:
267            if len(debug_trace) != 1:
268                debug_trace = debug_trace[1]
269            else:
270                raise error.general('no debug flags, can be: console,gdb,output,cov')
271        else:
272            debug_trace = ''
273        opts.defaults['exe_trace'] = debug_trace
274        job_trace = 'jobs' in debug_trace.split(',')
275        rtems_tools = opts.find_arg('--rtems-tools')
276        if rtems_tools:
277            if len(rtems_tools) != 2:
278                raise error.general('invalid RTEMS tools option')
279            rtems_tools = rtems_tools[1]
280        else:
281            rtems_tools = '%{_prefix}'
282        bsp = opts.find_arg('--rtems-bsp')
283        if bsp is None or len(bsp) != 2:
284            raise error.general('RTEMS BSP not provided or an invalid option')
285        bsp = config.load(bsp[1], opts)
286        bsp_config = opts.defaults.expand(opts.defaults['tester'])
287        coverage_enabled = opts.find_arg('--coverage')
288        if coverage_enabled:
289            cov_trace = 'cov' in debug_trace.split(',')
290            if len(coverage_enabled) == 2:
291                coverage_runner = coverage.coverage_run(opts.defaults,
292                                                        executables,
293                                                        symbol_set = coverage_enabled[1],
294                                                        trace = cov_trace)
295            else:
296                coverage_runner = coverage.coverage_run(opts.defaults,
297                                                        executables,
298                                                        trace = cov_trace)
299        report_mode = opts.find_arg('--report-mode')
300        if report_mode:
301            if report_mode[1] != 'failures' and \
302                    report_mode[1] != 'all' and \
303                    report_mode[1] != 'none':
304                raise error.general('invalid report mode')
305            report_mode = report_mode[1]
306        else:
307            report_mode = 'failures'
308        if len(executables) == 0:
309            raise error.general('no executables supplied')
310        start_time = datetime.datetime.now()
311        total = len(executables)
312        reports = report.report(total)
313        reporting = 1
314        jobs = int(opts.jobs(opts.defaults['_ncpus']))
315        exe = 0
316        finished = []
317        if jobs > len(executables):
318            jobs = len(executables)
319        while exe < total or len(tests) > 0:
320            if exe < total and len(tests) < jobs:
321                tst = test_run(exe + 1, total, reports,
322                               executables[exe],
323                               rtems_tools, bsp, bsp_config,
324                               opts)
325                exe += 1
326                tests += [tst]
327                if job_trace:
328                    _job_trace(tst, 'create',
329                               total, exe, tests, reporting)
330                tst.run()
331            else:
332                dead = [t for t in tests if not t.is_alive()]
333                tests[:] = [t for t in tests if t not in dead]
334                for tst in dead:
335                    if job_trace:
336                        _job_trace(tst, 'dead',
337                                   total, exe, tests, reporting)
338                    finished += [tst]
339                    tst.reraise()
340                del dead
341                if len(tests) >= jobs or exe >= total:
342                    time.sleep(0.250)
343                if len(finished):
344                    reporting = report_finished(reports,
345                                                report_mode,
346                                                reporting,
347                                                finished,
348                                                job_trace)
349        finished_time = datetime.datetime.now()
350        reporting = report_finished(reports, report_mode,
351                                    reporting, finished, job_trace)
352        if reporting < total:
353            log.warning('finished jobs does match: %d' % (reporting))
354            report_finished(reports, report_mode, -1, finished, job_trace)
355        reports.summary()
356        end_time = datetime.datetime.now()
357        average_time = 'Average test time: %s' % (str((end_time - start_time) / total))
358        total_time = 'Testing time     : %s' % (str(end_time - start_time))
359        log.notice(average_time)
360        log.notice(total_time)
361        if mail is not None and output is not None:
362            m_arch = opts.defaults.expand('%{arch}')
363            m_bsp = opts.defaults.expand('%{bsp}')
364            build = ' %s:' % (reports.get_config('build', not_found = ''))
365            subject = '[rtems-test] %s/%s:%s %s' % (m_arch,
366                                                    m_bsp,
367                                                    build,
368                                                    reports.score_card('short'))
369            np = 'Not present in test'
370            ver = reports.get_config('version', not_found = np)
371            build = reports.get_config('build', not_found = np)
372            tools = reports.get_config('tools', not_found = np)
373            body = [total_time, average_time,
374                    '', 'Host', '====', host.label(mode = 'all'),
375                    '', 'Configuration', '=============',
376                    'Version: %s' % (ver),
377                    'Build  : %s' % (build),
378                    'Tools  : %s' % (tools),
379                    '', 'Summary', '=======', '',
380                    reports.score_card(), '',
381                    reports.failures(),
382                    'Log', '===', ''] + output.get()
383            mail.send(to_addr, subject, os.linesep.join(body))
384        if coverage_enabled:
385            coverage_runner.run()
386
387    except error.general as gerr:
388        print(gerr)
389        sys.exit(1)
390    except error.internal as ierr:
391        print(ierr)
392        sys.exit(1)
393    except error.exit:
394        sys.exit(2)
395    except KeyboardInterrupt:
396        if opts is not None and opts.find_arg('--stacktrace'):
397            print('}} dumping:', threading.active_count())
398            for t in threading.enumerate():
399                print('}} ', t.name)
400            print(stacktraces.trace())
401        log.notice('abort: user terminated')
402        killall(tests)
403        sys.exit(1)
404    finally:
405        console.restore(stdtty)
406    sys.exit(0)
407
408if __name__ == "__main__":
409    run()
Note: See TracBrowser for help on using the repository browser.