source: rtems-tools/tester/rt/test.py @ 82c8788

5
Last change on this file since 82c8788 was b7c4753, checked in by Chris Johns <chrisj@…>, on 04/10/18 at 04:00:09

teater: Filter out an *.norun.* executables.
  • Property mode set to 100644
File size: 14.9 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2013-2018 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import copy
34import datetime
35import fnmatch
36import os
37import re
38import sys
39import threading
40import time
41
42from rtemstoolkit import configuration
43from rtemstoolkit import error
44from rtemstoolkit import host
45from rtemstoolkit import log
46from rtemstoolkit import path
47from rtemstoolkit import mailer
48from rtemstoolkit import reraise
49from rtemstoolkit import stacktraces
50from rtemstoolkit import version
51
52from . import bsps
53from . import config
54from . import console
55from . import options
56from . import report
57
58class log_capture(object):
59    def __init__(self):
60        self.log = []
61        log.capture = self.capture
62
63    def __str__(self):
64        return os.linesep.join(self.log)
65
66    def capture(self, text):
67        self.log += [l for l in text.replace(chr(13), '').splitlines()]
68
69    def get(self):
70        s = []
71        status = []
72        status_regx = re.compile('^\[\s*\d+/\s*\d+\] p:.+')
73        for l in self.log:
74            if status_regx.match(l):
75                status += [l]
76            else:
77                if len(status) == 1:
78                    s += [status[0]]
79                    status = []
80                elif len(status) > 1:
81                    s += [status[0]]
82                    if len(status) > 2:
83                        s += [' <<skipping passes>>']
84                    s += [status[-1]]
85                    status = []
86                s += [l]
87        return s
88
89class test(object):
90    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
91        self.index = index
92        self.total = total
93        self.report = report
94        self.bsp = bsp
95        self.bsp_config = bsp_config
96        self.opts = copy.copy(opts)
97        self.opts.defaults['test_index'] = str(index)
98        self.opts.defaults['test_total'] = str(total)
99        self.opts.defaults['bsp'] = bsp
100        self.opts.defaults['bsp_arch'] = '%{arch}'
101        if not path.isfile(executable):
102            raise error.general('cannot find executable: %s' % (executable))
103        self.opts.defaults['test_executable'] = executable
104        if rtems_tools:
105            rtems_tools_bin = path.join(self.opts.defaults.expand(rtems_tools), 'bin')
106            if not path.isdir(rtems_tools_bin):
107                raise error.general('cannot find RTEMS tools path: %s' % (rtems_tools_bin))
108            self.opts.defaults['rtems_tools'] = rtems_tools_bin
109        self.config = config.file(index, total, self.report, self.bsp_config, self.opts)
110
111    def run(self):
112        if self.config:
113            self.config.run()
114
115    def kill(self):
116        if self.config:
117            self.config.kill()
118
119class test_run(object):
120    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
121        self.test = None
122        self.result = None
123        self.start_time = None
124        self.end_time = None
125        self.index = copy.copy(index)
126        self.total = total
127        self.report = report
128        self.executable = copy.copy(executable)
129        self.rtems_tools = rtems_tools
130        self.bsp = bsp
131        self.bsp_config = bsp_config
132        self.opts = opts
133
134    def runner(self):
135        self.start_time = datetime.datetime.now()
136        try:
137            self.test = test(self.index, self.total, self.report,
138                             self.executable, self.rtems_tools,
139                             self.bsp, self.bsp_config,
140                             self.opts)
141            self.test.run()
142        except KeyboardInterrupt:
143            pass
144        except:
145            self.result = sys.exc_info()
146        self.end_time = datetime.datetime.now()
147
148    def run(self):
149        self.thread = threading.Thread(target = self.runner,
150                                       name = 'test[%s]' % path.basename(self.executable))
151        self.thread.start()
152
153    def is_alive(self):
154        return self.thread and self.thread.is_alive()
155
156    def reraise(self):
157        if self.result is not None:
158            reraise.reraise(*self.result)
159
160    def kill(self):
161        if self.test:
162            self.test.kill()
163
164def find_executables(paths, glob):
165    executables = []
166    for p in paths:
167        if path.isfile(p):
168            executables += [p]
169        elif path.isdir(p):
170            for root, dirs, files in os.walk(p, followlinks = True):
171                for f in files:
172                    if fnmatch.fnmatch(f.lower(), glob):
173                        executables += [path.join(root, f)]
174    norun = re.compile('.*\.norun.*')
175    executables = [e for e in executables if not norun.match(e)]
176    return sorted(executables)
177
178def report_finished(reports, report_mode, reporting, finished, job_trace):
179    processing = True
180    while processing:
181        processing = False
182        reported = []
183        for tst in finished:
184            if tst not in reported and \
185                    (reporting < 0 or tst.index == reporting):
186                if job_trace:
187                    log.notice('}} %*d: %s: %s (%d)' % (len(str(tst.total)), tst.index,
188                                                        path.basename(tst.executable),
189                                                        'reporting',
190                                                        reporting))
191                processing = True
192                reports.log(tst.executable, report_mode)
193                reported += [tst]
194                reporting += 1
195        finished[:] = [t for t in finished if t not in reported]
196        if len(reported):
197            del reported[:]
198            if job_trace:
199                print('}} threading:', threading.active_count())
200                for t in threading.enumerate():
201                    print('}} ', t.name)
202    return reporting
203
204def _job_trace(tst, msg, total, exe, active, reporting):
205    s = ''
206    for a in active:
207        s += ' %d:%s' % (a.index, path.basename(a.executable))
208    log.notice('}} %*d: %s: %s (%d %d %d%s)' % (len(str(tst.total)), tst.index,
209                                                path.basename(tst.executable),
210                                                msg,
211                                                reporting, total, exe, s))
212
213def killall(tests):
214    for test in tests:
215        test.kill()
216
217def run(command_path = None):
218    import sys
219    tests = []
220    stdtty = console.save()
221    opts = None
222    default_exefilter = '*.exe'
223    try:
224        optargs = { '--rtems-tools': 'The path to the RTEMS tools',
225                    '--rtems-bsp':   'The RTEMS BSP to run the test on',
226                    '--user-config': 'Path to your local user configuration INI file',
227                    '--report-mode': 'Reporting modes, failures (default),all,none',
228                    '--list-bsps':   'List the supported BSPs',
229                    '--debug-trace': 'Debug trace based on specific flags',
230                    '--filter':      'Glob that executables must match to run (default: ' +
231                              default_exefilter + ')',
232                    '--stacktrace':  'Dump a stack trace on a user termination (^C)' }
233        mailer.append_options(optargs)
234        opts = options.load(sys.argv,
235                            optargs = optargs,
236                            command_path = command_path)
237        mail = None
238        output = None
239        if opts.find_arg('--mail'):
240            mail = mailer.mail(opts)
241            # Request these now to generate any errors.
242            from_addr = mail.from_address()
243            smtp_host = mail.smtp_host()
244            to_addr = opts.find_arg('--mail-to')
245            if to_addr:
246                to_addr = to_addr[1]
247            else:
248                to_addr = 'build@rtems.org'
249            output = log_capture()
250        log.notice('RTEMS Testing - Tester, %s' % (version.str()))
251        if opts.find_arg('--list-bsps'):
252            bsps.list(opts)
253        exe_filter = opts.find_arg('--filter')
254        if exe_filter:
255            exe_filter = exe_filter[1]
256        else:
257            exe_filter = default_exefilter
258        opts.log_info()
259        log.output('Host: ' + host.label(mode = 'all'))
260        debug_trace = opts.find_arg('--debug-trace')
261        if debug_trace:
262            if len(debug_trace) != 1:
263                debug_trace = debug_trace[1]
264            else:
265                raise error.general('no debug flags, can be: console,gdb,output')
266        else:
267            debug_trace = ''
268        opts.defaults['exe_trace'] = debug_trace
269        job_trace = 'jobs' in debug_trace.split(',')
270        rtems_tools = opts.find_arg('--rtems-tools')
271        if rtems_tools:
272            if len(rtems_tools) != 2:
273                raise error.general('invalid RTEMS tools option')
274            rtems_tools = rtems_tools[1]
275        else:
276            rtems_tools = '%{_prefix}'
277        bsp = opts.find_arg('--rtems-bsp')
278        if bsp is None or len(bsp) != 2:
279            raise error.general('RTEMS BSP not provided or an invalid option')
280        bsp = config.load(bsp[1], opts)
281        bsp_config = opts.defaults.expand(opts.defaults['tester'])
282        report_mode = opts.find_arg('--report-mode')
283        if report_mode:
284            if report_mode[1] != 'failures' and \
285                    report_mode[1] != 'all' and \
286                    report_mode[1] != 'none':
287                raise error.general('invalid report mode')
288            report_mode = report_mode[1]
289        else:
290            report_mode = 'failures'
291        executables = find_executables(opts.params(), exe_filter)
292        if len(executables) == 0:
293            raise error.general('no executables supplied')
294        start_time = datetime.datetime.now()
295        total = len(executables)
296        reports = report.report(total)
297        reporting = 1
298        jobs = int(opts.jobs(opts.defaults['_ncpus']))
299        exe = 0
300        finished = []
301        if jobs > len(executables):
302            jobs = len(executables)
303        while exe < total or len(tests) > 0:
304            if exe < total and len(tests) < jobs:
305                tst = test_run(exe + 1, total, reports,
306                               executables[exe],
307                               rtems_tools, bsp, bsp_config,
308                               opts)
309                exe += 1
310                tests += [tst]
311                if job_trace:
312                    _job_trace(tst, 'create',
313                               total, exe, tests, reporting)
314                tst.run()
315            else:
316                dead = [t for t in tests if not t.is_alive()]
317                tests[:] = [t for t in tests if t not in dead]
318                for tst in dead:
319                    if job_trace:
320                        _job_trace(tst, 'dead',
321                                   total, exe, tests, reporting)
322                    finished += [tst]
323                    tst.reraise()
324                del dead
325                if len(tests) >= jobs or exe >= total:
326                    time.sleep(0.250)
327                if len(finished):
328                    reporting = report_finished(reports,
329                                                report_mode,
330                                                reporting,
331                                                finished,
332                                                job_trace)
333        finished_time = datetime.datetime.now()
334        reporting = report_finished(reports, report_mode,
335                                    reporting, finished, job_trace)
336        if reporting < total:
337            log.warning('finished jobs does match: %d' % (reporting))
338            report_finished(reports, report_mode, -1, finished, job_trace)
339        reports.summary()
340        end_time = datetime.datetime.now()
341        average_time = 'Average test time: %s' % (str((end_time - start_time) / total))
342        total_time = 'Testing time     : %s' % (str(end_time - start_time))
343        log.notice(average_time)
344        log.notice(total_time)
345        if mail is not None and output is not None:
346            m_arch = opts.defaults.expand('%{arch}')
347            m_bsp = opts.defaults.expand('%{bsp}')
348            build = ' %s:' % (reports.get_config('build', not_found = ''))
349            subject = '[rtems-test] %s/%s:%s %s' % (m_arch,
350                                                    m_bsp,
351                                                    build,
352                                                    reports.score_card('short'))
353            np = 'Not present in test'
354            ver = reports.get_config('version', not_found = np)
355            build = reports.get_config('build', not_found = np)
356            tools = reports.get_config('tools', not_found = np)
357            body = [total_time, average_time,
358                    '', 'Host', '====', host.label(mode = 'all'),
359                    '', 'Configuration', '=============',
360                    'Version: %s' % (ver),
361                    'Build  : %s' % (build),
362                    'Tools  : %s' % (tools),
363                    '', 'Summary', '=======', '',
364                    reports.score_card(), '',
365                    reports.failures(),
366                    'Log', '===', ''] + output.get()
367            mail.send(to_addr, subject, os.linesep.join(body))
368
369    except error.general as gerr:
370        print(gerr)
371        sys.exit(1)
372    except error.internal as ierr:
373        print(ierr)
374        sys.exit(1)
375    except error.exit:
376        sys.exit(2)
377    except KeyboardInterrupt:
378        if opts is not None and opts.find_arg('--stacktrace'):
379            print('}} dumping:', threading.active_count())
380            for t in threading.enumerate():
381                print('}} ', t.name)
382            print(stacktraces.trace())
383        log.notice('abort: user terminated')
384        killall(tests)
385        sys.exit(1)
386    finally:
387        console.restore(stdtty)
388    sys.exit(0)
389
390if __name__ == "__main__":
391    run()
Note: See TracBrowser for help on using the repository browser.