source: rtems-tools/tester/rt/test.py @ cec5878

5
Last change on this file since cec5878 was cec5878, checked in by Chris Johns <chrisj@…>, on 10/21/17 at 05:10:22

tester: Remove pass status messages from the email log posts.
  • Property mode set to 100644
File size: 14.4 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2013-2014 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import copy
34import datetime
35import fnmatch
36import os
37import re
38import sys
39import threading
40import time
41
42from rtemstoolkit import error
43from rtemstoolkit import log
44from rtemstoolkit import path
45from rtemstoolkit import mailer
46from rtemstoolkit import reraise
47from rtemstoolkit import stacktraces
48from rtemstoolkit import version
49
50from . import bsps
51from . import config
52from . import console
53from . import options
54from . import report
55
56class log_capture(object):
57    def __init__(self):
58        self.log = []
59        log.capture = self.capture
60
61    def __str__(self):
62        return os.linesep.join(self.log)
63
64    def capture(self, text):
65        self.log += [l for l in text.replace(chr(13), '').splitlines()]
66
67    def get(self):
68        s = []
69        status = []
70        status_regx = re.compile('^\[\s*\d+/\s*\d+\] p:.+')
71        for l in self.log:
72            if status_regx.match(l):
73                status += [l]
74            else:
75                if len(status) == 1:
76                    s += [status[0]]
77                    status = []
78                elif len(status) > 1:
79                    s += [status[0]]
80                    if len(status) > 2:
81                        s += [' <skipped passes>' + os.linesep]
82                    s += [status[-1]]
83                    status = []
84                s += [l]
85        return s
86
87class test(object):
88    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
89        self.index = index
90        self.total = total
91        self.report = report
92        self.bsp = bsp
93        self.bsp_config = bsp_config
94        self.opts = copy.copy(opts)
95        self.opts.defaults['test_index'] = str(index)
96        self.opts.defaults['test_total'] = str(total)
97        self.opts.defaults['bsp'] = bsp
98        self.opts.defaults['bsp_arch'] = '%%{%s_arch}' % (bsp)
99        self.opts.defaults['bsp_opts'] = '%%{%s_opts}' % (bsp)
100        if not path.isfile(executable):
101            raise error.general('cannot find executable: %s' % (executable))
102        self.opts.defaults['test_executable'] = executable
103        if rtems_tools:
104            rtems_tools_bin = path.join(self.opts.defaults.expand(rtems_tools), 'bin')
105            if not path.isdir(rtems_tools_bin):
106                raise error.general('cannot find RTEMS tools path: %s' % (rtems_tools_bin))
107            self.opts.defaults['rtems_tools'] = rtems_tools_bin
108        self.config = config.file(index, total, self.report, self.bsp_config, self.opts)
109
110    def run(self):
111        if self.config:
112            self.config.run()
113
114    def kill(self):
115        if self.config:
116            self.config.kill()
117
118class test_run(object):
119    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
120        self.test = None
121        self.result = None
122        self.start_time = None
123        self.end_time = None
124        self.index = copy.copy(index)
125        self.total = total
126        self.report = report
127        self.executable = copy.copy(executable)
128        self.rtems_tools = rtems_tools
129        self.bsp = bsp
130        self.bsp_config = bsp_config
131        self.opts = opts
132
133    def runner(self):
134        self.start_time = datetime.datetime.now()
135        try:
136            self.test = test(self.index, self.total, self.report,
137                             self.executable, self.rtems_tools,
138                             self.bsp, self.bsp_config,
139                             self.opts)
140            self.test.run()
141        except KeyboardInterrupt:
142            pass
143        except:
144            self.result = sys.exc_info()
145        self.end_time = datetime.datetime.now()
146
147    def run(self):
148        self.thread = threading.Thread(target = self.runner,
149                                       name = 'test[%s]' % path.basename(self.executable))
150        self.thread.start()
151
152    def is_alive(self):
153        return self.thread and self.thread.is_alive()
154
155    def reraise(self):
156        if self.result is not None:
157            reraise.reraise(*self.result)
158
159    def kill(self):
160        if self.test:
161            self.test.kill()
162
163def find_executables(paths, glob):
164    executables = []
165    for p in paths:
166        if path.isfile(p):
167            executables += [p]
168        elif path.isdir(p):
169            for root, dirs, files in os.walk(p, followlinks = True):
170                for f in files:
171                    if fnmatch.fnmatch(f.lower(), glob):
172                        executables += [path.join(root, f)]
173    return sorted(executables)
174
175def report_finished(reports, report_mode, reporting, finished, job_trace):
176    processing = True
177    while processing:
178        processing = False
179        reported = []
180        for tst in finished:
181            if tst not in reported and \
182                    (reporting < 0 or tst.index == reporting):
183                if job_trace:
184                    log.notice('}} %*d: %s: %s (%d)' % (len(str(tst.total)), tst.index,
185                                                        path.basename(tst.executable),
186                                                        'reporting',
187                                                        reporting))
188                processing = True
189                reports.log(tst.executable, report_mode)
190                reported += [tst]
191                reporting += 1
192        finished[:] = [t for t in finished if t not in reported]
193        if len(reported):
194            del reported[:]
195            if job_trace:
196                print('}} threading:', threading.active_count())
197                for t in threading.enumerate():
198                    print('}} ', t.name)
199    return reporting
200
201def _job_trace(tst, msg, total, exe, active, reporting):
202    s = ''
203    for a in active:
204        s += ' %d:%s' % (a.index, path.basename(a.executable))
205    log.notice('}} %*d: %s: %s (%d %d %d%s)' % (len(str(tst.total)), tst.index,
206                                                path.basename(tst.executable),
207                                                msg,
208                                                reporting, total, exe, s))
209
210def list_bsps(opts):
211    path_ = opts.defaults.expand('%%{_configdir}/bsps/*.mc')
212    bsps = path.collect_files(path_)
213    log.notice(' BSP List:')
214    for bsp in bsps:
215        log.notice('  %s' % (path.basename(bsp[:-3])))
216    raise error.exit()
217
218def killall(tests):
219    for test in tests:
220        test.kill()
221
222def run(command_path = None):
223    import sys
224    tests = []
225    stdtty = console.save()
226    opts = None
227    default_exefilter = '*.exe'
228    try:
229        optargs = { '--rtems-tools': 'The path to the RTEMS tools',
230                    '--rtems-bsp':   'The RTEMS BSP to run the test on',
231                    '--report-mode': 'Reporting modes, failures (default),all,none',
232                    '--list-bsps':   'List the supported BSPs',
233                    '--debug-trace': 'Debug trace based on specific flags',
234                    '--filter':      'Glob that executables must match to run (default: ' +
235                              default_exefilter + ')',
236                    '--stacktrace':  'Dump a stack trace on a user termination (^C)' }
237        mailer.append_options(optargs)
238        opts = options.load(sys.argv,
239                            optargs = optargs,
240                            command_path = command_path)
241        mail = None
242        output = None
243        if opts.find_arg('--mail'):
244            mail = mailer.mail(opts)
245            output = log_capture()
246        log.notice('RTEMS Testing - Tester, %s' % (version.str()))
247        if opts.find_arg('--list-bsps'):
248            bsps.list(opts)
249        exe_filter = opts.find_arg('--filter')
250        if exe_filter:
251            exe_filter = exe_filter[1]
252        else:
253            exe_filter = default_exefilter
254        opts.log_info()
255        debug_trace = opts.find_arg('--debug-trace')
256        if debug_trace:
257            if len(debug_trace) != 1:
258                debug_trace = debug_trace[1]
259            else:
260                raise error.general('no debug flags, can be: console,gdb,output')
261        else:
262            debug_trace = ''
263        opts.defaults['debug_trace'] = debug_trace
264        job_trace = 'jobs' in debug_trace.split(',')
265        rtems_tools = opts.find_arg('--rtems-tools')
266        if rtems_tools:
267            if len(rtems_tools) != 2:
268                raise error.general('invalid RTEMS tools option')
269            rtems_tools = rtems_tools[1]
270        else:
271            rtems_tools = '%{_prefix}'
272        bsp = opts.find_arg('--rtems-bsp')
273        if bsp is None or len(bsp) != 2:
274            raise error.general('RTEMS BSP not provided or invalid option')
275        opts.defaults.load('%%{_configdir}/bsps/%s.mc' % (bsp[1]))
276        bsp = opts.defaults.get('%{bsp}')
277        if not bsp:
278            raise error.general('BSP definition (%{bsp}) not found in the global map')
279        bsp = bsp[2]
280        if not opts.defaults.set_read_map(bsp):
281            raise error.general('no BSP map found')
282        bsp_script = opts.defaults.get(bsp)
283        if not bsp_script:
284            raise error.general('BSP script not found: %s' % (bsp))
285        bsp_config = opts.defaults.expand(opts.defaults[bsp])
286        report_mode = opts.find_arg('--report-mode')
287        if report_mode:
288            if report_mode[1] != 'failures' and \
289                    report_mode[1] != 'all' and \
290                    report_mode[1] != 'none':
291                raise error.general('invalid report mode')
292            report_mode = report_mode[1]
293        else:
294            report_mode = 'failures'
295        executables = find_executables(opts.params(), exe_filter)
296        if len(executables) == 0:
297            raise error.general('no executables supplied')
298        start_time = datetime.datetime.now()
299        total = len(executables)
300        reports = report.report(total)
301        reporting = 1
302        jobs = int(opts.jobs(opts.defaults['_ncpus']))
303        exe = 0
304        finished = []
305        if jobs > len(executables):
306            jobs = len(executables)
307        while exe < total or len(tests) > 0:
308            if exe < total and len(tests) < jobs:
309                tst = test_run(exe + 1, total, reports,
310                               executables[exe],
311                               rtems_tools, bsp, bsp_config,
312                               opts)
313                exe += 1
314                tests += [tst]
315                if job_trace:
316                    _job_trace(tst, 'create',
317                               total, exe, tests, reporting)
318                tst.run()
319            else:
320                dead = [t for t in tests if not t.is_alive()]
321                tests[:] = [t for t in tests if t not in dead]
322                for tst in dead:
323                    if job_trace:
324                        _job_trace(tst, 'dead',
325                                   total, exe, tests, reporting)
326                    finished += [tst]
327                    tst.reraise()
328                del dead
329                if len(tests) >= jobs or exe >= total:
330                    time.sleep(0.250)
331                if len(finished):
332                    reporting = report_finished(reports,
333                                                report_mode,
334                                                reporting,
335                                                finished,
336                                                job_trace)
337        finished_time = datetime.datetime.now()
338        reporting = report_finished(reports, report_mode,
339                                    reporting, finished, job_trace)
340        if reporting < total:
341            log.warning('finished jobs does match: %d' % (reporting))
342            report_finished(reports, report_mode, -1, finished, job_trace)
343        reports.summary()
344        end_time = datetime.datetime.now()
345        average_time = 'Average test time: %s' % (str((end_time - start_time) / total))
346        total_time = 'Testing time     : %s' % (str(end_time - start_time))
347        log.notice(average_time)
348        log.notice(total_time)
349        if mail is not None and output is not None:
350            to_addr = opts.find_arg('--mail-to')
351            if to_addr:
352                to_addr = to_addr[1]
353            else:
354                to_addr = 'build@rtems.org'
355            subject = '[rtems-test] %s: %s' % (str(start_time).split('.')[0], bsp)
356            body = [total_time, average_time,
357                    '', 'Summary', '=======', '',
358                    reports.score_card(), '',
359                    reports.failures(),
360                    '', 'Log', '===', ''] + output.get()
361            mail.send(to_addr, subject, os.linesep.join(body))
362
363    except error.general as gerr:
364        print(gerr)
365        sys.exit(1)
366    except error.internal as ierr:
367        print(ierr)
368        sys.exit(1)
369    except error.exit:
370        sys.exit(2)
371    except KeyboardInterrupt:
372        if opts is not None and opts.find_arg('--stacktrace'):
373            print('}} dumping:', threading.active_count())
374            for t in threading.enumerate():
375                print('}} ', t.name)
376            print(stacktraces.trace())
377        log.notice('abort: user terminated')
378        killall(tests)
379        sys.exit(1)
380    finally:
381        console.restore(stdtty)
382    sys.exit(0)
383
384if __name__ == "__main__":
385    run()
Note: See TracBrowser for help on using the repository browser.