source: rtems-tools/tester/rt/test.py @ 41d0c34

5
Last change on this file since 41d0c34 was 41d0c34, checked in by Chris Johns <chrisj@…>, on 10/24/17 at 22:03:43

tester: Add a BSP field to each BSP INI configuration section.

The INI section in a BSP configuration is the name of the INI file so
each BSP configuration section needs a BSP.

Updates #3204.

  • Property mode set to 100644
File size: 16.4 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2013-2014 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import copy
34import datetime
35import fnmatch
36import os
37import re
38import sys
39import threading
40import time
41
42from rtemstoolkit import configuration
43from rtemstoolkit import error
44from rtemstoolkit import log
45from rtemstoolkit import path
46from rtemstoolkit import mailer
47from rtemstoolkit import reraise
48from rtemstoolkit import stacktraces
49from rtemstoolkit import version
50
51from . import bsps
52from . import config
53from . import console
54from . import options
55from . import report
56
57class log_capture(object):
58    def __init__(self):
59        self.log = []
60        log.capture = self.capture
61
62    def __str__(self):
63        return os.linesep.join(self.log)
64
65    def capture(self, text):
66        self.log += [l for l in text.replace(chr(13), '').splitlines()]
67
68    def get(self):
69        s = []
70        status = []
71        status_regx = re.compile('^\[\s*\d+/\s*\d+\] p:.+')
72        for l in self.log:
73            if status_regx.match(l):
74                status += [l]
75            else:
76                if len(status) == 1:
77                    s += [status[0]]
78                    status = []
79                elif len(status) > 1:
80                    s += [status[0]]
81                    if len(status) > 2:
82                        s += [' <skipped passes>' + os.linesep]
83                    s += [status[-1]]
84                    status = []
85                s += [l]
86        return s
87
88class test(object):
89    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
90        self.index = index
91        self.total = total
92        self.report = report
93        self.bsp = bsp
94        self.bsp_config = bsp_config
95        self.opts = copy.copy(opts)
96        self.opts.defaults['test_index'] = str(index)
97        self.opts.defaults['test_total'] = str(total)
98        self.opts.defaults['bsp'] = bsp
99        self.opts.defaults['bsp_arch'] = '%{arch}'
100        if not path.isfile(executable):
101            raise error.general('cannot find executable: %s' % (executable))
102        self.opts.defaults['test_executable'] = executable
103        if rtems_tools:
104            rtems_tools_bin = path.join(self.opts.defaults.expand(rtems_tools), 'bin')
105            if not path.isdir(rtems_tools_bin):
106                raise error.general('cannot find RTEMS tools path: %s' % (rtems_tools_bin))
107            self.opts.defaults['rtems_tools'] = rtems_tools_bin
108        self.config = config.file(index, total, self.report, self.bsp_config, self.opts)
109
110    def run(self):
111        if self.config:
112            self.config.run()
113
114    def kill(self):
115        if self.config:
116            self.config.kill()
117
118class test_run(object):
119    def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts):
120        self.test = None
121        self.result = None
122        self.start_time = None
123        self.end_time = None
124        self.index = copy.copy(index)
125        self.total = total
126        self.report = report
127        self.executable = copy.copy(executable)
128        self.rtems_tools = rtems_tools
129        self.bsp = bsp
130        self.bsp_config = bsp_config
131        self.opts = opts
132
133    def runner(self):
134        self.start_time = datetime.datetime.now()
135        try:
136            self.test = test(self.index, self.total, self.report,
137                             self.executable, self.rtems_tools,
138                             self.bsp, self.bsp_config,
139                             self.opts)
140            self.test.run()
141        except KeyboardInterrupt:
142            pass
143        except:
144            self.result = sys.exc_info()
145        self.end_time = datetime.datetime.now()
146
147    def run(self):
148        self.thread = threading.Thread(target = self.runner,
149                                       name = 'test[%s]' % path.basename(self.executable))
150        self.thread.start()
151
152    def is_alive(self):
153        return self.thread and self.thread.is_alive()
154
155    def reraise(self):
156        if self.result is not None:
157            reraise.reraise(*self.result)
158
159    def kill(self):
160        if self.test:
161            self.test.kill()
162
163def find_executables(paths, glob):
164    executables = []
165    for p in paths:
166        if path.isfile(p):
167            executables += [p]
168        elif path.isdir(p):
169            for root, dirs, files in os.walk(p, followlinks = True):
170                for f in files:
171                    if fnmatch.fnmatch(f.lower(), glob):
172                        executables += [path.join(root, f)]
173    return sorted(executables)
174
175def report_finished(reports, report_mode, reporting, finished, job_trace):
176    processing = True
177    while processing:
178        processing = False
179        reported = []
180        for tst in finished:
181            if tst not in reported and \
182                    (reporting < 0 or tst.index == reporting):
183                if job_trace:
184                    log.notice('}} %*d: %s: %s (%d)' % (len(str(tst.total)), tst.index,
185                                                        path.basename(tst.executable),
186                                                        'reporting',
187                                                        reporting))
188                processing = True
189                reports.log(tst.executable, report_mode)
190                reported += [tst]
191                reporting += 1
192        finished[:] = [t for t in finished if t not in reported]
193        if len(reported):
194            del reported[:]
195            if job_trace:
196                print('}} threading:', threading.active_count())
197                for t in threading.enumerate():
198                    print('}} ', t.name)
199    return reporting
200
201def load_configuration(bsp, opts):
202    mandatory = ['bsp', 'arch', 'tester']
203    cfg = configuration.configuration()
204    path_ = opts.defaults.expand('%%{_configdir}/bsps/%s.ini' % (bsp))
205    ini_name = path.basename(path_)
206    for p in path.dirname(path_).split(':'):
207        if path.exists(path.join(p, ini_name)):
208            cfg.load(path.join(p, ini_name))
209            if not cfg.has_section(bsp):
210                raise error.general('bsp section not found in ini: [%s]' % (bsp))
211            item_names = cfg.get_item_names(bsp, err = False)
212            for m in mandatory:
213                if m not in item_names:
214                    raise error.general('mandatory item not found in bsp section: %s' % (m))
215            opts.defaults.set_write_map(bsp, add = True)
216            for i in cfg.get_items(bsp, flatten = False):
217                opts.defaults[i[0]] = i[1]
218            if not opts.defaults.set_read_map(bsp):
219                raise error.general('cannot set BSP read map: %s' % (bsp))
220            # Get a copy of the required fields we need
221            requires = cfg.comma_list(bsp, 'requires', err = False)
222            del cfg
223            user_config = opts.find_arg('--user-config')
224            if user_config is not None:
225                user_config = path.expanduser(user_config[1])
226                if not path.exists(user_config):
227                    raise error.general('cannot find user configuration file: %s' % (user_config))
228            else:
229                if 'HOME' in os.environ:
230                    user_config = path.join(os.environ['HOME'], '.rtemstesterrc')
231            if user_config:
232                if path.exists(user_config):
233                    cfg = configuration.configuration()
234                    cfg.load(user_config)
235                    if cfg.has_section(bsp):
236                        for i in cfg.get_items(bsp, flatten = False):
237                            opts.defaults[i[0]] = i[1]
238                        # Check for the required values.
239                        for r in requires:
240                            if opts.defaults.get(r) is None:
241                                raise error.general('user value missing, BSP %s requires: %s' % \
242                                                    (bsp, ', '.join(requires)))
243            return opts.defaults['bsp']
244    raise error.general('cannot find bsp configuration file: %s.ini' % (bsp))
245
246def _job_trace(tst, msg, total, exe, active, reporting):
247    s = ''
248    for a in active:
249        s += ' %d:%s' % (a.index, path.basename(a.executable))
250    log.notice('}} %*d: %s: %s (%d %d %d%s)' % (len(str(tst.total)), tst.index,
251                                                path.basename(tst.executable),
252                                                msg,
253                                                reporting, total, exe, s))
254
255def list_bsps(opts):
256    path_ = opts.defaults.expand('%%{_configdir}/bsps/*.mc')
257    bsps = path.collect_files(path_)
258    log.notice(' BSP List:')
259    for bsp in bsps:
260        log.notice('  %s' % (path.basename(bsp[:-3])))
261    raise error.exit()
262
263def killall(tests):
264    for test in tests:
265        test.kill()
266
267def run(command_path = None):
268    import sys
269    tests = []
270    stdtty = console.save()
271    opts = None
272    default_exefilter = '*.exe'
273    try:
274        optargs = { '--rtems-tools': 'The path to the RTEMS tools',
275                    '--rtems-bsp':   'The RTEMS BSP to run the test on',
276                    '--user-config': 'Path to your local user configuration INI file',
277                    '--report-mode': 'Reporting modes, failures (default),all,none',
278                    '--list-bsps':   'List the supported BSPs',
279                    '--debug-trace': 'Debug trace based on specific flags',
280                    '--filter':      'Glob that executables must match to run (default: ' +
281                              default_exefilter + ')',
282                    '--stacktrace':  'Dump a stack trace on a user termination (^C)' }
283        mailer.append_options(optargs)
284        opts = options.load(sys.argv,
285                            optargs = optargs,
286                            command_path = command_path)
287        mail = None
288        output = None
289        if opts.find_arg('--mail'):
290            mail = mailer.mail(opts)
291            output = log_capture()
292        log.notice('RTEMS Testing - Tester, %s' % (version.str()))
293        if opts.find_arg('--list-bsps'):
294            bsps.list(opts)
295        exe_filter = opts.find_arg('--filter')
296        if exe_filter:
297            exe_filter = exe_filter[1]
298        else:
299            exe_filter = default_exefilter
300        opts.log_info()
301        debug_trace = opts.find_arg('--debug-trace')
302        if debug_trace:
303            if len(debug_trace) != 1:
304                debug_trace = debug_trace[1]
305            else:
306                raise error.general('no debug flags, can be: console,gdb,output')
307        else:
308            debug_trace = ''
309        opts.defaults['debug_trace'] = debug_trace
310        job_trace = 'jobs' in debug_trace.split(',')
311        rtems_tools = opts.find_arg('--rtems-tools')
312        if rtems_tools:
313            if len(rtems_tools) != 2:
314                raise error.general('invalid RTEMS tools option')
315            rtems_tools = rtems_tools[1]
316        else:
317            rtems_tools = '%{_prefix}'
318        bsp = opts.find_arg('--rtems-bsp')
319        if bsp is None or len(bsp) != 2:
320            raise error.general('RTEMS BSP not provided or an invalid option')
321        bsp = load_configuration(bsp[1], opts)
322        bsp_config = opts.defaults.expand(opts.defaults['tester'])
323        report_mode = opts.find_arg('--report-mode')
324        if report_mode:
325            if report_mode[1] != 'failures' and \
326                    report_mode[1] != 'all' and \
327                    report_mode[1] != 'none':
328                raise error.general('invalid report mode')
329            report_mode = report_mode[1]
330        else:
331            report_mode = 'failures'
332        executables = find_executables(opts.params(), exe_filter)
333        if len(executables) == 0:
334            raise error.general('no executables supplied')
335        start_time = datetime.datetime.now()
336        total = len(executables)
337        reports = report.report(total)
338        reporting = 1
339        jobs = int(opts.jobs(opts.defaults['_ncpus']))
340        exe = 0
341        finished = []
342        if jobs > len(executables):
343            jobs = len(executables)
344        while exe < total or len(tests) > 0:
345            if exe < total and len(tests) < jobs:
346                tst = test_run(exe + 1, total, reports,
347                               executables[exe],
348                               rtems_tools, bsp, bsp_config,
349                               opts)
350                exe += 1
351                tests += [tst]
352                if job_trace:
353                    _job_trace(tst, 'create',
354                               total, exe, tests, reporting)
355                tst.run()
356            else:
357                dead = [t for t in tests if not t.is_alive()]
358                tests[:] = [t for t in tests if t not in dead]
359                for tst in dead:
360                    if job_trace:
361                        _job_trace(tst, 'dead',
362                                   total, exe, tests, reporting)
363                    finished += [tst]
364                    tst.reraise()
365                del dead
366                if len(tests) >= jobs or exe >= total:
367                    time.sleep(0.250)
368                if len(finished):
369                    reporting = report_finished(reports,
370                                                report_mode,
371                                                reporting,
372                                                finished,
373                                                job_trace)
374        finished_time = datetime.datetime.now()
375        reporting = report_finished(reports, report_mode,
376                                    reporting, finished, job_trace)
377        if reporting < total:
378            log.warning('finished jobs does match: %d' % (reporting))
379            report_finished(reports, report_mode, -1, finished, job_trace)
380        reports.summary()
381        end_time = datetime.datetime.now()
382        average_time = 'Average test time: %s' % (str((end_time - start_time) / total))
383        total_time = 'Testing time     : %s' % (str(end_time - start_time))
384        log.notice(average_time)
385        log.notice(total_time)
386        if mail is not None and output is not None:
387            to_addr = opts.find_arg('--mail-to')
388            if to_addr:
389                to_addr = to_addr[1]
390            else:
391                to_addr = 'build@rtems.org'
392            subject = '[rtems-test] %s: %s' % (str(start_time).split('.')[0], bsp)
393            body = [total_time, average_time,
394                    '', 'Summary', '=======', '',
395                    reports.score_card(), '',
396                    reports.failures(),
397                    '', 'Log', '===', ''] + output.get()
398            mail.send(to_addr, subject, os.linesep.join(body))
399
400    except error.general as gerr:
401        print(gerr)
402        sys.exit(1)
403    except error.internal as ierr:
404        print(ierr)
405        sys.exit(1)
406    except error.exit:
407        sys.exit(2)
408    except KeyboardInterrupt:
409        if opts is not None and opts.find_arg('--stacktrace'):
410            print('}} dumping:', threading.active_count())
411            for t in threading.enumerate():
412                print('}} ', t.name)
413            print(stacktraces.trace())
414        log.notice('abort: user terminated')
415        killall(tests)
416        sys.exit(1)
417    finally:
418        console.restore(stdtty)
419    sys.exit(0)
420
421if __name__ == "__main__":
422    run()
Note: See TracBrowser for help on using the repository browser.