1 | # |
---|
2 | # RTEMS Tools Project (http://www.rtems.org/) |
---|
3 | # Copyright 2013-2014 Chris Johns (chrisj@rtems.org) |
---|
4 | # All rights reserved. |
---|
5 | # |
---|
6 | # This file is part of the RTEMS Tools package in 'rtems-tools'. |
---|
7 | # |
---|
8 | # Redistribution and use in source and binary forms, with or without |
---|
9 | # modification, are permitted provided that the following conditions are met: |
---|
10 | # |
---|
11 | # 1. Redistributions of source code must retain the above copyright notice, |
---|
12 | # this list of conditions and the following disclaimer. |
---|
13 | # |
---|
14 | # 2. Redistributions in binary form must reproduce the above copyright notice, |
---|
15 | # this list of conditions and the following disclaimer in the documentation |
---|
16 | # and/or other materials provided with the distribution. |
---|
17 | # |
---|
18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
---|
19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
---|
20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
---|
21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
---|
22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
---|
23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
---|
24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
---|
25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
---|
26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
---|
27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
---|
28 | # POSSIBILITY OF SUCH DAMAGE. |
---|
29 | # |
---|
30 | |
---|
31 | from __future__ import print_function |
---|
32 | |
---|
33 | import copy |
---|
34 | import datetime |
---|
35 | import fnmatch |
---|
36 | import os |
---|
37 | import re |
---|
38 | import sys |
---|
39 | import threading |
---|
40 | import time |
---|
41 | |
---|
42 | from rtemstoolkit import configuration |
---|
43 | from rtemstoolkit import error |
---|
44 | from rtemstoolkit import log |
---|
45 | from rtemstoolkit import path |
---|
46 | from rtemstoolkit import mailer |
---|
47 | from rtemstoolkit import reraise |
---|
48 | from rtemstoolkit import stacktraces |
---|
49 | from rtemstoolkit import version |
---|
50 | |
---|
51 | from . import bsps |
---|
52 | from . import config |
---|
53 | from . import console |
---|
54 | from . import options |
---|
55 | from . import report |
---|
56 | |
---|
57 | class log_capture(object): |
---|
58 | def __init__(self): |
---|
59 | self.log = [] |
---|
60 | log.capture = self.capture |
---|
61 | |
---|
62 | def __str__(self): |
---|
63 | return os.linesep.join(self.log) |
---|
64 | |
---|
65 | def capture(self, text): |
---|
66 | self.log += [l for l in text.replace(chr(13), '').splitlines()] |
---|
67 | |
---|
68 | def get(self): |
---|
69 | s = [] |
---|
70 | status = [] |
---|
71 | status_regx = re.compile('^\[\s*\d+/\s*\d+\] p:.+') |
---|
72 | for l in self.log: |
---|
73 | if status_regx.match(l): |
---|
74 | status += [l] |
---|
75 | else: |
---|
76 | if len(status) == 1: |
---|
77 | s += [status[0]] |
---|
78 | status = [] |
---|
79 | elif len(status) > 1: |
---|
80 | s += [status[0]] |
---|
81 | if len(status) > 2: |
---|
82 | s += [' <skipped passes>' + os.linesep] |
---|
83 | s += [status[-1]] |
---|
84 | status = [] |
---|
85 | s += [l] |
---|
86 | return s |
---|
87 | |
---|
88 | class test(object): |
---|
89 | def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts): |
---|
90 | self.index = index |
---|
91 | self.total = total |
---|
92 | self.report = report |
---|
93 | self.bsp = bsp |
---|
94 | self.bsp_config = bsp_config |
---|
95 | self.opts = copy.copy(opts) |
---|
96 | self.opts.defaults['test_index'] = str(index) |
---|
97 | self.opts.defaults['test_total'] = str(total) |
---|
98 | self.opts.defaults['bsp'] = bsp |
---|
99 | self.opts.defaults['bsp_arch'] = '%{arch}' |
---|
100 | if not path.isfile(executable): |
---|
101 | raise error.general('cannot find executable: %s' % (executable)) |
---|
102 | self.opts.defaults['test_executable'] = executable |
---|
103 | if rtems_tools: |
---|
104 | rtems_tools_bin = path.join(self.opts.defaults.expand(rtems_tools), 'bin') |
---|
105 | if not path.isdir(rtems_tools_bin): |
---|
106 | raise error.general('cannot find RTEMS tools path: %s' % (rtems_tools_bin)) |
---|
107 | self.opts.defaults['rtems_tools'] = rtems_tools_bin |
---|
108 | self.config = config.file(index, total, self.report, self.bsp_config, self.opts) |
---|
109 | |
---|
110 | def run(self): |
---|
111 | if self.config: |
---|
112 | self.config.run() |
---|
113 | |
---|
114 | def kill(self): |
---|
115 | if self.config: |
---|
116 | self.config.kill() |
---|
117 | |
---|
118 | class test_run(object): |
---|
119 | def __init__(self, index, total, report, executable, rtems_tools, bsp, bsp_config, opts): |
---|
120 | self.test = None |
---|
121 | self.result = None |
---|
122 | self.start_time = None |
---|
123 | self.end_time = None |
---|
124 | self.index = copy.copy(index) |
---|
125 | self.total = total |
---|
126 | self.report = report |
---|
127 | self.executable = copy.copy(executable) |
---|
128 | self.rtems_tools = rtems_tools |
---|
129 | self.bsp = bsp |
---|
130 | self.bsp_config = bsp_config |
---|
131 | self.opts = opts |
---|
132 | |
---|
133 | def runner(self): |
---|
134 | self.start_time = datetime.datetime.now() |
---|
135 | try: |
---|
136 | self.test = test(self.index, self.total, self.report, |
---|
137 | self.executable, self.rtems_tools, |
---|
138 | self.bsp, self.bsp_config, |
---|
139 | self.opts) |
---|
140 | self.test.run() |
---|
141 | except KeyboardInterrupt: |
---|
142 | pass |
---|
143 | except: |
---|
144 | self.result = sys.exc_info() |
---|
145 | self.end_time = datetime.datetime.now() |
---|
146 | |
---|
147 | def run(self): |
---|
148 | self.thread = threading.Thread(target = self.runner, |
---|
149 | name = 'test[%s]' % path.basename(self.executable)) |
---|
150 | self.thread.start() |
---|
151 | |
---|
152 | def is_alive(self): |
---|
153 | return self.thread and self.thread.is_alive() |
---|
154 | |
---|
155 | def reraise(self): |
---|
156 | if self.result is not None: |
---|
157 | reraise.reraise(*self.result) |
---|
158 | |
---|
159 | def kill(self): |
---|
160 | if self.test: |
---|
161 | self.test.kill() |
---|
162 | |
---|
163 | def find_executables(paths, glob): |
---|
164 | executables = [] |
---|
165 | for p in paths: |
---|
166 | if path.isfile(p): |
---|
167 | executables += [p] |
---|
168 | elif path.isdir(p): |
---|
169 | for root, dirs, files in os.walk(p, followlinks = True): |
---|
170 | for f in files: |
---|
171 | if fnmatch.fnmatch(f.lower(), glob): |
---|
172 | executables += [path.join(root, f)] |
---|
173 | return sorted(executables) |
---|
174 | |
---|
175 | def report_finished(reports, report_mode, reporting, finished, job_trace): |
---|
176 | processing = True |
---|
177 | while processing: |
---|
178 | processing = False |
---|
179 | reported = [] |
---|
180 | for tst in finished: |
---|
181 | if tst not in reported and \ |
---|
182 | (reporting < 0 or tst.index == reporting): |
---|
183 | if job_trace: |
---|
184 | log.notice('}} %*d: %s: %s (%d)' % (len(str(tst.total)), tst.index, |
---|
185 | path.basename(tst.executable), |
---|
186 | 'reporting', |
---|
187 | reporting)) |
---|
188 | processing = True |
---|
189 | reports.log(tst.executable, report_mode) |
---|
190 | reported += [tst] |
---|
191 | reporting += 1 |
---|
192 | finished[:] = [t for t in finished if t not in reported] |
---|
193 | if len(reported): |
---|
194 | del reported[:] |
---|
195 | if job_trace: |
---|
196 | print('}} threading:', threading.active_count()) |
---|
197 | for t in threading.enumerate(): |
---|
198 | print('}} ', t.name) |
---|
199 | return reporting |
---|
200 | |
---|
201 | def load_configuration(bsp, opts): |
---|
202 | mandatory = ['bsp', 'arch', 'tester'] |
---|
203 | cfg = configuration.configuration() |
---|
204 | path_ = opts.defaults.expand('%%{_configdir}/bsps/%s.ini' % (bsp)) |
---|
205 | ini_name = path.basename(path_) |
---|
206 | for p in path.dirname(path_).split(':'): |
---|
207 | if path.exists(path.join(p, ini_name)): |
---|
208 | cfg.load(path.join(p, ini_name)) |
---|
209 | if not cfg.has_section(bsp): |
---|
210 | raise error.general('bsp section not found in ini: [%s]' % (bsp)) |
---|
211 | item_names = cfg.get_item_names(bsp, err = False) |
---|
212 | for m in mandatory: |
---|
213 | if m not in item_names: |
---|
214 | raise error.general('mandatory item not found in bsp section: %s' % (m)) |
---|
215 | opts.defaults.set_write_map(bsp, add = True) |
---|
216 | for i in cfg.get_items(bsp, flatten = False): |
---|
217 | opts.defaults[i[0]] = i[1] |
---|
218 | if not opts.defaults.set_read_map(bsp): |
---|
219 | raise error.general('cannot set BSP read map: %s' % (bsp)) |
---|
220 | # Get a copy of the required fields we need |
---|
221 | requires = cfg.comma_list(bsp, 'requires', err = False) |
---|
222 | del cfg |
---|
223 | user_config = opts.find_arg('--user-config') |
---|
224 | if user_config is not None: |
---|
225 | user_config = path.expanduser(user_config[1]) |
---|
226 | if not path.exists(user_config): |
---|
227 | raise error.general('cannot find user configuration file: %s' % (user_config)) |
---|
228 | else: |
---|
229 | if 'HOME' in os.environ: |
---|
230 | user_config = path.join(os.environ['HOME'], '.rtemstesterrc') |
---|
231 | if user_config: |
---|
232 | if path.exists(user_config): |
---|
233 | cfg = configuration.configuration() |
---|
234 | cfg.load(user_config) |
---|
235 | if cfg.has_section(bsp): |
---|
236 | for i in cfg.get_items(bsp, flatten = False): |
---|
237 | opts.defaults[i[0]] = i[1] |
---|
238 | # Check for the required values. |
---|
239 | for r in requires: |
---|
240 | if opts.defaults.get(r) is None: |
---|
241 | raise error.general('user value missing, BSP %s requires: %s' % \ |
---|
242 | (bsp, ', '.join(requires))) |
---|
243 | return opts.defaults['bsp'] |
---|
244 | raise error.general('cannot find bsp configuration file: %s.ini' % (bsp)) |
---|
245 | |
---|
246 | def _job_trace(tst, msg, total, exe, active, reporting): |
---|
247 | s = '' |
---|
248 | for a in active: |
---|
249 | s += ' %d:%s' % (a.index, path.basename(a.executable)) |
---|
250 | log.notice('}} %*d: %s: %s (%d %d %d%s)' % (len(str(tst.total)), tst.index, |
---|
251 | path.basename(tst.executable), |
---|
252 | msg, |
---|
253 | reporting, total, exe, s)) |
---|
254 | |
---|
255 | def list_bsps(opts): |
---|
256 | path_ = opts.defaults.expand('%%{_configdir}/bsps/*.mc') |
---|
257 | bsps = path.collect_files(path_) |
---|
258 | log.notice(' BSP List:') |
---|
259 | for bsp in bsps: |
---|
260 | log.notice(' %s' % (path.basename(bsp[:-3]))) |
---|
261 | raise error.exit() |
---|
262 | |
---|
263 | def killall(tests): |
---|
264 | for test in tests: |
---|
265 | test.kill() |
---|
266 | |
---|
267 | def run(command_path = None): |
---|
268 | import sys |
---|
269 | tests = [] |
---|
270 | stdtty = console.save() |
---|
271 | opts = None |
---|
272 | default_exefilter = '*.exe' |
---|
273 | try: |
---|
274 | optargs = { '--rtems-tools': 'The path to the RTEMS tools', |
---|
275 | '--rtems-bsp': 'The RTEMS BSP to run the test on', |
---|
276 | '--user-config': 'Path to your local user configuration INI file', |
---|
277 | '--report-mode': 'Reporting modes, failures (default),all,none', |
---|
278 | '--list-bsps': 'List the supported BSPs', |
---|
279 | '--debug-trace': 'Debug trace based on specific flags', |
---|
280 | '--filter': 'Glob that executables must match to run (default: ' + |
---|
281 | default_exefilter + ')', |
---|
282 | '--stacktrace': 'Dump a stack trace on a user termination (^C)' } |
---|
283 | mailer.append_options(optargs) |
---|
284 | opts = options.load(sys.argv, |
---|
285 | optargs = optargs, |
---|
286 | command_path = command_path) |
---|
287 | mail = None |
---|
288 | output = None |
---|
289 | if opts.find_arg('--mail'): |
---|
290 | mail = mailer.mail(opts) |
---|
291 | output = log_capture() |
---|
292 | log.notice('RTEMS Testing - Tester, %s' % (version.str())) |
---|
293 | if opts.find_arg('--list-bsps'): |
---|
294 | bsps.list(opts) |
---|
295 | exe_filter = opts.find_arg('--filter') |
---|
296 | if exe_filter: |
---|
297 | exe_filter = exe_filter[1] |
---|
298 | else: |
---|
299 | exe_filter = default_exefilter |
---|
300 | opts.log_info() |
---|
301 | debug_trace = opts.find_arg('--debug-trace') |
---|
302 | if debug_trace: |
---|
303 | if len(debug_trace) != 1: |
---|
304 | debug_trace = debug_trace[1] |
---|
305 | else: |
---|
306 | raise error.general('no debug flags, can be: console,gdb,output') |
---|
307 | else: |
---|
308 | debug_trace = '' |
---|
309 | opts.defaults['debug_trace'] = debug_trace |
---|
310 | job_trace = 'jobs' in debug_trace.split(',') |
---|
311 | rtems_tools = opts.find_arg('--rtems-tools') |
---|
312 | if rtems_tools: |
---|
313 | if len(rtems_tools) != 2: |
---|
314 | raise error.general('invalid RTEMS tools option') |
---|
315 | rtems_tools = rtems_tools[1] |
---|
316 | else: |
---|
317 | rtems_tools = '%{_prefix}' |
---|
318 | bsp = opts.find_arg('--rtems-bsp') |
---|
319 | if bsp is None or len(bsp) != 2: |
---|
320 | raise error.general('RTEMS BSP not provided or an invalid option') |
---|
321 | bsp = load_configuration(bsp[1], opts) |
---|
322 | bsp_config = opts.defaults.expand(opts.defaults['tester']) |
---|
323 | report_mode = opts.find_arg('--report-mode') |
---|
324 | if report_mode: |
---|
325 | if report_mode[1] != 'failures' and \ |
---|
326 | report_mode[1] != 'all' and \ |
---|
327 | report_mode[1] != 'none': |
---|
328 | raise error.general('invalid report mode') |
---|
329 | report_mode = report_mode[1] |
---|
330 | else: |
---|
331 | report_mode = 'failures' |
---|
332 | executables = find_executables(opts.params(), exe_filter) |
---|
333 | if len(executables) == 0: |
---|
334 | raise error.general('no executables supplied') |
---|
335 | start_time = datetime.datetime.now() |
---|
336 | total = len(executables) |
---|
337 | reports = report.report(total) |
---|
338 | reporting = 1 |
---|
339 | jobs = int(opts.jobs(opts.defaults['_ncpus'])) |
---|
340 | exe = 0 |
---|
341 | finished = [] |
---|
342 | if jobs > len(executables): |
---|
343 | jobs = len(executables) |
---|
344 | while exe < total or len(tests) > 0: |
---|
345 | if exe < total and len(tests) < jobs: |
---|
346 | tst = test_run(exe + 1, total, reports, |
---|
347 | executables[exe], |
---|
348 | rtems_tools, bsp, bsp_config, |
---|
349 | opts) |
---|
350 | exe += 1 |
---|
351 | tests += [tst] |
---|
352 | if job_trace: |
---|
353 | _job_trace(tst, 'create', |
---|
354 | total, exe, tests, reporting) |
---|
355 | tst.run() |
---|
356 | else: |
---|
357 | dead = [t for t in tests if not t.is_alive()] |
---|
358 | tests[:] = [t for t in tests if t not in dead] |
---|
359 | for tst in dead: |
---|
360 | if job_trace: |
---|
361 | _job_trace(tst, 'dead', |
---|
362 | total, exe, tests, reporting) |
---|
363 | finished += [tst] |
---|
364 | tst.reraise() |
---|
365 | del dead |
---|
366 | if len(tests) >= jobs or exe >= total: |
---|
367 | time.sleep(0.250) |
---|
368 | if len(finished): |
---|
369 | reporting = report_finished(reports, |
---|
370 | report_mode, |
---|
371 | reporting, |
---|
372 | finished, |
---|
373 | job_trace) |
---|
374 | finished_time = datetime.datetime.now() |
---|
375 | reporting = report_finished(reports, report_mode, |
---|
376 | reporting, finished, job_trace) |
---|
377 | if reporting < total: |
---|
378 | log.warning('finished jobs does match: %d' % (reporting)) |
---|
379 | report_finished(reports, report_mode, -1, finished, job_trace) |
---|
380 | reports.summary() |
---|
381 | end_time = datetime.datetime.now() |
---|
382 | average_time = 'Average test time: %s' % (str((end_time - start_time) / total)) |
---|
383 | total_time = 'Testing time : %s' % (str(end_time - start_time)) |
---|
384 | log.notice(average_time) |
---|
385 | log.notice(total_time) |
---|
386 | if mail is not None and output is not None: |
---|
387 | to_addr = opts.find_arg('--mail-to') |
---|
388 | if to_addr: |
---|
389 | to_addr = to_addr[1] |
---|
390 | else: |
---|
391 | to_addr = 'build@rtems.org' |
---|
392 | subject = '[rtems-test] %s: %s' % (str(start_time).split('.')[0], bsp) |
---|
393 | body = [total_time, average_time, |
---|
394 | '', 'Summary', '=======', '', |
---|
395 | reports.score_card(), '', |
---|
396 | reports.failures(), |
---|
397 | '', 'Log', '===', ''] + output.get() |
---|
398 | mail.send(to_addr, subject, os.linesep.join(body)) |
---|
399 | |
---|
400 | except error.general as gerr: |
---|
401 | print(gerr) |
---|
402 | sys.exit(1) |
---|
403 | except error.internal as ierr: |
---|
404 | print(ierr) |
---|
405 | sys.exit(1) |
---|
406 | except error.exit: |
---|
407 | sys.exit(2) |
---|
408 | except KeyboardInterrupt: |
---|
409 | if opts is not None and opts.find_arg('--stacktrace'): |
---|
410 | print('}} dumping:', threading.active_count()) |
---|
411 | for t in threading.enumerate(): |
---|
412 | print('}} ', t.name) |
---|
413 | print(stacktraces.trace()) |
---|
414 | log.notice('abort: user terminated') |
---|
415 | killall(tests) |
---|
416 | sys.exit(1) |
---|
417 | finally: |
---|
418 | console.restore(stdtty) |
---|
419 | sys.exit(0) |
---|
420 | |
---|
421 | if __name__ == "__main__": |
---|
422 | run() |
---|