Context Navigation

source: rtems-tools/tester/rt/check.py @ 096c95e

5

Last change on this file since 096c95e was 096c95e, checked in by Chris Johns <chrisj@…>, on 04/27/17 at 20:36:32
rtems-bsp-tester: Make the failure and pass report more compact.
Property mode set to `100755`
File size: 46.8 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2016-2017 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Redistribution and use in source and binary forms, with or without
9	# modification, are permitted provided that the following conditions are met:
10	#
11	# 1. Redistributions of source code must retain the above copyright notice,
12	# this list of conditions and the following disclaimer.
13	#
14	# 2. Redistributions in binary form must reproduce the above copyright notice,
15	# this list of conditions and the following disclaimer in the documentation
16	# and/or other materials provided with the distribution.
17	#
18	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21	# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22	# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23	# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24	# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25	# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26	# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27	# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28	# POSSIBILITY OF SUCH DAMAGE.
29	#
30
31	from __future__ import print_function
32
33	import argparse
34	import datetime
35	import operator
36	import os
37	import re
38	import sys
39	import textwrap
40
41	import pprint
42
43	try:
44	import configparser
45	except:
46	import ConfigParser as configparser
47
48	from rtemstoolkit import execute
49	from rtemstoolkit import error
50	from rtemstoolkit import host
51	from rtemstoolkit import log
52	from rtemstoolkit import path
53	from rtemstoolkit import textbox
54	from rtemstoolkit import version
55
56	def rtems_version():
57	return version.version()
58
59	def wrap(line, lineend = '', indent = 0, width = 75):
60	if type(line) is tuple or type(line) is list:
61	if len(line) >= 2:
62	s1 = line[0]
63	else:
64	s1 = ''
65	s2 = line[1:]
66	elif type(line) is str:
67	s1 = ''
68	s2 = [line]
69	else:
70	raise error.internal('line is not a tuple, list or string')
71	s1len = len(s1)
72	s = ''
73	first = True
74	for ss in s2:
75	if type(ss) is not str and type(ss) is not unicode:
76	raise error.internal('text needs to be a string')
77	for l in textwrap.wrap(ss, width = width - s1len - indent - 1):
78	s += '%s%s%s%s%s' % (' ' * indent, s1, l, lineend, os.linesep)
79	if first and s1len > 0:
80	s1 = ' ' * s1len
81	if lineend != '':
82	s = s[:0 - len(os.linesep) - 1] + os.linesep
83	return s
84
85	def title():
86	return 'RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str())
87
88	def command_line():
89	return wrap(('command: ', ' '.join(sys.argv)), lineend = '\\')
90
91	class warnings_errors:
92
93	def __init__(self, rtems):
94	self.rtems = path.host(rtems)
95	self.reset()
96	self.groups = { 'groups' : ['Shared', 'BSP', 'Network', 'Tests',
97	'LibCPU', 'CPU Kit'],
98	'exclude' : '.Makefile.',
99	'CPU Kit' : '.cpukit/.',
100	'Network' : '.libnetworking/.\|.librpc/.',
101	'Tests' : '.testsuites/.',
102	'BSP' : '.libbsp/.',
103	'LibCPU' : '.libcpu/.',
104	'Shared' : '.shared/.' }
105	self.arch = None
106	self.bsp = None
107	self.build = None
108
109	def _opts(self, arch = None, bsp = None, build = None):
110	if arch is None:
111	arch = self.arch
112	if bsp is None:
113	bsp = self.bsp
114	if build is None:
115	build = self.build
116	return arch, bsp, build
117
118	def _key(self, arch, bsp, build):
119	arch, bsp, build = self._opts(arch, bsp, build)
120	return '%s/%s-%s' % (arch, bsp, build)
121
122	def _get_warnings(self, arch = None, bsp = None, build = None):
123	arch, bsp, build = self._opts(arch = arch, bsp = bsp, build = build)
124	if arch is None:
125	arch = '.*'
126	if bsp is None:
127	bsp = '.*'
128	if build is None:
129	build = '.*'
130	selector = re.compile('^%s/%s-%s$' % (arch, bsp, build))
131	warnings = [w for w in self.warnings if selector.match(w)]
132	return sorted(warnings)
133
134	def _total(self, archive):
135	total = 0
136	for a in archive:
137	total += archive[a]
138	return total
139
140	def _analyze(self, warnings, exclude):
141	def _group(data, category, name, warning, count, groups, group_regx):
142	if 'groups' not in data:
143	data['groups'] = { }
144	if category not in data['groups']:
145	data['groups'][category] = { 'totals' : { } }
146	if name not in data['groups'][category]:
147	data['groups'][category][name] = { }
148	for group in groups:
149	if group not in data['groups'][category]['totals']:
150	data['groups'][category]['totals'][group] = 0
151	if group not in data['groups'][category][name]:
152	data['groups'][category][name][group] = 0
153	if group_regx[group].match(warning):
154	data['groups'][category][name][group] += count
155	data['groups'][category]['totals'][group] += count
156	break
157
158	def _update(data, category, name, warning, count, groups, group_regx):
159	if category not in data:
160	data[category] = { }
161	if name not in data[category]:
162	data[category][name] = { }
163	if warning not in data[category][name]:
164	data[category][name][warning] = 0
165	data[category][name][warning] += count
166	_group(data, category, name, w, count, groups, group_regx)
167
168	data = { }
169	group_regx = { }
170	for group in self.groups['groups']:
171	group_regx[group] = re.compile(self.groups[group])
172	exclude_regx = re.compile(exclude)
173	for warning in warnings:
174	arch = warning.split('/', 1)[0]
175	arch_bsp = warning.split('-', 1)[0]
176	build = warning.split('-', 1)[1]
177	for w in self.warnings[warning]:
178	if not exclude_regx.match(w):
179	count = self.warnings[warning][w]
180	_update(data, 'arch', arch, w, count,
181	self.groups['groups'], group_regx)
182	_update(data, 'arch_bsp', arch_bsp, w, count,
183	self.groups['groups'], group_regx)
184	_update(data, 'build', build, w, count,
185	self.groups['groups'], group_regx)
186	for category in ['arch', 'arch_bsp', 'build']:
187	common = {}
188	for name in data[category]:
189	for w in data[category][name]:
190	if w not in common:
191	for other in [n for n in data[category] if n != name]:
192	if w in data[category][other]:
193	common[w] = data[category][name][w]
194	_group(data, category, 'common', w, common[w],
195	self.groups['groups'], group_regx)
196	data[category]['common'] = common
197	return data
198
199	def _report_category(self, label, warnings, group_counts):
200	width = 70
201	cols_1 = [width]
202	cols_2 = [8, width - 8]
203	cols_4 = textbox.even_columns(4, width)
204	cols_2_4 = textbox.merge_columns([cols_2, cols_4])
205	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
206	s += textbox.row(cols_1, [' ' + label], indent = 1)
207	s += textbox.line(cols_1, marker = '+', indent = 1)
208	builds = ['common'] + sorted([b for b in warnings if b != 'common'])
209	common = warnings['common']
210	for build in builds:
211	build_warnings = warnings[build]
212	if build is not 'common':
213	build_warnings = [w for w in build_warnings if w not in common]
214	s += textbox.row(cols_1,
215	[' %s : %d warning(s)' % (build,
216	len(build_warnings))],
217	indent = 1)
218	if len(build_warnings) == 0:
219	s += textbox.line(cols_1, marker = '+', indent = 1)
220	else:
221	s += textbox.line(cols_4, marker = '+', indent = 1)
222	if build not in group_counts:
223	gs = [0 for group in self.groups['groups']]
224	else:
225	gs = []
226	for g in range(0, len(self.groups['groups'])):
227	group = self.groups['groups'][g]
228	if group in group_counts[build]:
229	count = group_counts[build][group]
230	else:
231	count = 0
232	gs += ['%*s' % (cols_4[g % 4] - 2,
233	'%s : %4d' % (group, count))]
234	for row in range(0, len(self.groups['groups']), 4):
235	if row + 4 > len(self.groups['groups']):
236	d = gs[row:] + \
237	['' for r in range(row,
238	len(self.groups['groups']))]
239	else:
240	d = gs[row:+4]
241	s += textbox.row(cols_4, d, indent = 1)
242	s += textbox.line(cols_2_4, marker = '+', indent = 1)
243	vw = sorted([(w, warnings[build][w]) for w in build_warnings],
244	key = operator.itemgetter(1),
245	reverse = True)
246	for w in vw:
247	c1 = '%6d' % w[1]
248	for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3):
249	s += textbox.row(cols_2, [c1, l], indent = 1)
250	c1 = ' ' * 6
251	s += textbox.line(cols_2, marker = '+', indent = 1)
252	return s
253
254	def _report_warning_map(self):
255	builds = self.messages['warnings']
256	width = 70
257	cols_1 = [width]
258	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
259	s += textbox.row(cols_1, [' Warning Map'], indent = 1)
260	s += textbox.line(cols_1, marker = '+', indent = 1)
261	for build in builds:
262	messages = builds[build]
263	s += textbox.row(cols_1, [' %s : %d' % (build, len(messages))], indent = 1)
264	s += textbox.line(cols_1, marker = '+', indent = 1)
265	for msg in messages:
266	for l in textwrap.wrap(msg, width = width - 3):
267	s += textbox.row(cols_1, [' ' + l], indent = 1)
268	for l in textwrap.wrap(messages[msg], width = width - 3 - 4):
269	s += textbox.row(cols_1, [' ' + l], indent = 1)
270	s += textbox.line(cols_1, marker = '+', indent = 1)
271	return s
272
273	def report(self):
274	arch, bsp, build = self._opts()
275	warnings = self._get_warnings(arch, bsp, build)
276	total = 0
277	for build in warnings:
278	total += self._total(self.warnings[build])
279	if total == 0:
280	s = ' No warnings'
281	else:
282	data = self._analyze(warnings, self.groups['exclude'])
283	s = self._report_category('By Architecture (total : %d)' % (total),
284	data['arch'], data['groups']['arch'])
285	s += os.linesep
286	s += self._report_category('By BSP (total : %d)' % (total),
287	data['arch_bsp'], data['groups']['arch_bsp'])
288	s += os.linesep
289	s += self._report_category('By Build (total : %d)' % (total),
290	data['build'], data['groups']['build'])
291	s += os.linesep
292	s += self._report_warning_map()
293	s += os.linesep
294
295	return s
296
297	def set_build(self, arch, bsp, build):
298	self.arch = arch
299	self.bsp = bsp
300	self.build = build
301	self.build_key = '%s/%s-%s' % (arch, bsp, build)
302	if self.build_key not in self.warnings:
303	self.warnings[self.build_key] = {}
304	if self.build_key not in self.errors:
305	self.errors[self.build_key] = {}
306
307	def clear_build(self):
308	self.arch = None
309	self.bsp = None
310	self.build = None
311	self.build_key = None
312
313	def get_warning_count(self):
314	return self.warning_count
315
316	def get_error_count(self):
317	return self.error_count
318
319	def reset(self):
320	self.warnings = { }
321	self.warning_count = 0
322	self.errors = { }
323	self.error_count = 0
324	self.messages = { 'warnings' : { }, 'errors' : { } }
325
326	def get_warning_messages(self, arch = None, bsp = None, build = None):
327	key = self._key(arch, bsp, build)
328	if key not in self.messages['warnings']:
329	return []
330	messages = self.messages['warnings'][key]
331	return ['%s %s' % (m, messages[m]) for m in messages]
332
333	def get_error_messages(self, arch = None, bsp = None, build = None):
334	key = self._key(arch, bsp, build)
335	if key not in self.messages['errors']:
336	return []
337	messages = self.messages['errors'][key]
338	return ['%s %s' % (m, messages[m]) for m in messages]
339
340	def output(self, text):
341	def _line_split(line, source_base):
342	ls = line.split(' ', 1)
343	fname = ls[0].split(':')
344	#
345	# Ignore compiler option warnings.
346	#
347	if len(fname) < 4:
348	return None
349	p = path.abspath(fname[0])
350	p = p.replace(source_base, '')
351	if path.isabspath(p):
352	p = p[1:]
353	return p, fname[1], fname[2], ls[1]
354
355	if self.build_key is not None and \
356	(' warning:' in text or ' error:' in text):
357	for l in text.splitlines():
358	if ' warning:' in l:
359	self.warning_count += 1
360	archive = self.warnings[self.build_key]
361	messages = 'warnings'
362	elif ' error:' in l:
363	self.error_count += 1
364	archive = self.errors[self.build_key]
365	messages = 'errors'
366	else:
367	continue
368	line_parts = _line_split(l, self.rtems)
369	if line_parts is not None:
370	src, line, pos, msg = line_parts
371	where = '%s:%s:%s' % (src, line, pos)
372	if where not in archive:
373	archive[where] = 1
374	else:
375	archive[where] += 1
376	if self.build_key not in self.messages[messages]:
377	self.messages[messages][self.build_key] = { }
378	self.messages[messages][self.build_key][where] = msg
379
380	log.output(text)
381
382	class results:
383
384	def __init__(self):
385	self.passes = []
386	self.fails = []
387
388	def _arch_bsp(self, arch, bsp):
389	return '%s/%s' % (arch, bsp)
390
391	def add(self, good, arch, bsp, configure, warnings, error_messages):
392	if good:
393	self.passes += [(arch, bsp, configure, warnings, None)]
394	else:
395	self.fails += [(arch, bsp, configure, warnings, error_messages)]
396
397	def report(self):
398	log.notice('* Passes: %d Failures: %d' %
399	(len(self.passes), len(self.fails)))
400	log.output()
401	log.output('Build Report')
402	log.output(' Passes: %d Failures: %d' %
403	(len(self.passes), len(self.fails)))
404	log.output(' Failures:')
405	if len(self.fails) == 0:
406	log.output(' None')
407	else:
408	max_col = 0
409	for f in self.fails:
410	arch_bsp = self._arch_bsp(f[0], f[1])
411	if len(arch_bsp) > max_col:
412	max_col = len(arch_bsp)
413	for f in self.fails:
414	config_cmd = f[2]
415	config_at = config_cmd.find('configure')
416	if config_at != -1:
417	config_cmd = config_cmd[config_at:]
418	log.output(' %*s:' % (max_col + 2, self._arch_bsp(f[0], f[1])))
419	s1 = ' ' * 6
420	log.output(wrap([s1, config_cmd], lineend = '\\', width = 75))
421	if f[4] is not None:
422	s1 = ' ' * len(s1)
423	for msg in f[4]:
424	log.output(wrap([s1, msg], lineend = '\\'))
425	log.output(' Passes:')
426	if len(self.passes) == 0:
427	log.output(' None')
428	else:
429	max_col = 0
430	for f in self.passes:
431	arch_bsp = self._arch_bsp(f[0], f[1])
432	if len(arch_bsp) > max_col:
433	max_col = len(arch_bsp)
434	for f in self.passes:
435	config_cmd = f[2]
436	config_at = config_cmd.find('configure')
437	if config_at != -1:
438	config_cmd = config_cmd[config_at:]
439	log.output(' %*s: %5d:' % (max_col + 2,
440	self._arch_bsp(f[0], f[1]),
441	f[3]))
442	log.output(wrap([' ' * 6, config_cmd], lineend = '\\', width = 75))
443
444	class configuration:
445
446	def __init__(self):
447	self.config = configparser.ConfigParser()
448	self.name = None
449	self.archs = { }
450	self.builds_ = { }
451	self.profiles = { }
452	self.configurations = { }
453
454	def __str__(self):
455	import pprint
456	s = self.name + os.linesep
457	s += 'Archs:' + os.linesep + \
458	pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
459	s += 'Builds:' + os.linesep + \
460	pprint.pformat(self.builds_, indent = 1, width = 80) + os.linesep
461	s += 'Profiles:' + os.linesep + \
462	pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
463	return s
464
465	def _get_item(self, section, label, err = True):
466	try:
467	rec = self.config.get(section, label).replace(os.linesep, ' ')
468	return rec
469	except:
470	if err:
471	raise error.general('config: no "%s" found in "%s"' % (label, section))
472	return None
473
474	def _get_items(self, section, err = True):
475	try:
476	items = [(name, key.replace(os.linesep, ' ')) for name, key in self.config.items(section)]
477	return items
478	except:
479	if err:
480	raise error.general('config: section "%s" not found' % (section))
481	return []
482
483	def _comma_list(self, section, label, error = True):
484	items = self._get_item(section, label, error)
485	if items is None:
486	return []
487	return sorted(set([a.strip() for a in items.split(',')]))
488
489	def _get_item_names(self, section, err = True):
490	try:
491	return [item[0] for item in self.config.items(section)]
492	except:
493	if err:
494	raise error.general('config: section "%s" not found' % (section))
495	return []
496
497	def _build_options(self, build, nesting = 0):
498	if ':' in build:
499	section, name = build.split(':', 1)
500	opts = [self._get_item(section, name)]
501	return opts
502	builds = self.builds_['builds']
503	if build not in builds:
504	raise error.general('build %s not found' % (build))
505	if nesting > 20:
506	raise error.general('nesting build %s' % (build))
507	options = []
508	for option in self.builds_['builds'][build]:
509	if ':' in option:
510	section, name = option.split(':', 1)
511	opts = [self._get_item(section, name)]
512	else:
513	opts = self._options(option, nesting + 1)
514	for opt in opts:
515	if opt not in options:
516	options += [opt]
517	return options
518
519	def load(self, name, build):
520	if not path.exists(name):
521	raise error.general('config: cannot read configuration: %s' % (name))
522	self.name = name
523	try:
524	self.config.read(name)
525	except configparser.ParsingError as ce:
526	raise error.general('config: %s' % (ce))
527	archs = []
528	self.profiles['profiles'] = self._comma_list('profiles', 'profiles', error = False)
529	if len(self.profiles['profiles']) == 0:
530	self.profiles['profiles'] = ['tier-%d' % (t) for t in range(1,4)]
531	for p in self.profiles['profiles']:
532	profile = {}
533	profile['name'] = p
534	profile['archs'] = self._comma_list(profile['name'], 'archs')
535	archs += profile['archs']
536	for arch in profile['archs']:
537	bsps = 'bsps_%s' % (arch)
538	profile[bsps] = self._comma_list(profile['name'], bsps)
539	self.profiles[profile['name']] = profile
540	for a in set(archs):
541	arch = {}
542	arch['excludes'] = {}
543	for exclude in self._comma_list(a, 'exclude', error = False):
544	arch['excludes'][exclude] = ['all']
545	for i in self._get_items(a, False):
546	if i[0].startswith('exclude_'):
547	exclude = i[0][len('exclude_'):]
548	if exclude not in arch['excludes']:
549	arch['excludes'][exclude] = []
550	arch['excludes'][exclude] += sorted(set([b.strip() for b in i[1].split(',')]))
551	arch['bsps'] = self._comma_list(a, 'bsps', error = False)
552	for b in arch['bsps']:
553	arch[b] = {}
554	arch[b]['bspopts'] = self._comma_list(a, 'bspopts_%s' % (b), error = False)
555	self.archs[a] = arch
556	builds = {}
557	builds['default'] = self._get_item('builds', 'default')
558	if build is None:
559	build = builds['default']
560	builds['config'] = { }
561	for config in self._get_items('config'):
562	builds['config'][config[0]] = config[1]
563	builds['build'] = build
564	builds_ = self._get_item_names('builds')
565	builds['builds'] = {}
566	for build in builds_:
567	build_builds = self._comma_list('builds', build)
568	has_config = False
569	has_build = False
570	for b in build_builds:
571	if ':' in b:
572	if has_build:
573	raise error.general('config and build in build: %s' % (build))
574	has_config = True
575	else:
576	if has_config:
577	raise error.general('config and build in build: %s' % (build))
578	has_build = True
579	builds['builds'][build] = build_builds
580	self.builds_ = builds
581
582	def build(self):
583	return self.builds_['build']
584
585	def builds(self):
586	if self.builds_['build'] in self.builds_['builds']:
587	build = self.builds_['builds'][self.builds_['build']]
588	if ':' in build[0]:
589	return [self.builds_['build']]
590	return build
591	return None
592
593	def build_options(self, build):
594	return ' '.join(self._build_options(build))
595
596	def excludes(self, arch):
597	excludes = self.archs[arch]['excludes'].keys()
598	for exclude in self.archs[arch]['excludes']:
599	if 'all' not in self.archs[arch]['excludes'][exclude]:
600	excludes.remove(exclude)
601	return sorted(excludes)
602
603	def archs(self):
604	return sorted(self.archs.keys())
605
606	def arch_present(self, arch):
607	return arch in self.archs
608
609	def arch_bsps(self, arch):
610	return sorted(self.archs[arch]['bsps'])
611
612	def bsp_present(self, arch, bsp):
613	return bsp in self.archs[arch]['bsps']
614
615	def bsp_excludes(self, arch, bsp):
616	excludes = self.archs[arch]['excludes'].keys()
617	for exclude in self.archs[arch]['excludes']:
618	if bsp not in self.archs[arch]['excludes'][exclude]:
619	excludes.remove(exclude)
620	return sorted(excludes)
621
622	def bspopts(self, arch, bsp):
623	return self.archs[arch][bsp]['bspopts']
624
625	def profile_present(self, profile):
626	return profile in self.profiles
627
628	def profile_archs(self, profile):
629	return self.profiles[profile]['archs']
630
631	def profile_arch_bsps(self, profile, arch):
632	return self.profiles[profile]['bsps_%s' % (arch)]
633
634	def report(self, profiles = True, builds = True, architectures = True):
635	width = 70
636	cols_1 = [width]
637	cols_2 = [10, width - 10]
638	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
639	s1 = ' File'
640	colon = ':'
641	for l in textwrap.wrap(self.name, width = cols_2[1] - 3):
642	s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1)
643	colon = ' '
644	s1 = ' ' * len(s1)
645	s += textbox.line(cols_1, marker = '+', indent = 1)
646	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
647	if profiles:
648	profiles = sorted(self.profiles['profiles'])
649	bsps = 0
650	for profile in profiles:
651	archs = sorted(self.profiles[profile]['archs'])
652	for arch in archs:
653	bsps += len(self.profiles[profile]['bsps_%s' % (arch)])
654	s += textbox.row(cols_1,
655	[' Profiles : %d/%d' % (len(archs), bsps)],
656	indent = 1)
657	for profile in profiles:
658	textbox.row(cols_2,
659	[profile, self.profiles[profile]['name']],
660	indent = 1)
661	s += textbox.line(cols_1, marker = '+', indent = 1)
662	for profile in profiles:
663	s += textbox.row(cols_1, [' %s' % (profile)], indent = 1)
664	profile = self.profiles[profile]
665	archs = sorted(profile['archs'])
666	for arch in archs:
667	s += textbox.line(cols_2, marker = '+', indent = 1)
668	s1 = ' ' + arch
669	for l in textwrap.wrap(', '.join(profile['bsps_%s' % (arch)]),
670	width = cols_2[1] - 2):
671	s += textbox.row(cols_2, [s1, ' ' + l], indent = 1)
672	s1 = ' ' * len(s1)
673	s += textbox.line(cols_2, marker = '+', indent = 1)
674	if builds:
675	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
676	s += textbox.row(cols_1,
677	[' Builds: %s (default)' % (self.builds_['default'])],
678	indent = 1)
679	builds = self.builds_['builds']
680	bsize = 0
681	for build in builds:
682	if len(build) > bsize:
683	bsize = len(build)
684	cols_b = [bsize + 2, width - bsize - 2]
685	s += textbox.line(cols_b, marker = '+', indent = 1)
686	for build in builds:
687	s1 = ' ' + build
688	for l in textwrap.wrap(', '.join(builds[build]),
689	width = cols_b[1] - 2):
690	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
691	s1 = ' ' * len(s1)
692	s += textbox.line(cols_b, marker = '+', indent = 1)
693	configs = self.builds_['config']
694	s += textbox.row(cols_1,
695	[' Configure Options: %d' % (len(configs))],
696	indent = 1)
697	csize = 0
698	for config in configs:
699	if len(config) > csize:
700	csize = len(config)
701	cols_c = [csize + 3, width - csize - 3]
702	s += textbox.line(cols_c, marker = '+', indent = 1)
703	for config in configs:
704	s1 = ' ' + config
705	for l in textwrap.wrap(configs[config], width = cols_c[1] - 3):
706	s += textbox.row(cols_c, [s1, ' ' + l], indent = 1)
707	s1 = ' ' * len(s1)
708	s += textbox.line(cols_c, marker = '+', indent = 1)
709	if architectures:
710	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
711	archs = sorted(self.archs.keys())
712	bsps = 0
713	asize = 0
714	for arch in archs:
715	if len(arch) > asize:
716	asize = len(arch)
717	bsps += len(self.archs[arch]['bsps'])
718	s += textbox.row(cols_1,
719	[' Architectures : %d (bsps: %d)' % (len(archs), bsps)],
720	indent = 1)
721	cols_a = [asize + 2, width - asize - 2]
722	s += textbox.line(cols_a, marker = '+', indent = 1)
723	for arch in archs:
724	s += textbox.row(cols_a,
725	[' ' + arch, ' %d' % (len(self.archs[arch]['bsps']))],
726	indent = 1)
727	s += textbox.line(cols_a, marker = '+', indent = 1)
728	for archn in archs:
729	arch = self.archs[archn]
730	if len(arch['bsps']) > 0:
731	bsize = 0
732	for bsp in arch['bsps']:
733	if len(bsp) > bsize:
734	bsize = len(bsp)
735	cols_b = [bsize + 3, width - bsize - 3]
736	s += textbox.row(cols_1, [' ' + archn + ':'], indent = 1)
737	s += textbox.line(cols_b, marker = '+', indent = 1)
738	for bsp in arch['bsps']:
739	s1 = ' ' + bsp
740	bspopts = ' '.join(arch[bsp]['bspopts'])
741	if len(bspopts):
742	for l in textwrap.wrap('bopt: ' + bspopts,
743	width = cols_b[1] - 3):
744	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
745	s1 = ' ' * len(s1)
746	excludes = []
747	for exclude in arch['excludes']:
748	if bsp in arch['excludes'][exclude]:
749	excludes += [exclude]
750	excludes = ', '.join(excludes)
751	if len(excludes):
752	for l in textwrap.wrap('ex: ' + excludes,
753	width = cols_b[1] - 3):
754	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
755	s1 = ' ' * len(s1)
756	if len(bspopts) == 0 and len(excludes) == 0:
757	s += textbox.row(cols_b, [s1, ' '], indent = 1)
758	s += textbox.line(cols_b, marker = '+', indent = 1)
759	return s
760
761	class build:
762
763	def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
764	self.config = config
765	self.build_dir = build_dir
766	self.rtems_version = version
767	self.prefix = prefix
768	self.tools = tools
769	self.rtems = rtems
770	self.options = options
771	self.errors = { 'configure': 0,
772	'build': 0,
773	'tests': 0,
774	'fails': []}
775	self.counts = { 'h' : 0,
776	'exes' : 0,
777	'objs' : 0,
778	'libs' : 0 }
779	self.warnings_errors = warnings_errors(rtems)
780	self.results = results()
781	if not path.exists(path.join(rtems, 'configure')) or \
782	not path.exists(path.join(rtems, 'Makefile.in')) or \
783	not path.exists(path.join(rtems, 'cpukit')):
784	raise error.general('RTEMS source path does not look like RTEMS')
785
786	def _error_str(self):
787	return 'Status: configure:%d build:%d' % \
788	(self.errors['configure'], self.errors['build'])
789
790	def _path(self, arch, bsp):
791	return path.join(self.build_dir, arch, bsp)
792
793	def _archs(self, build_data):
794	return sorted(build_data.keys())
795
796	def _bsps(self, arch):
797	return self.config.arch_bsps(arch)
798
799	def _build(self):
800	return self.config.build()
801
802	def _builds(self, arch, bsp):
803	builds = self.config.builds()
804	if builds is None:
805	return None
806	for b in self.config.excludes(arch):
807	if b in builds:
808	builds.remove(b)
809	for b in self.config.bsp_excludes(arch, bsp):
810	if b in builds:
811	builds.remove(b)
812	return builds
813
814	def _arch_bsp_dir_make(self, arch, bsp):
815	if not path.exists(self._path(arch, bsp)):
816	path.mkdir(self._path(arch, bsp))
817
818	def _arch_bsp_dir_clean(self, arch, bsp):
819	if path.exists(self._path(arch, bsp)):
820	path.removeall(self._path(arch, bsp))
821
822	def _config_command(self, commands, arch, bsp):
823	if type(commands) is not list:
824	commands = [commands]
825	cmd = [path.join(self.rtems, 'configure')]
826	commands += self.config.bspopts(arch, bsp)
827	for c in commands:
828	c = c.replace('@PREFIX@', self.prefix)
829	c = c.replace('@RTEMS_VERSION@', self.rtems_version)
830	c = c.replace('@ARCH@', arch)
831	c = c.replace('@BSP@', bsp)
832	cmd += [c]
833	return ' '.join(cmd)
834
835	def _build_set(self, builds):
836	build_set = { }
837	for build in builds:
838	build_set[build] = self.config.build_options(build)
839	return build_set
840
841	def _build_dir(self, arch, bsp, build):
842	return path.join(self._path(arch, bsp), build)
843
844	def _count_files(self, arch, bsp, build):
845	counts = { 'h' : 0,
846	'exes' : 0,
847	'objs' : 0,
848	'libs' : 0 }
849	for root, dirs, files in os.walk(self._build_dir(arch, bsp, build)):
850	for file in files:
851	if file.endswith('.exe'):
852	counts['exes'] += 1
853	elif file.endswith('.o'):
854	counts['objs'] += 1
855	elif file.endswith('.a'):
856	counts['libs'] += 1
857	elif file.endswith('.h'):
858	counts['h'] += 1
859	for f in self.counts:
860	if f in counts:
861	self.counts[f] += counts[f]
862	return counts
863
864	def _have_failures(self, fails):
865	return len(fails) != 0
866
867	def _warnings_report(self):
868	if self.options['warnings-report'] is not None:
869	with open(self.options['warnings-report'], 'w') as f:
870	f.write(title() + os.linesep)
871	f.write(os.linesep)
872	f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'),
873	os.linesep))
874	f.write(os.linesep)
875	f.write(command_line() + os.linesep)
876	f.write(self.warnings_errors.report())
877
878	def _finished(self):
879	log.notice('+ warnings:%d exes:%d objs:%d libs:%d' % \
880	(self.warnings_errors.get_warning_count(), self.counts['exes'],
881	self.counts['objs'], self.counts['libs']))
882	log.output()
883	log.output('Warnings:')
884	log.output(self.warnings_errors.report())
885	log.output()
886	log.notice('Failures:')
887	log.notice(self.failures_report(self.errors['fails']))
888	self._warnings_report()
889
890	def failures_report(self, fails):
891	if not self._have_failures(fails):
892	return ' No failure(s)'
893	absize = 0
894	bsize = 0
895	ssize = 0
896	for f in fails:
897	arch_bsp = '%s/%s' % (f[1], f[2])
898	if len(arch_bsp) > absize:
899	absize = len(arch_bsp)
900	if len(f[3]) > bsize:
901	bsize = len(f[3])
902	if len(f[0]) > ssize:
903	ssize = len(f[0])
904	fc = 1
905	s = ''
906	for f in fails:
907	fcl = ' %3d' % (fc)
908	arch_bsp = '%s/%s' % (f[1], f[2])
909	state = f[0]
910	s += '%s %-s %-s %-*s:%s' % \
911	(fcl, bsize, f[3], absize, arch_bsp, ssize, state, os.linesep)
912	s1 = ' ' * 6
913	s += wrap((s1, 'configure: ' + f[4]), lineend = '\\', width = 75)
914	for e in self.warnings_errors.get_error_messages(f[1], f[2], f[3]):
915	s += wrap([s1, 'error: ' + e])
916	fc += 1
917	return s
918
919	def build_arch_bsp(self, arch, bsp):
920	if not self.config.bsp_present(arch, bsp):
921	raise error.general('BSP not found: %s/%s' % (arch, bsp))
922	log.output('-' * 70)
923	log.notice('] BSP: %s/%s' % (arch, bsp))
924	log.notice('. Creating: %s' % (self._path(arch, bsp)))
925	self._arch_bsp_dir_clean(arch, bsp)
926	self._arch_bsp_dir_make(arch, bsp)
927	builds = self._builds(arch, bsp)
928	if builds is None:
929	raise error.general('build not found: %s' % (self._build()))
930	build_set = self._build_set(builds)
931	bsp_start = datetime.datetime.now()
932	env_path = os.environ['PATH']
933	os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
934	os.pathsep + os.environ['PATH']
935	fails = []
936	for bs in sorted(build_set.keys()):
937	self.warnings_errors.set_build(arch, bsp, bs)
938	start = datetime.datetime.now()
939	log.output('- ' * 35)
940	log.notice('. Configuring: %s' % (bs))
941	try:
942	warnings = self.warnings_errors.get_warning_count()
943	result = '+ Pass'
944	bpath = self._build_dir(arch, bsp, bs)
945	good = True
946	error_messages = None
947	path.mkdir(bpath)
948	config_cmd = self._config_command(build_set[bs], arch, bsp)
949	cmd = config_cmd
950	e = execute.capture_execution(log = self.warnings_errors)
951	log.output(wrap(('run: ', cmd), lineend = '\\'))
952	if self.options['dry-run']:
953	exit_code = 0
954	else:
955	exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
956	if exit_code != 0:
957	result = '- FAIL'
958	failure = ('configure', arch, bsp, bs, config_cmd)
959	fails += [failure]
960	self.errors['configure'] += 1
961	self.errors['fails'] += [failure]
962	log.notice('- Configure failed: %s' % (bs))
963	log.output('cmd failed: %s' % (cmd))
964	good = False
965	else:
966	log.notice('. Building: %s' % (bs))
967	cmd = 'make'
968	if 'jobs' in self.options:
969	cmd += ' -j %s' % (self.options['jobs'])
970	log.output('run: ' + cmd)
971	if self.options['dry-run']:
972	exit_code = 0
973	else:
974	exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
975	if exit_code != 0:
976	error_messages = self.warnings_errors.get_error_messages()
977	result = '- FAIL'
978	failure = ('build', arch, bsp, bs, config_cmd, error_messages)
979	fails += [failure]
980	self.errors['build'] += 1
981	self.errors['fails'] += [failure]
982	log.notice('- FAIL: %s: %s' % (bs, self._error_str()))
983	log.output('cmd failed: %s' % (cmd))
984	good = False
985	files = self._count_files(arch, bsp, bs)
986	log.notice('%s: %s: warnings:%d exes:%d objs:%s libs:%d' % \
987	(result, bs,
988	self.warnings_errors.get_warning_count() - warnings,
989	files['exes'], files['objs'], files['libs']))
990	log.notice(' %s' % (self._error_str()))
991	self.results.add(good, arch, bsp, config_cmd,
992	self.warnings_errors.get_warning_count() - warnings,
993	error_messages)
994	if not good and self.options['stop-on-error']:
995	raise error.general('Configuring %s failed' % (bs))
996	finally:
997	end = datetime.datetime.now()
998	if not self.options['no-clean']:
999	log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs)))
1000	path.removeall(self._build_dir(arch, bsp, bs))
1001	log.notice('^ Time %s' % (str(end - start)))
1002	self.warnings_errors.clear_build()
1003	bsp_end = datetime.datetime.now()
1004	log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start)))
1005	log.output('Failure Report:')
1006	log.output(self.failures_report(fails))
1007	os.environ['PATH'] = env_path
1008
1009	def build_bsp(self, arch, bsp):
1010	self.build_arch_bsp(arch, bsp)
1011	self._finished()
1012
1013	def build_arch(self, arch):
1014	start = datetime.datetime.now()
1015	log.output('=' * 70)
1016	log.notice(']] Architecture: %s' % (arch))
1017	if not self.config.arch_present(arch):
1018	raise error.general('Architecture not found: %s' % (arch))
1019	for bsp in self._bsps(arch):
1020	self.build_arch_bsp(arch, bsp)
1021	end = datetime.datetime.now()
1022	log.notice('^ Architecture Time %s' % (str(end - start)))
1023	self._finished()
1024
1025	def build(self):
1026	for arch in self.config.archs():
1027	self.build_arch(arch)
1028	log.notice('^ Profile Time %s' % (str(end - start)))
1029	self._finished()
1030
1031	def build_profile(self, profile):
1032	if not self.config.profile_present(profile):
1033	raise error.general('Profile not found: %s' % (profile))
1034	start = datetime.datetime.now()
1035	log.notice(']] Profile: %s' % (profile))
1036	for arch in self.config.profile_archs(profile):
1037	for bsp in self.config.profile_arch_bsps(profile, arch):
1038	self.build_arch_bsp(arch, bsp)
1039	end = datetime.datetime.now()
1040	log.notice('^ Profile Time %s' % (str(end - start)))
1041	self._finished()
1042
1043	def run_args(args):
1044	b = None
1045	ec = 0
1046	try:
1047	#
1048	# On Windows MSYS2 prepends a path to itself to the environment
1049	# path. This means the RTEMS specific automake is not found and which
1050	# breaks the bootstrap. We need to remove the prepended path. Also
1051	# remove any ACLOCAL paths from the environment.
1052	#
1053	if os.name == 'nt':
1054	cspath = os.environ['PATH'].split(os.pathsep)
1055	if 'msys' in cspath[0] and cspath[0].endswith('bin'):
1056	os.environ['PATH'] = os.pathsep.join(cspath[1:])
1057
1058	top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
1059	prefix = '/opt/rtems/%s' % (rtems_version())
1060	tools = prefix
1061	build_dir = 'bsp-builds'
1062	logf = 'bsp-build-%s.txt' % \
1063	(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
1064	config_file = path.join(top, 'share', 'rtems', 'tester',
1065	'rtems', 'rtems-bsps.ini')
1066	if not path.exists(config_file):
1067	config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')
1068
1069	argsp = argparse.ArgumentParser()
1070	argsp.add_argument('--prefix', help = 'Prefix to build the BSP.',
1071	type = str)
1072	argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.',
1073	type = str)
1074	argsp.add_argument('--rtems', help = 'The RTEMS source tree.',
1075	type = str)
1076	argsp.add_argument('--config-report', help = 'Report the configuration.',
1077	action = 'store_true')
1078	argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
1079	type = str, default = None)
1080	argsp.add_argument('--build-path', help = 'Path to build in.',
1081	type = str)
1082	argsp.add_argument('--log', help = 'Log file.', type = str)
1083	argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
1084	action = 'store_true')
1085	argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
1086	action = 'store_true')
1087	argsp.add_argument('--profiles', help = 'Build the listed profiles.',
1088	type = str, default = 'tier-1')
1089	argsp.add_argument('--build', help = 'Build name to build.',
1090	type = str, default='all')
1091	argsp.add_argument('--arch', help = 'Build the specific architecture.',
1092	type = str)
1093	argsp.add_argument('--bsp', help = 'Build the specific BSP.',
1094	type = str)
1095	argsp.add_argument('--jobs', help = 'Number of jobs to run.',
1096	type = int, default = host.cpus())
1097	argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
1098	action = 'store_true')
1099
1100	opts = argsp.parse_args(args[1:])
1101	if opts.log is not None:
1102	logf = opts.log
1103	log.default = log.log([logf])
1104	log.notice(title())
1105	log.output(command_line())
1106	if opts.rtems is None:
1107	raise error.general('No RTEMS source provided on the command line')
1108	if opts.prefix is not None:
1109	prefix = path.shell(opts.prefix)
1110	if opts.rtems_tools is not None:
1111	tools = path.shell(opts.rtems_tools)
1112	if opts.build_path is not None:
1113	build_dir = path.shell(opts.build_path)
1114	if opts.bsp is not None and opts.arch is None:
1115	raise error.general('BSP provided but no architecture')
1116
1117	config = configuration()
1118	config.load(config_file, opts.build)
1119
1120	if opts.config_report:
1121	log.notice('Configuration Report:')
1122	log.notice(config.report())
1123	sys.exit(0)
1124
1125	options = { 'stop-on-error' : opts.stop_on_error,
1126	'no-clean' : opts.no_clean,
1127	'dry-run' : opts.dry_run,
1128	'jobs' : opts.jobs,
1129	'warnings-report' : opts.warnings_report }
1130
1131	b = build(config, rtems_version(), prefix, tools,
1132	path.shell(opts.rtems), build_dir, options)
1133	if opts.arch is not None:
1134	if opts.bsp is not None:
1135	b.build_bsp(opts.arch, opts.bsp)
1136	else:
1137	b.build_arch(opts.arch)
1138	else:
1139	for profile in opts.profiles.split(','):
1140	b.build_profile(profile.strip())
1141
1142	except error.general as gerr:
1143	print(gerr)
1144	print('BSP Build FAILED', file = sys.stderr)
1145	ec = 1
1146	except error.internal as ierr:
1147	print(ierr)
1148	print('BSP Build FAILED', file = sys.stderr)
1149	ec = 1
1150	except error.exit as eerr:
1151	pass
1152	except KeyboardInterrupt:
1153	log.notice('abort: user terminated')
1154	ec = 1
1155	if b is not None:
1156	b.results.report()
1157	sys.exit(ec)
1158
1159	def run():
1160	run_args(sys.argv)
1161
1162	if __name__ == "__main__":
1163	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: