Context Navigation

source: rtems-tools/tester/rt/check.py @ 9a8a03c

5

Last change on this file since 9a8a03c was 9a8a03c, checked in by Chris Johns <chrisj@…>, on 05/15/17 at 01:29:46

rtems-bsp-builder: Add all architectures and BSPs.

Add support to include other INI files.
Add support for extended interpolation on Python 2.7.

Property mode set to 100755

File size: 49.2 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2016-2017 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Redistribution and use in source and binary forms, with or without
9	# modification, are permitted provided that the following conditions are met:
10	#
11	# 1. Redistributions of source code must retain the above copyright notice,
12	# this list of conditions and the following disclaimer.
13	#
14	# 2. Redistributions in binary form must reproduce the above copyright notice,
15	# this list of conditions and the following disclaimer in the documentation
16	# and/or other materials provided with the distribution.
17	#
18	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21	# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22	# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23	# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24	# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25	# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26	# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27	# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28	# POSSIBILITY OF SUCH DAMAGE.
29	#
30
31	from __future__ import print_function
32
33	import argparse
34	import copy
35	import datetime
36	import operator
37	import os
38	import re
39	import sys
40	import textwrap
41
42	import pprint
43
44	try:
45	import configparser
46	except:
47	import ConfigParser as configparser
48
49	from rtemstoolkit import execute
50	from rtemstoolkit import error
51	from rtemstoolkit import host
52	from rtemstoolkit import log
53	from rtemstoolkit import path
54	from rtemstoolkit import textbox
55	from rtemstoolkit import version
56
57	def rtems_version():
58	return version.version()
59
60	def wrap(line, lineend = '', indent = 0, width = 75):
61	if type(line) is tuple or type(line) is list:
62	if len(line) >= 2:
63	s1 = line[0]
64	else:
65	s1 = ''
66	s2 = line[1:]
67	elif type(line) is str:
68	s1 = ''
69	s2 = [line]
70	else:
71	raise error.internal('line is not a tuple, list or string')
72	s1len = len(s1)
73	s = ''
74	first = True
75	for ss in s2:
76	if type(ss) is not str and type(ss) is not unicode:
77	raise error.internal('text needs to be a string')
78	for l in textwrap.wrap(ss, width = width - s1len - indent - 1):
79	s += '%s%s%s%s%s' % (' ' * indent, s1, l, lineend, os.linesep)
80	if first and s1len > 0:
81	s1 = ' ' * s1len
82	if lineend != '':
83	s = s[:0 - len(os.linesep) - 1] + os.linesep
84	return s
85
86	def title():
87	return 'RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str())
88
89	def command_line():
90	return wrap(('command: ', ' '.join(sys.argv)), lineend = '\\')
91
92	class warnings_errors:
93
94	def __init__(self, rtems):
95	self.rtems = path.host(rtems)
96	self.reset()
97	self.groups = { 'groups' : ['Shared', 'BSP', 'Network', 'Tests',
98	'LibCPU', 'CPU Kit'],
99	'exclude' : '.Makefile.',
100	'CPU Kit' : '.cpukit/.',
101	'Network' : '.libnetworking/.\|.librpc/.',
102	'Tests' : '.testsuites/.',
103	'BSP' : '.libbsp/.',
104	'LibCPU' : '.libcpu/.',
105	'Shared' : '.shared/.' }
106	self.arch = None
107	self.bsp = None
108	self.build = None
109
110	def _opts(self, arch = None, bsp = None, build = None):
111	if arch is None:
112	arch = self.arch
113	if bsp is None:
114	bsp = self.bsp
115	if build is None:
116	build = self.build
117	return arch, bsp, build
118
119	def _key(self, arch, bsp, build):
120	arch, bsp, build = self._opts(arch, bsp, build)
121	return '%s/%s-%s' % (arch, bsp, build)
122
123	def _get_warnings(self, arch = None, bsp = None, build = None):
124	arch, bsp, build = self._opts(arch = arch, bsp = bsp, build = build)
125	if arch is None:
126	arch = '.*'
127	if bsp is None:
128	bsp = '.*'
129	if build is None:
130	build = '.*'
131	selector = re.compile('^%s/%s-%s$' % (arch, bsp, build))
132	warnings = [w for w in self.warnings if selector.match(w)]
133	return sorted(warnings)
134
135	def _total(self, archive):
136	total = 0
137	for a in archive:
138	total += archive[a]
139	return total
140
141	def _analyze(self, warnings, exclude):
142	def _group(data, category, name, warning, count, groups, group_regx):
143	if 'groups' not in data:
144	data['groups'] = { }
145	if category not in data['groups']:
146	data['groups'][category] = { 'totals' : { } }
147	if name not in data['groups'][category]:
148	data['groups'][category][name] = { }
149	for group in groups:
150	if group not in data['groups'][category]['totals']:
151	data['groups'][category]['totals'][group] = 0
152	if group not in data['groups'][category][name]:
153	data['groups'][category][name][group] = 0
154	if group_regx[group].match(warning):
155	data['groups'][category][name][group] += count
156	data['groups'][category]['totals'][group] += count
157	break
158
159	def _update(data, category, name, warning, count, groups, group_regx):
160	if category not in data:
161	data[category] = { }
162	if name not in data[category]:
163	data[category][name] = { }
164	if warning not in data[category][name]:
165	data[category][name][warning] = 0
166	data[category][name][warning] += count
167	_group(data, category, name, w, count, groups, group_regx)
168
169	data = { }
170	group_regx = { }
171	for group in self.groups['groups']:
172	group_regx[group] = re.compile(self.groups[group])
173	exclude_regx = re.compile(exclude)
174	for warning in warnings:
175	arch = warning.split('/', 1)[0]
176	arch_bsp = warning.split('-', 1)[0]
177	build = warning.split('-', 1)[1]
178	for w in self.warnings[warning]:
179	if not exclude_regx.match(w):
180	count = self.warnings[warning][w]
181	_update(data, 'arch', arch, w, count,
182	self.groups['groups'], group_regx)
183	_update(data, 'arch_bsp', arch_bsp, w, count,
184	self.groups['groups'], group_regx)
185	_update(data, 'build', build, w, count,
186	self.groups['groups'], group_regx)
187	for category in ['arch', 'arch_bsp', 'build']:
188	common = {}
189	for name in data[category]:
190	for w in data[category][name]:
191	if w not in common:
192	for other in [n for n in data[category] if n != name]:
193	if w in data[category][other]:
194	common[w] = data[category][name][w]
195	_group(data, category, 'common', w, common[w],
196	self.groups['groups'], group_regx)
197	data[category]['common'] = common
198	return data
199
200	def _report_category(self, label, warnings, group_counts):
201	width = 70
202	cols_1 = [width]
203	cols_2 = [8, width - 8]
204	cols_4 = textbox.even_columns(4, width)
205	cols_2_4 = textbox.merge_columns([cols_2, cols_4])
206	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
207	s += textbox.row(cols_1, [' ' + label], indent = 1)
208	s += textbox.line(cols_1, marker = '+', indent = 1)
209	builds = ['common'] + sorted([b for b in warnings if b != 'common'])
210	common = warnings['common']
211	for build in builds:
212	build_warnings = warnings[build]
213	if build is not 'common':
214	build_warnings = [w for w in build_warnings if w not in common]
215	s += textbox.row(cols_1,
216	[' %s : %d warning(s)' % (build,
217	len(build_warnings))],
218	indent = 1)
219	if len(build_warnings) == 0:
220	s += textbox.line(cols_1, marker = '+', indent = 1)
221	else:
222	s += textbox.line(cols_4, marker = '+', indent = 1)
223	if build not in group_counts:
224	gs = [0 for group in self.groups['groups']]
225	else:
226	gs = []
227	for g in range(0, len(self.groups['groups'])):
228	group = self.groups['groups'][g]
229	if group in group_counts[build]:
230	count = group_counts[build][group]
231	else:
232	count = 0
233	gs += ['%*s' % (cols_4[g % 4] - 2,
234	'%s : %4d' % (group, count))]
235	for row in range(0, len(self.groups['groups']), 4):
236	if row + 4 > len(self.groups['groups']):
237	d = gs[row:] + \
238	['' for r in range(row,
239	len(self.groups['groups']))]
240	else:
241	d = gs[row:+4]
242	s += textbox.row(cols_4, d, indent = 1)
243	s += textbox.line(cols_2_4, marker = '+', indent = 1)
244	vw = sorted([(w, warnings[build][w]) for w in build_warnings],
245	key = operator.itemgetter(1),
246	reverse = True)
247	for w in vw:
248	c1 = '%6d' % w[1]
249	for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3):
250	s += textbox.row(cols_2, [c1, l], indent = 1)
251	c1 = ' ' * 6
252	s += textbox.line(cols_2, marker = '+', indent = 1)
253	return s
254
255	def _report_warning_map(self):
256	builds = self.messages['warnings']
257	width = 70
258	cols_1 = [width]
259	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
260	s += textbox.row(cols_1, [' Warning Map'], indent = 1)
261	s += textbox.line(cols_1, marker = '+', indent = 1)
262	for build in builds:
263	messages = builds[build]
264	s += textbox.row(cols_1, [' %s : %d' % (build, len(messages))], indent = 1)
265	s += textbox.line(cols_1, marker = '+', indent = 1)
266	for msg in messages:
267	for l in textwrap.wrap(msg, width = width - 3):
268	s += textbox.row(cols_1, [' ' + l], indent = 1)
269	for l in textwrap.wrap(messages[msg], width = width - 3 - 4):
270	s += textbox.row(cols_1, [' ' + l], indent = 1)
271	s += textbox.line(cols_1, marker = '+', indent = 1)
272	return s
273
274	def report(self):
275	arch, bsp, build = self._opts()
276	warnings = self._get_warnings(arch, bsp, build)
277	total = 0
278	for build in warnings:
279	total += self._total(self.warnings[build])
280	if total == 0:
281	s = ' No warnings'
282	else:
283	data = self._analyze(warnings, self.groups['exclude'])
284	s = self._report_category('By Architecture (total : %d)' % (total),
285	data['arch'], data['groups']['arch'])
286	s += os.linesep
287	s += self._report_category('By BSP (total : %d)' % (total),
288	data['arch_bsp'], data['groups']['arch_bsp'])
289	s += os.linesep
290	s += self._report_category('By Build (total : %d)' % (total),
291	data['build'], data['groups']['build'])
292	s += os.linesep
293	s += self._report_warning_map()
294	s += os.linesep
295
296	return s
297
298	def set_build(self, arch, bsp, build):
299	self.arch = arch
300	self.bsp = bsp
301	self.build = build
302	self.build_key = '%s/%s-%s' % (arch, bsp, build)
303	if self.build_key not in self.warnings:
304	self.warnings[self.build_key] = {}
305	if self.build_key not in self.errors:
306	self.errors[self.build_key] = {}
307
308	def clear_build(self):
309	self.arch = None
310	self.bsp = None
311	self.build = None
312	self.build_key = None
313
314	def get_warning_count(self):
315	return self.warning_count
316
317	def get_error_count(self):
318	return self.error_count
319
320	def reset(self):
321	self.warnings = { }
322	self.warning_count = 0
323	self.errors = { }
324	self.error_count = 0
325	self.messages = { 'warnings' : { }, 'errors' : { } }
326
327	def get_warning_messages(self, arch = None, bsp = None, build = None):
328	key = self._key(arch, bsp, build)
329	if key not in self.messages['warnings']:
330	return []
331	messages = self.messages['warnings'][key]
332	return ['%s %s' % (m, messages[m]) for m in messages]
333
334	def get_error_messages(self, arch = None, bsp = None, build = None):
335	key = self._key(arch, bsp, build)
336	if key not in self.messages['errors']:
337	return []
338	messages = self.messages['errors'][key]
339	return ['%s %s' % (m, messages[m]) for m in messages]
340
341	def output(self, text):
342	def _line_split(line, source_base):
343	ls = line.split(' ', 1)
344	fname = ls[0].split(':')
345	#
346	# Ignore compiler option warnings.
347	#
348	if len(fname) < 4:
349	return None
350	p = path.abspath(fname[0])
351	p = p.replace(source_base, '')
352	if path.isabspath(p):
353	p = p[1:]
354	return p, fname[1], fname[2], ls[1]
355
356	if self.build_key is not None and \
357	(' warning:' in text or ' error:' in text):
358	for l in text.splitlines():
359	if ' warning:' in l:
360	self.warning_count += 1
361	archive = self.warnings[self.build_key]
362	messages = 'warnings'
363	elif ' error:' in l:
364	self.error_count += 1
365	archive = self.errors[self.build_key]
366	messages = 'errors'
367	else:
368	continue
369	line_parts = _line_split(l, self.rtems)
370	if line_parts is not None:
371	src, line, pos, msg = line_parts
372	where = '%s:%s:%s' % (src, line, pos)
373	if where not in archive:
374	archive[where] = 1
375	else:
376	archive[where] += 1
377	if self.build_key not in self.messages[messages]:
378	self.messages[messages][self.build_key] = { }
379	self.messages[messages][self.build_key][where] = msg
380
381	log.output(text)
382
383	class results:
384
385	def __init__(self):
386	self.passes = []
387	self.fails = []
388
389	def _arch_bsp(self, arch, bsp):
390	return '%s/%s' % (arch, bsp)
391
392	def add(self, good, arch, bsp, configure, warnings, error_messages):
393	if good:
394	self.passes += [(arch, bsp, configure, warnings, None)]
395	else:
396	self.fails += [(arch, bsp, configure, warnings, error_messages)]
397
398	def report(self):
399	log.notice('* Passes: %d Failures: %d' %
400	(len(self.passes), len(self.fails)))
401	log.output()
402	log.output('Build Report')
403	log.output(' Passes: %d Failures: %d' %
404	(len(self.passes), len(self.fails)))
405	log.output(' Failures:')
406	if len(self.fails) == 0:
407	log.output(' None')
408	else:
409	max_col = 0
410	for f in self.fails:
411	arch_bsp = self._arch_bsp(f[0], f[1])
412	if len(arch_bsp) > max_col:
413	max_col = len(arch_bsp)
414	for f in self.fails:
415	config_cmd = f[2]
416	config_at = config_cmd.find('configure')
417	if config_at != -1:
418	config_cmd = config_cmd[config_at:]
419	log.output(' %*s:' % (max_col + 2, self._arch_bsp(f[0], f[1])))
420	s1 = ' ' * 6
421	log.output(wrap([s1, config_cmd], lineend = '\\', width = 75))
422	if f[4] is not None:
423	s1 = ' ' * len(s1)
424	for msg in f[4]:
425	log.output(wrap([s1, msg], lineend = '\\'))
426	log.output(' Passes:')
427	if len(self.passes) == 0:
428	log.output(' None')
429	else:
430	max_col = 0
431	for f in self.passes:
432	arch_bsp = self._arch_bsp(f[0], f[1])
433	if len(arch_bsp) > max_col:
434	max_col = len(arch_bsp)
435	for f in self.passes:
436	config_cmd = f[2]
437	config_at = config_cmd.find('configure')
438	if config_at != -1:
439	config_cmd = config_cmd[config_at:]
440	log.output(' %s (%5d):' % (self._arch_bsp(f[0], f[1]), f[3]))
441	log.output(wrap([' ' * 6, config_cmd], lineend = '\\', width = 75))
442
443	class configuration:
444
445	def __init__(self):
446	self.config = configparser.ConfigParser()
447	self.name = None
448	self.ini = None
449	self.archs = { }
450	self.builds_ = { }
451	self.profiles = { }
452	self.configurations = { }
453	self.macro_filter = re.compile('\$\{.+\}')
454
455	def __str__(self):
456	import pprint
457	s = self.name + os.linesep
458	s += 'Archs:' + os.linesep + \
459	pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
460	s += 'Builds:' + os.linesep + \
461	pprint.pformat(self.builds_, indent = 1, width = 80) + os.linesep
462	s += 'Profiles:' + os.linesep + \
463	pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
464	return s
465
466	def _get_item(self, section, label, err = True):
467	try:
468	rec = self.config.get(section, label).replace(os.linesep, ' ')
469	except:
470	if err:
471	raise error.general('config: no "%s" found in "%s"' % (label, section))
472	return None
473	#
474	# On Python 2.7 there is no extended interpolation so add support here.
475	# On Python 3 this should happen automatically and so the findall
476	# should find nothing.
477	#
478	for m in self.macro_filter.findall(rec):
479	if ':' not in m:
480	raise error.general('config: interpolation is ${section:value}: %s' % (m))
481	section_value = m[2:-1].split(':')
482	if len(section_value) != 2:
483	raise error.general('config: interpolation is ${section:value}: %s' % (m))
484	try:
485	ref = self.config.get(section_value[0],
486	section_value[1]).replace(os.linesep, ' ')
487	rec = rec.replace(m, ref)
488	except:
489	pass
490	return rec
491
492	def _get_items(self, section, err = True):
493	try:
494	items = [(name, key.replace(os.linesep, ' ')) \
495	for name, key in self.config.items(section)]
496	return items
497	except:
498	if err:
499	raise error.general('config: section "%s" not found' % (section))
500	return []
501
502	def _comma_list(self, section, label, err = True):
503	items = self._get_item(section, label, err)
504	if items is None:
505	return []
506	return sorted(set([a.strip() for a in items.split(',')]))
507
508	def _get_item_names(self, section, err = True):
509	try:
510	return [item[0] for item in self.config.items(section)]
511	except:
512	if err:
513	raise error.general('config: section "%s" not found' % (section))
514	return []
515
516	def _load(self, name):
517	#
518	# Load all the files.
519	#
520	self.ini = { 'base' : path.dirname(name),
521	'files' : [] }
522	includes = [name]
523	still_loading = True
524	while still_loading:
525	still_loading = False
526	for include in includes:
527	if not path.exists(include):
528	rebased_inc = path.join(self.ini['base'],
529	path.basename(include))
530	if not path.exists(rebased_inc):
531	e = 'config: cannot find configuration: %s' % (include)
532	raise error.general(e)
533	include = rebased_inc
534	if include not in self.ini['files']:
535	try:
536	self.config.read(include)
537	except configparser.ParsingError as ce:
538	raise error.general('config: %s' % (ce))
539	still_loading = True
540	self.ini['files'] += [include]
541	includes = []
542	if still_loading:
543	for section in self.config.sections():
544	includes += self._comma_list(section, 'include', err = False)
545
546	def _build_options(self, build, nesting = 0):
547	if ':' in build:
548	section, name = build.split(':', 1)
549	opts = [self._get_item(section, name)]
550	return opts
551	builds = self.builds_['builds']
552	if build not in builds:
553	raise error.general('build %s not found' % (build))
554	if nesting > 20:
555	raise error.general('nesting build %s' % (build))
556	options = []
557	for option in self.builds_['builds'][build]:
558	if ':' in option:
559	section, name = option.split(':', 1)
560	opts = [self._get_item(section, name)]
561	else:
562	opts = self._options(option, nesting + 1)
563	for opt in opts:
564	if opt not in options:
565	options += [opt]
566	return options
567
568	def load(self, name, build):
569	self._load(name)
570	archs = []
571	self.profiles['profiles'] = self._comma_list('profiles', 'profiles', err = False)
572	if len(self.profiles['profiles']) == 0:
573	self.profiles['profiles'] = ['tier-%d' % (t) for t in range(1,4)]
574	for p in self.profiles['profiles']:
575	profile = {}
576	profile['name'] = p
577	profile['archs'] = self._comma_list(profile['name'], 'archs')
578	archs += profile['archs']
579	for arch in profile['archs']:
580	bsps = 'bsps_%s' % (arch)
581	profile[bsps] = self._comma_list(profile['name'], bsps)
582	self.profiles[profile['name']] = profile
583	for a in set(archs):
584	arch = {}
585	arch['excludes'] = {}
586	for exclude in self._comma_list(a, 'exclude', err = False):
587	arch['excludes'][exclude] = ['all']
588	for i in self._get_items(a, False):
589	if i[0].startswith('exclude-'):
590	exclude = i[0][len('exclude-'):]
591	if exclude not in arch['excludes']:
592	arch['excludes'][exclude] = []
593	arch['excludes'][exclude] += sorted(set([b.strip() for b in i[1].split(',')]))
594	arch['bsps'] = self._comma_list(a, 'bsps', err = False)
595	for b in arch['bsps']:
596	arch[b] = {}
597	arch[b]['bspopts'] = self._comma_list(a, 'bspopts_%s' % (b), err = False)
598	self.archs[a] = arch
599	builds = {}
600	builds['default'] = self._get_item('builds', 'default')
601	if build is None:
602	build = builds['default']
603	builds['config'] = { }
604	for config in self._get_items('config'):
605	builds['config'][config[0]] = config[1]
606	builds['build'] = build
607	builds_ = self._get_item_names('builds')
608	builds['builds'] = {}
609	for build in builds_:
610	build_builds = self._comma_list('builds', build)
611	has_config = False
612	has_build = False
613	for b in build_builds:
614	if ':' in b:
615	if has_build:
616	raise error.general('config and build in build: %s' % (build))
617	has_config = True
618	else:
619	if has_config:
620	raise error.general('config and build in build: %s' % (build))
621	has_build = True
622	builds['builds'][build] = build_builds
623	self.builds_ = builds
624
625	def build(self):
626	return self.builds_['build']
627
628	def builds(self):
629	if self.builds_['build'] in self.builds_['builds']:
630	build = copy.copy(self.builds_['builds'][self.builds_['build']])
631	if ':' in build[0]:
632	return [self.builds_['build']]
633	return build
634	return None
635
636	def build_options(self, build):
637	return ' '.join(self._build_options(build))
638
639	def excludes(self, arch):
640	excludes = self.archs[arch]['excludes'].keys()
641	for exclude in self.archs[arch]['excludes']:
642	if 'all' not in self.archs[arch]['excludes'][exclude]:
643	excludes.remove(exclude)
644	return sorted(excludes)
645
646	def archs(self):
647	return sorted(self.archs.keys())
648
649	def arch_present(self, arch):
650	return arch in self.archs
651
652	def arch_bsps(self, arch):
653	return sorted(self.archs[arch]['bsps'])
654
655	def bsp_present(self, arch, bsp):
656	return bsp in self.archs[arch]['bsps']
657
658	def bsp_excludes(self, arch, bsp):
659	excludes = self.archs[arch]['excludes'].keys()
660	for exclude in self.archs[arch]['excludes']:
661	if 'all' not in self.archs[arch]['excludes'][exclude] and \
662	bsp not in self.archs[arch]['excludes'][exclude]:
663	excludes.remove(exclude)
664	return sorted(excludes)
665
666	def bspopts(self, arch, bsp):
667	return self.archs[arch][bsp]['bspopts']
668
669	def profile_present(self, profile):
670	return profile in self.profiles
671
672	def profile_archs(self, profile):
673	return self.profiles[profile]['archs']
674
675	def profile_arch_bsps(self, profile, arch):
676	return self.profiles[profile]['bsps_%s' % (arch)]
677
678	def report(self, profiles = True, builds = True, architectures = True):
679	width = 70
680	cols_1 = [width]
681	cols_2 = [10, width - 10]
682	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
683	s1 = ' File(s)'
684	for f in self.ini['files']:
685	colon = ':'
686	for l in textwrap.wrap(f, width = cols_2[1] - 3):
687	s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1)
688	colon = ' '
689	s1 = ' ' * len(s1)
690	s += textbox.line(cols_1, marker = '+', indent = 1)
691	s += os.linesep
692	if profiles:
693	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
694	profiles = sorted(self.profiles['profiles'])
695	archs = []
696	bsps = []
697	for profile in profiles:
698	archs += self.profiles[profile]['archs']
699	for arch in sorted(self.profiles[profile]['archs']):
700	bsps += self.profiles[profile]['bsps_%s' % (arch)]
701	archs = len(set(archs))
702	bsps = len(set(bsps))
703	s += textbox.row(cols_1,
704	[' Profiles : %d (archs:%d, bsps:%d)' % \
705	(len(profiles), archs, bsps)],
706	indent = 1)
707	for profile in profiles:
708	textbox.row(cols_2,
709	[profile, self.profiles[profile]['name']],
710	indent = 1)
711	s += textbox.line(cols_1, marker = '+', indent = 1)
712	for profile in profiles:
713	s += textbox.row(cols_1, [' %s' % (profile)], indent = 1)
714	profile = self.profiles[profile]
715	archs = sorted(profile['archs'])
716	for arch in archs:
717	arch_bsps = ', '.join(profile['bsps_%s' % (arch)])
718	if len(arch_bsps) > 0:
719	s += textbox.line(cols_2, marker = '+', indent = 1)
720	s1 = ' ' + arch
721	for l in textwrap.wrap(arch_bsps,
722	width = cols_2[1] - 3):
723	s += textbox.row(cols_2, [s1, ' ' + l], indent = 1)
724	s1 = ' ' * len(s1)
725	s += textbox.line(cols_2, marker = '+', indent = 1)
726	s += os.linesep
727	if builds:
728	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
729	s += textbox.row(cols_1,
730	[' Builds: %s (default)' % (self.builds_['default'])],
731	indent = 1)
732	builds = self.builds_['builds']
733	bsize = 0
734	for build in builds:
735	if len(build) > bsize:
736	bsize = len(build)
737	cols_b = [bsize + 2, width - bsize - 2]
738	s += textbox.line(cols_b, marker = '+', indent = 1)
739	for build in builds:
740	s1 = ' ' + build
741	for l in textwrap.wrap(', '.join(builds[build]),
742	width = cols_b[1] - 3):
743	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
744	s1 = ' ' * len(s1)
745	s += textbox.line(cols_b, marker = '+', indent = 1)
746	configs = self.builds_['config']
747	s += textbox.row(cols_1,
748	[' Configure Options: %d' % (len(configs))],
749	indent = 1)
750	csize = 0
751	for config in configs:
752	if len(config) > csize:
753	csize = len(config)
754	cols_c = [csize + 3, width - csize - 3]
755	s += textbox.line(cols_c, marker = '+', indent = 1)
756	for config in configs:
757	s1 = ' ' + config
758	for l in textwrap.wrap(configs[config], width = cols_c[1] - 3):
759	s += textbox.row(cols_c, [s1, ' ' + l], indent = 1)
760	s1 = ' ' * len(s1)
761	s += textbox.line(cols_c, marker = '+', indent = 1)
762	s += os.linesep
763	if architectures:
764	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
765	archs = sorted(self.archs.keys())
766	bsps = 0
767	asize = 0
768	for arch in archs:
769	if len(arch) > asize:
770	asize = len(arch)
771	bsps += len(self.archs[arch]['bsps'])
772	s += textbox.row(cols_1,
773	[' Architectures : %d (bsps: %d)' % (len(archs), bsps)],
774	indent = 1)
775	cols_a = [asize + 2, width - asize - 2]
776	s += textbox.line(cols_a, marker = '+', indent = 1)
777	for arch in archs:
778	s += textbox.row(cols_a,
779	[' ' + arch, ' %d' % (len(self.archs[arch]['bsps']))],
780	indent = 1)
781	s += textbox.line(cols_a, marker = '+', indent = 1)
782	for archn in archs:
783	arch = self.archs[archn]
784	if len(arch['bsps']) > 0:
785	bsize = 0
786	for bsp in arch['bsps']:
787	if len(bsp) > bsize:
788	bsize = len(bsp)
789	cols_b = [bsize + 3, width - bsize - 3]
790	s += textbox.row(cols_1, [' ' + archn + ':'], indent = 1)
791	s += textbox.line(cols_b, marker = '+', indent = 1)
792	for bsp in arch['bsps']:
793	s1 = ' ' + bsp
794	bspopts = ' '.join(arch[bsp]['bspopts'])
795	if len(bspopts):
796	for l in textwrap.wrap('bopt: ' + bspopts,
797	width = cols_b[1] - 3):
798	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
799	s1 = ' ' * len(s1)
800	excludes = []
801	for exclude in arch['excludes']:
802	if 'all' in arch['excludes'][exclude] or \
803	bsp in arch['excludes'][exclude]:
804	excludes += [exclude]
805	excludes = ', '.join(excludes)
806	if len(excludes):
807	for l in textwrap.wrap('ex: ' + excludes,
808	width = cols_b[1] - 3):
809	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
810	s1 = ' ' * len(s1)
811	if len(bspopts) == 0 and len(excludes) == 0:
812	s += textbox.row(cols_b, [s1, ' '], indent = 1)
813	s += textbox.line(cols_b, marker = '+', indent = 1)
814	s += os.linesep
815	return s
816
817	class build:
818
819	def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
820	self.config = config
821	self.build_dir = build_dir
822	self.rtems_version = version
823	self.prefix = prefix
824	self.tools = tools
825	self.rtems = rtems
826	self.options = options
827	self.errors = { 'configure': 0,
828	'build': 0,
829	'tests': 0,
830	'fails': []}
831	self.counts = { 'h' : 0,
832	'exes' : 0,
833	'objs' : 0,
834	'libs' : 0 }
835	self.warnings_errors = warnings_errors(rtems)
836	self.results = results()
837	if not path.exists(path.join(rtems, 'configure')) or \
838	not path.exists(path.join(rtems, 'Makefile.in')) or \
839	not path.exists(path.join(rtems, 'cpukit')):
840	raise error.general('RTEMS source path does not look like RTEMS')
841
842	def _error_str(self):
843	return 'Status: configure:%d build:%d' % \
844	(self.errors['configure'], self.errors['build'])
845
846	def _path(self, arch, bsp):
847	return path.join(self.build_dir, arch, bsp)
848
849	def _archs(self, build_data):
850	return sorted(build_data.keys())
851
852	def _bsps(self, arch):
853	return self.config.arch_bsps(arch)
854
855	def _build(self):
856	return self.config.build()
857
858	def _builds(self, arch, bsp):
859	builds = self.config.builds()
860	if builds is None:
861	return None
862	excludes = set(self.config.excludes(arch) +
863	self.config.bsp_excludes(arch, bsp))
864	remove = []
865	for e in excludes:
866	remove += [b for b in builds if e in b]
867	for b in remove:
868	builds.remove(b)
869	return builds
870
871	def _arch_bsp_dir_make(self, arch, bsp):
872	if not path.exists(self._path(arch, bsp)):
873	path.mkdir(self._path(arch, bsp))
874
875	def _arch_bsp_dir_clean(self, arch, bsp):
876	if path.exists(self._path(arch, bsp)):
877	path.removeall(self._path(arch, bsp))
878
879	def _config_command(self, commands, arch, bsp):
880	if type(commands) is not list:
881	commands = [commands]
882	cmd = [path.join(self.rtems, 'configure')]
883	commands += self.config.bspopts(arch, bsp)
884	for c in commands:
885	c = c.replace('@PREFIX@', self.prefix)
886	c = c.replace('@RTEMS_VERSION@', self.rtems_version)
887	c = c.replace('@ARCH@', arch)
888	c = c.replace('@BSP@', bsp)
889	cmd += [c]
890	return ' '.join(cmd)
891
892	def _build_set(self, builds):
893	build_set = { }
894	for build in builds:
895	build_set[build] = self.config.build_options(build)
896	return build_set
897
898	def _build_dir(self, arch, bsp, build):
899	return path.join(self._path(arch, bsp), build)
900
901	def _count_files(self, arch, bsp, build):
902	counts = { 'h' : 0,
903	'exes' : 0,
904	'objs' : 0,
905	'libs' : 0 }
906	for root, dirs, files in os.walk(self._build_dir(arch, bsp, build)):
907	for file in files:
908	if file.endswith('.exe'):
909	counts['exes'] += 1
910	elif file.endswith('.o'):
911	counts['objs'] += 1
912	elif file.endswith('.a'):
913	counts['libs'] += 1
914	elif file.endswith('.h'):
915	counts['h'] += 1
916	for f in self.counts:
917	if f in counts:
918	self.counts[f] += counts[f]
919	return counts
920
921	def _have_failures(self, fails):
922	return len(fails) != 0
923
924	def _warnings_report(self):
925	if self.options['warnings-report'] is not None:
926	with open(self.options['warnings-report'], 'w') as f:
927	f.write(title() + os.linesep)
928	f.write(os.linesep)
929	f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'),
930	os.linesep))
931	f.write(os.linesep)
932	f.write(command_line() + os.linesep)
933	f.write(self.warnings_errors.report())
934
935	def _finished(self):
936	log.notice('+ warnings:%d exes:%d objs:%d libs:%d' % \
937	(self.warnings_errors.get_warning_count(), self.counts['exes'],
938	self.counts['objs'], self.counts['libs']))
939	log.output()
940	log.output('Warnings:')
941	log.output(self.warnings_errors.report())
942	log.output()
943	log.notice('Failures:')
944	log.notice(self.failures_report(self.errors['fails']))
945	self._warnings_report()
946
947	def failures_report(self, fails):
948	if not self._have_failures(fails):
949	return ' No failure(s)'
950	absize = 0
951	bsize = 0
952	ssize = 0
953	for f in fails:
954	arch_bsp = '%s/%s' % (f[1], f[2])
955	if len(arch_bsp) > absize:
956	absize = len(arch_bsp)
957	if len(f[3]) > bsize:
958	bsize = len(f[3])
959	if len(f[0]) > ssize:
960	ssize = len(f[0])
961	fc = 1
962	s = ''
963	for f in fails:
964	fcl = ' %3d' % (fc)
965	arch_bsp = '%s/%s' % (f[1], f[2])
966	state = f[0]
967	s += '%s %-s %-s %-*s:%s' % \
968	(fcl, bsize, f[3], absize, arch_bsp, ssize, state, os.linesep)
969	s1 = ' ' * 6
970	s += wrap((s1, 'configure: ' + f[4]), lineend = '\\', width = 75)
971	for e in self.warnings_errors.get_error_messages(f[1], f[2], f[3]):
972	s += wrap([s1, 'error: ' + e])
973	fc += 1
974	return s
975
976	def build_arch_bsp(self, arch, bsp):
977	if not self.config.bsp_present(arch, bsp):
978	raise error.general('BSP not found: %s/%s' % (arch, bsp))
979	log.output('-' * 70)
980	log.notice('] BSP: %s/%s' % (arch, bsp))
981	log.notice('. Creating: %s' % (self._path(arch, bsp)))
982	self._arch_bsp_dir_clean(arch, bsp)
983	self._arch_bsp_dir_make(arch, bsp)
984	builds = self._builds(arch, bsp)
985	if builds is None:
986	raise error.general('build not found: %s' % (self._build()))
987	build_set = self._build_set(builds)
988	bsp_start = datetime.datetime.now()
989	env_path = os.environ['PATH']
990	os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
991	os.pathsep + os.environ['PATH']
992	fails = []
993	for bs in sorted(build_set.keys()):
994	self.warnings_errors.set_build(arch, bsp, bs)
995	start = datetime.datetime.now()
996	log.output('- ' * 35)
997	log.notice('. Configuring: %s' % (bs))
998	try:
999	warnings = self.warnings_errors.get_warning_count()
1000	result = '+ Pass'
1001	bpath = self._build_dir(arch, bsp, bs)
1002	good = True
1003	error_messages = None
1004	path.mkdir(bpath)
1005	config_cmd = self._config_command(build_set[bs], arch, bsp)
1006	cmd = config_cmd
1007	e = execute.capture_execution(log = self.warnings_errors)
1008	log.output(wrap(('run: ', cmd), lineend = '\\'))
1009	if self.options['dry-run']:
1010	exit_code = 0
1011	else:
1012	exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
1013	if exit_code != 0:
1014	result = '- FAIL'
1015	failure = ('configure', arch, bsp, bs, config_cmd)
1016	fails += [failure]
1017	self.errors['configure'] += 1
1018	self.errors['fails'] += [failure]
1019	log.notice('- Configure failed: %s' % (bs))
1020	log.output('cmd failed: %s' % (cmd))
1021	good = False
1022	else:
1023	log.notice('. Building: %s' % (bs))
1024	cmd = 'make'
1025	if 'jobs' in self.options:
1026	cmd += ' -j %s' % (self.options['jobs'])
1027	log.output('run: ' + cmd)
1028	if self.options['dry-run']:
1029	exit_code = 0
1030	else:
1031	exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
1032	if exit_code != 0:
1033	error_messages = self.warnings_errors.get_error_messages()
1034	result = '- FAIL'
1035	failure = ('build', arch, bsp, bs, config_cmd, error_messages)
1036	fails += [failure]
1037	self.errors['build'] += 1
1038	self.errors['fails'] += [failure]
1039	log.notice('- FAIL: %s: %s' % (bs, self._error_str()))
1040	log.output('cmd failed: %s' % (cmd))
1041	good = False
1042	files = self._count_files(arch, bsp, bs)
1043	log.notice('%s: %s: warnings:%d exes:%d objs:%s libs:%d' % \
1044	(result, bs,
1045	self.warnings_errors.get_warning_count() - warnings,
1046	files['exes'], files['objs'], files['libs']))
1047	log.notice(' %s' % (self._error_str()))
1048	self.results.add(good, arch, bsp, config_cmd,
1049	self.warnings_errors.get_warning_count() - warnings,
1050	error_messages)
1051	if not good and self.options['stop-on-error']:
1052	raise error.general('Configuring %s failed' % (bs))
1053	finally:
1054	end = datetime.datetime.now()
1055	if not self.options['no-clean']:
1056	log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs)))
1057	path.removeall(self._build_dir(arch, bsp, bs))
1058	log.notice('^ Time %s' % (str(end - start)))
1059	self.warnings_errors.clear_build()
1060	bsp_end = datetime.datetime.now()
1061	log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start)))
1062	log.output('Failure Report:')
1063	log.output(self.failures_report(fails))
1064	os.environ['PATH'] = env_path
1065
1066	def build_bsp(self, arch, bsp):
1067	self.build_arch_bsp(arch, bsp)
1068	self._finished()
1069
1070	def build_arch(self, arch):
1071	start = datetime.datetime.now()
1072	log.output('=' * 70)
1073	log.notice(']] Architecture: %s' % (arch))
1074	if not self.config.arch_present(arch):
1075	raise error.general('Architecture not found: %s' % (arch))
1076	for bsp in self._bsps(arch):
1077	self.build_arch_bsp(arch, bsp)
1078	end = datetime.datetime.now()
1079	log.notice('^ Architecture Time %s' % (str(end - start)))
1080	self._finished()
1081
1082	def build(self):
1083	for arch in self.config.archs():
1084	self.build_arch(arch)
1085	log.notice('^ Profile Time %s' % (str(end - start)))
1086	self._finished()
1087
1088	def build_profile(self, profile):
1089	if not self.config.profile_present(profile):
1090	raise error.general('Profile not found: %s' % (profile))
1091	start = datetime.datetime.now()
1092	log.notice(']] Profile: %s' % (profile))
1093	for arch in self.config.profile_archs(profile):
1094	for bsp in self.config.profile_arch_bsps(profile, arch):
1095	self.build_arch_bsp(arch, bsp)
1096	end = datetime.datetime.now()
1097	log.notice('^ Profile Time %s' % (str(end - start)))
1098	self._finished()
1099
1100	def run_args(args):
1101	b = None
1102	ec = 0
1103	try:
1104	#
1105	# On Windows MSYS2 prepends a path to itself to the environment
1106	# path. This means the RTEMS specific automake is not found and which
1107	# breaks the bootstrap. We need to remove the prepended path. Also
1108	# remove any ACLOCAL paths from the environment.
1109	#
1110	if os.name == 'nt':
1111	cspath = os.environ['PATH'].split(os.pathsep)
1112	if 'msys' in cspath[0] and cspath[0].endswith('bin'):
1113	os.environ['PATH'] = os.pathsep.join(cspath[1:])
1114
1115	top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
1116	prefix = '/opt/rtems/%s' % (rtems_version())
1117	tools = prefix
1118	build_dir = 'bsp-builds'
1119	logf = 'bsp-build-%s.txt' % \
1120	(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
1121	config_file = path.join(top, 'share', 'rtems', 'tester',
1122	'rtems', 'rtems-bsps.ini')
1123	if not path.exists(config_file):
1124	config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')
1125
1126	argsp = argparse.ArgumentParser()
1127	argsp.add_argument('--prefix', help = 'Prefix to build the BSP.',
1128	type = str)
1129	argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.',
1130	type = str)
1131	argsp.add_argument('--rtems', help = 'The RTEMS source tree.',
1132	type = str)
1133	argsp.add_argument('--config-report', help = 'Report the configuration.',
1134	action = 'store_true')
1135	argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
1136	type = str, default = None)
1137	argsp.add_argument('--build-path', help = 'Path to build in.',
1138	type = str)
1139	argsp.add_argument('--log', help = 'Log file.', type = str)
1140	argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
1141	action = 'store_true')
1142	argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
1143	action = 'store_true')
1144	argsp.add_argument('--profiles', help = 'Build the listed profiles.',
1145	type = str, default = 'tier-1')
1146	argsp.add_argument('--build', help = 'Build name to build.',
1147	type = str, default='all')
1148	argsp.add_argument('--arch', help = 'Build the specific architecture.',
1149	type = str)
1150	argsp.add_argument('--bsp', help = 'Build the specific BSP.',
1151	type = str)
1152	argsp.add_argument('--jobs', help = 'Number of jobs to run.',
1153	type = int, default = host.cpus())
1154	argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
1155	action = 'store_true')
1156
1157	opts = argsp.parse_args(args[1:])
1158	if opts.log is not None:
1159	logf = opts.log
1160	log.default = log.log([logf])
1161	log.notice(title())
1162	log.output(command_line())
1163
1164	config = configuration()
1165	config.load(config_file, opts.build)
1166
1167	if opts.config_report:
1168	log.notice('Configuration Report:')
1169	log.notice(config.report())
1170	sys.exit(0)
1171
1172	if opts.rtems is None:
1173	raise error.general('No RTEMS source provided on the command line')
1174	if opts.prefix is not None:
1175	prefix = path.shell(opts.prefix)
1176	if opts.rtems_tools is not None:
1177	tools = path.shell(opts.rtems_tools)
1178	if opts.build_path is not None:
1179	build_dir = path.shell(opts.build_path)
1180	if opts.bsp is not None and opts.arch is None:
1181	raise error.general('BSP provided but no architecture')
1182
1183	options = { 'stop-on-error' : opts.stop_on_error,
1184	'no-clean' : opts.no_clean,
1185	'dry-run' : opts.dry_run,
1186	'jobs' : opts.jobs,
1187	'warnings-report' : opts.warnings_report }
1188
1189	b = build(config, rtems_version(), prefix, tools,
1190	path.shell(opts.rtems), build_dir, options)
1191	if opts.arch is not None:
1192	if opts.bsp is not None:
1193	b.build_bsp(opts.arch, opts.bsp)
1194	else:
1195	b.build_arch(opts.arch)
1196	else:
1197	for profile in opts.profiles.split(','):
1198	b.build_profile(profile.strip())
1199
1200	except error.general as gerr:
1201	print(gerr)
1202	print('BSP Build FAILED', file = sys.stderr)
1203	ec = 1
1204	except error.internal as ierr:
1205	print(ierr)
1206	print('BSP Build FAILED', file = sys.stderr)
1207	ec = 1
1208	except error.exit as eerr:
1209	pass
1210	except KeyboardInterrupt:
1211	log.notice('abort: user terminated')
1212	ec = 1
1213	if b is not None:
1214	b.results.report()
1215	sys.exit(ec)
1216
1217	def run():
1218	run_args(sys.argv)
1219
1220	if __name__ == "__main__":
1221	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: