Context Navigation

source: rtems-tools/tester/rt/check.py @ c2df65b

5

Last change on this file since c2df65b was c2df65b, checked in by Chris Johns <chrisj@…>, on 04/26/17 at 10:01:23
rtems-bsp-builder: Fix access when there are no messages.
Property mode set to `100755`
File size: 46.5 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2016-2017 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Redistribution and use in source and binary forms, with or without
9	# modification, are permitted provided that the following conditions are met:
10	#
11	# 1. Redistributions of source code must retain the above copyright notice,
12	# this list of conditions and the following disclaimer.
13	#
14	# 2. Redistributions in binary form must reproduce the above copyright notice,
15	# this list of conditions and the following disclaimer in the documentation
16	# and/or other materials provided with the distribution.
17	#
18	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21	# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22	# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23	# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24	# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25	# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26	# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27	# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28	# POSSIBILITY OF SUCH DAMAGE.
29	#
30
31	from __future__ import print_function
32
33	import argparse
34	import datetime
35	import operator
36	import os
37	import re
38	import sys
39	import textwrap
40
41	import pprint
42
43	try:
44	import configparser
45	except:
46	import ConfigParser as configparser
47
48	from rtemstoolkit import execute
49	from rtemstoolkit import error
50	from rtemstoolkit import host
51	from rtemstoolkit import log
52	from rtemstoolkit import path
53	from rtemstoolkit import textbox
54	from rtemstoolkit import version
55
56	def rtems_version():
57	return version.version()
58
59	def wrap(line, lineend = '', indent = 0):
60	if type(line) is tuple or type(line) is list:
61	if len(line) >= 2:
62	s1 = line[0]
63	else:
64	s1 = ''
65	s2 = line[1:]
66	elif type(line) is str:
67	s1 = ''
68	s2 = [line]
69	else:
70	raise error.internal('line is not a tuple, list or string')
71	s = ''
72	first = True
73	for ss in s2:
74	if type(ss) is not str and type(ss) is not unicode:
75	raise error.internal('text needs to be a string')
76	for l in textwrap.wrap(ss):
77	s += '%s%s%s%s%s' % (' ' * indent, s1, l, lineend, os.linesep)
78	if first and len(s1) > 0:
79	s1 = ' ' * len(s1)
80	if lineend != '':
81	s = s[:0 - len(os.linesep) - 1] + os.linesep
82	return s
83
84	def title():
85	return 'RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str())
86
87	def command_line():
88	return wrap(('command: ', ' '.join(sys.argv)), lineend = '\\')
89
90	class warnings_errors:
91
92	def __init__(self, rtems):
93	self.rtems = path.host(rtems)
94	self.reset()
95	self.groups = { 'groups' : ['Shared', 'BSP', 'Network', 'Tests',
96	'LibCPU', 'CPU Kit'],
97	'exclude' : '.Makefile.',
98	'CPU Kit' : '.cpukit/.',
99	'Network' : '.libnetworking/.',
100	'Tests' : '.testsuites/.',
101	'BSP' : '.libbsp/.',
102	'LibCPU' : '.libcpu/.',
103	'Shared' : '.shared/.' }
104	self.arch = None
105	self.bsp = None
106	self.build = None
107
108	def _opts(self, arch = None, bsp = None, build = None):
109	if arch is None:
110	arch = self.arch
111	if bsp is None:
112	bsp = self.bsp
113	if build is None:
114	build = self.build
115	return arch, bsp, build
116
117	def _key(self, arch, bsp, build):
118	arch, bsp, build = self._opts(arch, bsp, build)
119	return '%s/%s-%s' % (arch, bsp, build)
120
121	def _get_warnings(self, arch = None, bsp = None, build = None):
122	arch, bsp, build = self._opts(arch = arch, bsp = bsp, build = build)
123	if arch is None:
124	arch = '.*'
125	if bsp is None:
126	bsp = '.*'
127	if build is None:
128	build = '.*'
129	selector = re.compile('^%s/%s-%s$' % (arch, bsp, build))
130	warnings = [w for w in self.warnings if selector.match(w)]
131	return sorted(warnings)
132
133	def _total(self, archive):
134	total = 0
135	for a in archive:
136	total += archive[a]
137	return total
138
139	def _analyze(self, warnings, exclude):
140	def _group(data, category, name, warning, count, groups, group_regx):
141	if 'groups' not in data:
142	data['groups'] = { }
143	if category not in data['groups']:
144	data['groups'][category] = { 'totals' : { } }
145	if name not in data['groups'][category]:
146	data['groups'][category][name] = { }
147	for group in groups:
148	if group not in data['groups'][category]['totals']:
149	data['groups'][category]['totals'][group] = 0
150	if group not in data['groups'][category][name]:
151	data['groups'][category][name][group] = 0
152	if group_regx[group].match(warning):
153	data['groups'][category][name][group] += count
154	data['groups'][category]['totals'][group] += count
155	break
156
157	def _update(data, category, name, warning, count, groups, group_regx):
158	if category not in data:
159	data[category] = { }
160	if name not in data[category]:
161	data[category][name] = { }
162	if warning not in data[category][name]:
163	data[category][name][warning] = 0
164	data[category][name][warning] += count
165	_group(data, category, name, w, count, groups, group_regx)
166
167	data = { }
168	group_regx = { }
169	for group in self.groups['groups']:
170	group_regx[group] = re.compile(self.groups[group])
171	exclude_regx = re.compile(exclude)
172	for warning in warnings:
173	arch = warning.split('/', 1)[0]
174	arch_bsp = warning.split('-', 1)[0]
175	build = warning.split('-', 1)[1]
176	for w in self.warnings[warning]:
177	if not exclude_regx.match(w):
178	count = self.warnings[warning][w]
179	_update(data, 'arch', arch, w, count,
180	self.groups['groups'], group_regx)
181	_update(data, 'arch_bsp', arch_bsp, w, count,
182	self.groups['groups'], group_regx)
183	_update(data, 'build', build, w, count,
184	self.groups['groups'], group_regx)
185	for category in ['arch', 'arch_bsp', 'build']:
186	common = {}
187	for name in data[category]:
188	for w in data[category][name]:
189	if w not in common:
190	for other in [n for n in data[category] if n != name]:
191	if w in data[category][other]:
192	common[w] = data[category][name][w]
193	_group(data, category, 'common', w, common[w],
194	self.groups['groups'], group_regx)
195	data[category]['common'] = common
196	return data
197
198	def _report_category(self, label, warnings, group_counts):
199	width = 70
200	cols_1 = [width]
201	cols_2 = [8, width - 8]
202	cols_4 = textbox.even_columns(4, width)
203	cols_2_4 = textbox.merge_columns([cols_2, cols_4])
204	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
205	s += textbox.row(cols_1, [' ' + label], indent = 1)
206	s += textbox.line(cols_1, marker = '+', indent = 1)
207	builds = ['common'] + sorted([b for b in warnings if b != 'common'])
208	common = warnings['common']
209	for build in builds:
210	build_warnings = warnings[build]
211	if build is not 'common':
212	build_warnings = [w for w in build_warnings if w not in common]
213	s += textbox.row(cols_1,
214	[' %s : %d warning(s)' % (build,
215	len(build_warnings))],
216	indent = 1)
217	if len(build_warnings) == 0:
218	s += textbox.line(cols_1, marker = '+', indent = 1)
219	else:
220	s += textbox.line(cols_4, marker = '+', indent = 1)
221	if build not in group_counts:
222	gs = [0 for group in self.groups['groups']]
223	else:
224	gs = []
225	for g in range(0, len(self.groups['groups'])):
226	group = self.groups['groups'][g]
227	gs += ['%*s' % (cols_4[g % 4] - 2,
228	'%s : %4d' % \
229	(group,
230	group_counts[build][group]))]
231	for row in range(0, len(self.groups['groups']), 4):
232	if row + 4 > len(self.groups['groups']):
233	d = gs[row:] + \
234	['' for r in range(row,
235	len(self.groups['groups']))]
236	else:
237	d = gs[row:+4]
238	s += textbox.row(cols_4, d, indent = 1)
239	s += textbox.line(cols_2_4, marker = '+', indent = 1)
240	vw = sorted([(w, warnings[build][w]) for w in build_warnings],
241	key = operator.itemgetter(1),
242	reverse = True)
243	for w in vw:
244	c1 = '%6d' % w[1]
245	for l in textwrap.wrap(' ' + w[0]):
246	s += textbox.row(cols_2, [c1, l], indent = 1)
247	c1 = ' ' * 6
248	s += textbox.line(cols_2, marker = '+', indent = 1)
249	return s
250
251	def _report_warning_map(self):
252	builds = self.messages['warnings']
253	width = 70
254	cols_1 = [width]
255	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
256	s += textbox.row(cols_1, [' Warning Map'], indent = 1)
257	s += textbox.line(cols_1, marker = '+', indent = 1)
258	for build in builds:
259	messages = builds[build]
260	s += textbox.row(cols_1, [' %s : %d' % (build, len(messages))], indent = 1)
261	s += textbox.line(cols_1, marker = '+', indent = 1)
262	for msg in messages:
263	for l in textwrap.wrap(msg, width = width - 3):
264	s += textbox.row(cols_1, [' ' + l], indent = 1)
265	for l in textwrap.wrap(messages[msg], width = width - 3 - 4):
266	s += textbox.row(cols_1, [' ' + l], indent = 1)
267	s += textbox.line(cols_1, marker = '+', indent = 1)
268	return s
269
270	def report(self):
271	arch, bsp, build = self._opts()
272	warnings = self._get_warnings(arch, bsp, build)
273	total = 0
274	for build in warnings:
275	total += self._total(self.warnings[build])
276	if total == 0:
277	s = ' No warnings'
278	else:
279	data = self._analyze(warnings, self.groups['exclude'])
280	s = self._report_category('By Architecture (total : %d)' % (total),
281	data['arch'], data['groups']['arch'])
282	s += os.linesep
283	s += self._report_category('By BSP (total : %d)' % (total),
284	data['arch_bsp'], data['groups']['arch_bsp'])
285	s += os.linesep
286	s += self._report_category('By Build (total : %d)' % (total),
287	data['build'], data['groups']['build'])
288	s += os.linesep
289	s += self._report_warning_map()
290	s += os.linesep
291
292	return s
293
294	def set_build(self, arch, bsp, build):
295	self.arch = arch
296	self.bsp = bsp
297	self.build = build
298	self.build_key = '%s/%s-%s' % (arch, bsp, build)
299	if self.build_key not in self.warnings:
300	self.warnings[self.build_key] = {}
301	if self.build_key not in self.errors:
302	self.errors[self.build_key] = {}
303
304	def clear_build(self):
305	self.arch = None
306	self.bsp = None
307	self.build = None
308	self.build_key = None
309
310	def get_warning_count(self):
311	return self.warning_count
312
313	def get_error_count(self):
314	return self.error_count
315
316	def reset(self):
317	self.warnings = { }
318	self.warning_count = 0
319	self.errors = { }
320	self.error_count = 0
321	self.messages = { 'warnings' : { }, 'errors' : { } }
322
323	def get_warning_messages(self, arch = None, bsp = None, build = None):
324	key = self._key(arch, bsp, build)
325	if key not in self.messages['warnings']:
326	return []
327	messages = self.messages['warnings'][key]
328	return ['%s %s' % (m, messages[m]) for m in messages]
329
330	def get_error_messages(self, arch = None, bsp = None, build = None):
331	key = self._key(arch, bsp, build)
332	if key not in self.messages['errors']:
333	return []
334	messages = self.messages['errors'][key]
335	return ['%s %s' % (m, messages[m]) for m in messages]
336
337	def output(self, text):
338	def _line_split(line, source_base):
339	ls = line.split(' ', 1)
340	fname = ls[0].split(':')
341	#
342	# Ignore compiler option warnings.
343	#
344	if len(fname) < 4:
345	return None
346	p = path.abspath(fname[0])
347	p = p.replace(source_base, '')
348	if path.isabspath(p):
349	p = p[1:]
350	return p, fname[1], fname[2], ls[1]
351
352	if self.build_key is not None and \
353	(' warning:' in text or ' error:' in text):
354	for l in text.splitlines():
355	if ' warning:' in l:
356	self.warning_count += 1
357	archive = self.warnings[self.build_key]
358	messages = 'warnings'
359	elif ' error:' in l:
360	self.error_count += 1
361	archive = self.errors[self.build_key]
362	messages = 'errors'
363	else:
364	continue
365	line_parts = _line_split(l, self.rtems)
366	if line_parts is not None:
367	src, line, pos, msg = line_parts
368	where = '%s:%s:%s' % (src, line, pos)
369	if where not in archive:
370	archive[where] = 1
371	else:
372	archive[where] += 1
373	if self.build_key not in self.messages[messages]:
374	self.messages[messages][self.build_key] = { }
375	self.messages[messages][self.build_key][where] = msg
376
377	log.output(text)
378
379	class results:
380
381	def __init__(self):
382	self.passes = []
383	self.fails = []
384
385	def _arch_bsp(self, arch, bsp):
386	return '%s/%s' % (arch, bsp)
387
388	def add(self, good, arch, bsp, configure, warnings, error_messages):
389	if good:
390	self.passes += [(arch, bsp, configure, warnings, None)]
391	else:
392	self.fails += [(arch, bsp, configure, warnings, error_messages)]
393
394	def report(self):
395	log.notice('* Passes: %d Failures: %d' %
396	(len(self.passes), len(self.fails)))
397	log.output()
398	log.output('Build Report')
399	log.output(' Passes: %d Failures: %d' %
400	(len(self.passes), len(self.fails)))
401	log.output(' Failures:')
402	if len(self.fails) == 0:
403	log.output(' None')
404	else:
405	max_col = 0
406	for f in self.fails:
407	arch_bsp = self._arch_bsp(f[0], f[1])
408	if len(arch_bsp) > max_col:
409	max_col = len(arch_bsp)
410	for f in self.fails:
411	config_cmd = f[2]
412	config_at = config_cmd.find('configure')
413	if config_at != -1:
414	config_cmd = config_cmd[config_at:]
415	s1 = ' %*s: ' % (max_col + 2, self._arch_bsp(f[0], f[1]))
416	log.output(wrap([s1, config_cmd], lineend = '\\'))
417	if f[4] is not None:
418	s1 = ' ' * len(s1)
419	for msg in f[4]:
420	log.output(wrap([s1, msg], lineend = '\\'))
421	log.output(' Passes:')
422	if len(self.passes) == 0:
423	log.output(' None')
424	else:
425	max_col = 0
426	for f in self.passes:
427	arch_bsp = self._arch_bsp(f[0], f[1])
428	if len(arch_bsp) > max_col:
429	max_col = len(arch_bsp)
430	for f in self.passes:
431	config_cmd = f[2]
432	config_at = config_cmd.find('configure')
433	if config_at != -1:
434	config_cmd = config_cmd[config_at:]
435	log.output(wrap((' %*s: %5d ' % (max_col + 2,
436	self._arch_bsp(f[0], f[1]),
437	f[3]),
438	config_cmd),
439	lineend = '\\'))
440
441	class configuration:
442
443	def __init__(self):
444	self.config = configparser.ConfigParser()
445	self.name = None
446	self.archs = { }
447	self.builds_ = { }
448	self.profiles = { }
449	self.configurations = { }
450
451	def __str__(self):
452	import pprint
453	s = self.name + os.linesep
454	s += 'Archs:' + os.linesep + \
455	pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
456	s += 'Builds:' + os.linesep + \
457	pprint.pformat(self.builds_, indent = 1, width = 80) + os.linesep
458	s += 'Profiles:' + os.linesep + \
459	pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
460	return s
461
462	def _get_item(self, section, label, err = True):
463	try:
464	rec = self.config.get(section, label).replace(os.linesep, ' ')
465	return rec
466	except:
467	if err:
468	raise error.general('config: no "%s" found in "%s"' % (label, section))
469	return None
470
471	def _get_items(self, section, err = True):
472	try:
473	items = [(name, key.replace(os.linesep, ' ')) for name, key in self.config.items(section)]
474	return items
475	except:
476	if err:
477	raise error.general('config: section "%s" not found' % (section))
478	return []
479
480	def _comma_list(self, section, label, error = True):
481	items = self._get_item(section, label, error)
482	if items is None:
483	return []
484	return sorted(set([a.strip() for a in items.split(',')]))
485
486	def _get_item_names(self, section, err = True):
487	try:
488	return [item[0] for item in self.config.items(section)]
489	except:
490	if err:
491	raise error.general('config: section "%s" not found' % (section))
492	return []
493
494	def _build_options(self, build, nesting = 0):
495	if ':' in build:
496	section, name = build.split(':', 1)
497	opts = [self._get_item(section, name)]
498	return opts
499	builds = self.builds_['builds']
500	if build not in builds:
501	raise error.general('build %s not found' % (build))
502	if nesting > 20:
503	raise error.general('nesting build %s' % (build))
504	options = []
505	for option in self.builds_['builds'][build]:
506	if ':' in option:
507	section, name = option.split(':', 1)
508	opts = [self._get_item(section, name)]
509	else:
510	opts = self._options(option, nesting + 1)
511	for opt in opts:
512	if opt not in options:
513	options += [opt]
514	return options
515
516	def load(self, name, build):
517	if not path.exists(name):
518	raise error.general('config: cannot read configuration: %s' % (name))
519	self.name = name
520	try:
521	self.config.read(name)
522	except configparser.ParsingError as ce:
523	raise error.general('config: %s' % (ce))
524	archs = []
525	self.profiles['profiles'] = self._comma_list('profiles', 'profiles', error = False)
526	if len(self.profiles['profiles']) == 0:
527	self.profiles['profiles'] = ['tier-%d' % (t) for t in range(1,4)]
528	for p in self.profiles['profiles']:
529	profile = {}
530	profile['name'] = p
531	profile['archs'] = self._comma_list(profile['name'], 'archs')
532	archs += profile['archs']
533	for arch in profile['archs']:
534	bsps = 'bsps_%s' % (arch)
535	profile[bsps] = self._comma_list(profile['name'], bsps)
536	self.profiles[profile['name']] = profile
537	for a in set(archs):
538	arch = {}
539	arch['excludes'] = {}
540	for exclude in self._comma_list(a, 'exclude', error = False):
541	arch['excludes'][exclude] = ['all']
542	for i in self._get_items(a, False):
543	if i[0].startswith('exclude_'):
544	exclude = i[0][len('exclude_'):]
545	if exclude not in arch['excludes']:
546	arch['excludes'][exclude] = []
547	arch['excludes'][exclude] += sorted(set([b.strip() for b in i[1].split(',')]))
548	arch['bsps'] = self._comma_list(a, 'bsps', error = False)
549	for b in arch['bsps']:
550	arch[b] = {}
551	arch[b]['bspopts'] = self._comma_list(a, 'bspopts_%s' % (b), error = False)
552	self.archs[a] = arch
553	builds = {}
554	builds['default'] = self._get_item('builds', 'default')
555	if build is None:
556	build = builds['default']
557	builds['config'] = { }
558	for config in self._get_items('config'):
559	builds['config'][config[0]] = config[1]
560	builds['build'] = build
561	builds_ = self._get_item_names('builds')
562	builds['builds'] = {}
563	for build in builds_:
564	build_builds = self._comma_list('builds', build)
565	has_config = False
566	has_build = False
567	for b in build_builds:
568	if ':' in b:
569	if has_build:
570	raise error.general('config and build in build: %s' % (build))
571	has_config = True
572	else:
573	if has_config:
574	raise error.general('config and build in build: %s' % (build))
575	has_build = True
576	builds['builds'][build] = build_builds
577	self.builds_ = builds
578
579	def build(self):
580	return self.builds_['build']
581
582	def builds(self):
583	if self.builds_['build'] in self.builds_['builds']:
584	build = self.builds_['builds'][self.builds_['build']]
585	if ':' in build[0]:
586	return [self.builds_['build']]
587	return build
588	return None
589
590	def build_options(self, build):
591	return ' '.join(self._build_options(build))
592
593	def excludes(self, arch):
594	excludes = self.archs[arch]['excludes'].keys()
595	for exclude in self.archs[arch]['excludes']:
596	if 'all' not in self.archs[arch]['excludes'][exclude]:
597	excludes.remove(exclude)
598	return sorted(excludes)
599
600	def archs(self):
601	return sorted(self.archs.keys())
602
603	def arch_present(self, arch):
604	return arch in self.archs
605
606	def arch_bsps(self, arch):
607	return sorted(self.archs[arch]['bsps'])
608
609	def bsp_present(self, arch, bsp):
610	return bsp in self.archs[arch]['bsps']
611
612	def bsp_excludes(self, arch, bsp):
613	excludes = self.archs[arch]['excludes'].keys()
614	for exclude in self.archs[arch]['excludes']:
615	if bsp not in self.archs[arch]['excludes'][exclude]:
616	excludes.remove(exclude)
617	return sorted(excludes)
618
619	def bspopts(self, arch, bsp):
620	return self.archs[arch][bsp]['bspopts']
621
622	def profile_present(self, profile):
623	return profile in self.profiles
624
625	def profile_archs(self, profile):
626	return self.profiles[profile]['archs']
627
628	def profile_arch_bsps(self, profile, arch):
629	return self.profiles[profile]['bsps_%s' % (arch)]
630
631	def report(self, profiles = True, builds = True, architectures = True):
632	width = 70
633	cols_1 = [width]
634	cols_2 = [10, width - 10]
635	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
636	s1 = ' File'
637	colon = ':'
638	for l in textwrap.wrap(self.name, width = cols_2[1] - 3):
639	s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1)
640	colon = ' '
641	s1 = ' ' * len(s1)
642	s += textbox.line(cols_1, marker = '+', indent = 1)
643	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
644	if profiles:
645	profiles = sorted(self.profiles['profiles'])
646	bsps = 0
647	for profile in profiles:
648	archs = sorted(self.profiles[profile]['archs'])
649	for arch in archs:
650	bsps += len(self.profiles[profile]['bsps_%s' % (arch)])
651	s += textbox.row(cols_1,
652	[' Profiles : %d/%d' % (len(archs), bsps)],
653	indent = 1)
654	for profile in profiles:
655	textbox.row(cols_2,
656	[profile, self.profiles[profile]['name']],
657	indent = 1)
658	s += textbox.line(cols_1, marker = '+', indent = 1)
659	for profile in profiles:
660	s += textbox.row(cols_1, [' %s' % (profile)], indent = 1)
661	profile = self.profiles[profile]
662	archs = sorted(profile['archs'])
663	for arch in archs:
664	s += textbox.line(cols_2, marker = '+', indent = 1)
665	s1 = ' ' + arch
666	for l in textwrap.wrap(', '.join(profile['bsps_%s' % (arch)]),
667	width = cols_2[1] - 2):
668	s += textbox.row(cols_2, [s1, ' ' + l], indent = 1)
669	s1 = ' ' * len(s1)
670	s += textbox.line(cols_2, marker = '+', indent = 1)
671	if builds:
672	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
673	s += textbox.row(cols_1,
674	[' Builds: %s (default)' % (self.builds_['default'])],
675	indent = 1)
676	builds = self.builds_['builds']
677	bsize = 0
678	for build in builds:
679	if len(build) > bsize:
680	bsize = len(build)
681	cols_b = [bsize + 2, width - bsize - 2]
682	s += textbox.line(cols_b, marker = '+', indent = 1)
683	for build in builds:
684	s1 = ' ' + build
685	for l in textwrap.wrap(', '.join(builds[build]),
686	width = cols_b[1] - 2):
687	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
688	s1 = ' ' * len(s1)
689	s += textbox.line(cols_b, marker = '+', indent = 1)
690	configs = self.builds_['config']
691	s += textbox.row(cols_1,
692	[' Configure Options: %d' % (len(configs))],
693	indent = 1)
694	csize = 0
695	for config in configs:
696	if len(config) > csize:
697	csize = len(config)
698	cols_c = [csize + 3, width - csize - 3]
699	s += textbox.line(cols_c, marker = '+', indent = 1)
700	for config in configs:
701	s1 = ' ' + config
702	for l in textwrap.wrap(configs[config], width = cols_c[1] - 3):
703	s += textbox.row(cols_c, [s1, ' ' + l], indent = 1)
704	s1 = ' ' * len(s1)
705	s += textbox.line(cols_c, marker = '+', indent = 1)
706	if architectures:
707	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
708	archs = sorted(self.archs.keys())
709	bsps = 0
710	asize = 0
711	for arch in archs:
712	if len(arch) > asize:
713	asize = len(arch)
714	bsps += len(self.archs[arch]['bsps'])
715	s += textbox.row(cols_1,
716	[' Architectures : %d (bsps: %d)' % (len(archs), bsps)],
717	indent = 1)
718	cols_a = [asize + 2, width - asize - 2]
719	s += textbox.line(cols_a, marker = '+', indent = 1)
720	for arch in archs:
721	s += textbox.row(cols_a,
722	[' ' + arch, ' %d' % (len(self.archs[arch]['bsps']))],
723	indent = 1)
724	s += textbox.line(cols_a, marker = '+', indent = 1)
725	for archn in archs:
726	arch = self.archs[archn]
727	if len(arch['bsps']) > 0:
728	bsize = 0
729	for bsp in arch['bsps']:
730	if len(bsp) > bsize:
731	bsize = len(bsp)
732	cols_b = [bsize + 3, width - bsize - 3]
733	s += textbox.row(cols_1, [' ' + archn + ':'], indent = 1)
734	s += textbox.line(cols_b, marker = '+', indent = 1)
735	for bsp in arch['bsps']:
736	s1 = ' ' + bsp
737	bspopts = ' '.join(arch[bsp]['bspopts'])
738	if len(bspopts):
739	for l in textwrap.wrap('bopt: ' + bspopts,
740	width = cols_b[1] - 3):
741	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
742	s1 = ' ' * len(s1)
743	excludes = []
744	for exclude in arch['excludes']:
745	if bsp in arch['excludes'][exclude]:
746	excludes += [exclude]
747	excludes = ', '.join(excludes)
748	if len(excludes):
749	for l in textwrap.wrap('ex: ' + excludes,
750	width = cols_b[1] - 3):
751	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
752	s1 = ' ' * len(s1)
753	if len(bspopts) == 0 and len(excludes) == 0:
754	s += textbox.row(cols_b, [s1, ' '], indent = 1)
755	s += textbox.line(cols_b, marker = '+', indent = 1)
756	return s
757
758	class build:
759
760	def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
761	self.config = config
762	self.build_dir = build_dir
763	self.rtems_version = version
764	self.prefix = prefix
765	self.tools = tools
766	self.rtems = rtems
767	self.options = options
768	self.errors = { 'configure': 0,
769	'build': 0,
770	'tests': 0,
771	'fails': []}
772	self.counts = { 'h' : 0,
773	'exes' : 0,
774	'objs' : 0,
775	'libs' : 0 }
776	self.warnings_errors = warnings_errors(rtems)
777	self.results = results()
778	if not path.exists(path.join(rtems, 'configure')) or \
779	not path.exists(path.join(rtems, 'Makefile.in')) or \
780	not path.exists(path.join(rtems, 'cpukit')):
781	raise error.general('RTEMS source path does not look like RTEMS')
782
783	def _error_str(self):
784	return 'Status: configure:%d build:%d' % \
785	(self.errors['configure'], self.errors['build'])
786
787	def _path(self, arch, bsp):
788	return path.join(self.build_dir, arch, bsp)
789
790	def _archs(self, build_data):
791	return sorted(build_data.keys())
792
793	def _bsps(self, arch):
794	return self.config.arch_bsps(arch)
795
796	def _build(self):
797	return self.config.build()
798
799	def _builds(self, arch, bsp):
800	builds = self.config.builds()
801	if builds is None:
802	return None
803	for b in self.config.excludes(arch):
804	if b in builds:
805	builds.remove(b)
806	for b in self.config.bsp_excludes(arch, bsp):
807	if b in builds:
808	builds.remove(b)
809	return builds
810
811	def _arch_bsp_dir_make(self, arch, bsp):
812	if not path.exists(self._path(arch, bsp)):
813	path.mkdir(self._path(arch, bsp))
814
815	def _arch_bsp_dir_clean(self, arch, bsp):
816	if path.exists(self._path(arch, bsp)):
817	path.removeall(self._path(arch, bsp))
818
819	def _config_command(self, commands, arch, bsp):
820	if type(commands) is not list:
821	commands = [commands]
822	cmd = [path.join(self.rtems, 'configure')]
823	commands += self.config.bspopts(arch, bsp)
824	for c in commands:
825	c = c.replace('@PREFIX@', self.prefix)
826	c = c.replace('@RTEMS_VERSION@', self.rtems_version)
827	c = c.replace('@ARCH@', arch)
828	c = c.replace('@BSP@', bsp)
829	cmd += [c]
830	return ' '.join(cmd)
831
832	def _build_set(self, builds):
833	build_set = { }
834	for build in builds:
835	build_set[build] = self.config.build_options(build)
836	return build_set
837
838	def _build_dir(self, arch, bsp, build):
839	return path.join(self._path(arch, bsp), build)
840
841	def _count_files(self, arch, bsp, build):
842	counts = { 'h' : 0,
843	'exes' : 0,
844	'objs' : 0,
845	'libs' : 0 }
846	for root, dirs, files in os.walk(self._build_dir(arch, bsp, build)):
847	for file in files:
848	if file.endswith('.exe'):
849	counts['exes'] += 1
850	elif file.endswith('.o'):
851	counts['objs'] += 1
852	elif file.endswith('.a'):
853	counts['libs'] += 1
854	elif file.endswith('.h'):
855	counts['h'] += 1
856	for f in self.counts:
857	if f in counts:
858	self.counts[f] += counts[f]
859	return counts
860
861	def _have_failures(self, fails):
862	return len(fails) != 0
863
864	def _warnings_report(self):
865	if self.options['warnings-report'] is not None:
866	with open(self.options['warnings-report'], 'w') as f:
867	f.write(title() + os.linesep)
868	f.write(os.linesep)
869	f.write('Date: %s%s' % (datetime.date.today().strftime('%c'), os.linesep))
870	f.write(os.linesep)
871	f.write(command_line() + os.linesep)
872	f.write(self.warnings_errors.report())
873
874	def _finished(self):
875	log.notice('+ warnings:%d exes:%d objs:%d libs:%d' % \
876	(self.warnings_errors.get_warning_count(), self.counts['exes'],
877	self.counts['objs'], self.counts['libs']))
878	log.output()
879	log.output('Warnings:')
880	log.output(self.warnings_errors.report())
881	log.output()
882	log.notice('Failures:')
883	log.notice(self.failures_report(self.errors['fails']))
884	self._warnings_report()
885
886	def failures_report(self, fails):
887	if not self._have_failures(fails):
888	return ' No failure(s)'
889	absize = 0
890	bsize = 0
891	ssize = 0
892	for f in fails:
893	arch_bsp = '%s/%s' % (f[1], f[2])
894	if len(arch_bsp) > absize:
895	absize = len(arch_bsp)
896	if len(f[3]) > bsize:
897	bsize = len(f[3])
898	if len(f[0]) > ssize:
899	ssize = len(f[0])
900	fc = 1
901	s = ''
902	for f in fails:
903	fcl = '%3d' % (fc)
904	arch_bsp = '%s/%s' % (f[1], f[2])
905	state = f[0]
906	s1 = '%s %-s %-s %-*s ' % (fcl, bsize, f[3], absize, arch_bsp, ssize, state)
907	s += wrap((s1, f[4]), lineend = '\\')
908	s1 = ' ' * (len(s1) + 2)
909	for e in self.warnings_errors.get_error_messages(f[1], f[2], f[3]):
910	s += wrap([s1, 'error: ' + e])
911	fc += 1
912	return s
913
914	def build_arch_bsp(self, arch, bsp):
915	if not self.config.bsp_present(arch, bsp):
916	raise error.general('BSP not found: %s/%s' % (arch, bsp))
917	log.output('-' * 70)
918	log.notice('] BSP: %s/%s' % (arch, bsp))
919	log.notice('. Creating: %s' % (self._path(arch, bsp)))
920	self._arch_bsp_dir_clean(arch, bsp)
921	self._arch_bsp_dir_make(arch, bsp)
922	builds = self._builds(arch, bsp)
923	if builds is None:
924	raise error.general('build not found: %s' % (self._build()))
925	build_set = self._build_set(builds)
926	bsp_start = datetime.datetime.now()
927	env_path = os.environ['PATH']
928	os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
929	os.pathsep + os.environ['PATH']
930	fails = []
931	for bs in sorted(build_set.keys()):
932	self.warnings_errors.set_build(arch, bsp, bs)
933	start = datetime.datetime.now()
934	log.output('- ' * 35)
935	log.notice('. Configuring: %s' % (bs))
936	try:
937	warnings = self.warnings_errors.get_warning_count()
938	result = '+ Pass'
939	bpath = self._build_dir(arch, bsp, bs)
940	good = True
941	error_messages = None
942	path.mkdir(bpath)
943	config_cmd = self._config_command(build_set[bs], arch, bsp)
944	cmd = config_cmd
945	e = execute.capture_execution(log = self.warnings_errors)
946	log.output(wrap(('run: ', cmd), lineend = '\\'))
947	if self.options['dry-run']:
948	exit_code = 0
949	else:
950	exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
951	if exit_code != 0:
952	result = '- FAIL'
953	failure = ('configure', arch, bsp, bs, config_cmd)
954	fails += [failure]
955	self.errors['configure'] += 1
956	self.errors['fails'] += [failure]
957	log.notice('- Configure failed: %s' % (bs))
958	log.output('cmd failed: %s' % (cmd))
959	good = False
960	else:
961	log.notice('. Building: %s' % (bs))
962	cmd = 'make'
963	if 'jobs' in self.options:
964	cmd += ' -j %s' % (self.options['jobs'])
965	log.output('run: ' + cmd)
966	if self.options['dry-run']:
967	exit_code = 0
968	else:
969	exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
970	if exit_code != 0:
971	error_messages = self.warnings_errors.get_error_messages()
972	result = '- FAIL'
973	failure = ('build', arch, bsp, bs, config_cmd, error_messages)
974	fails += [failure]
975	self.errors['build'] += 1
976	self.errors['fails'] += [failure]
977	log.notice('- FAIL: %s: %s' % (bs, self._error_str()))
978	log.output('cmd failed: %s' % (cmd))
979	good = False
980	files = self._count_files(arch, bsp, bs)
981	log.notice('%s: %s: warnings:%d exes:%d objs:%s libs:%d' % \
982	(result, bs,
983	self.warnings_errors.get_warning_count() - warnings,
984	files['exes'], files['objs'], files['libs']))
985	log.notice(' %s' % (self._error_str()))
986	self.results.add(good, arch, bsp, config_cmd,
987	self.warnings_errors.get_warning_count() - warnings,
988	error_messages)
989	if not good and self.options['stop-on-error']:
990	raise error.general('Configuring %s failed' % (bs))
991	finally:
992	end = datetime.datetime.now()
993	if not self.options['no-clean']:
994	log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs)))
995	path.removeall(self._build_dir(arch, bsp, bs))
996	log.notice('^ Time %s' % (str(end - start)))
997	self.warnings_errors.clear_build()
998	bsp_end = datetime.datetime.now()
999	log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start)))
1000	log.output('Failure Report:')
1001	log.output(self.failures_report(fails))
1002	os.environ['PATH'] = env_path
1003
1004	def build_bsp(self, arch, bsp):
1005	self.build_arch_bsp(arch, bsp)
1006	self._finished()
1007
1008	def build_arch(self, arch):
1009	start = datetime.datetime.now()
1010	log.output('=' * 70)
1011	log.notice(']] Architecture: %s' % (arch))
1012	if not self.config.arch_present(arch):
1013	raise error.general('Architecture not found: %s' % (arch))
1014	for bsp in self._bsps(arch):
1015	self.build_arch_bsp(arch, bsp)
1016	end = datetime.datetime.now()
1017	log.notice('^ Architecture Time %s' % (str(end - start)))
1018	self._finished()
1019
1020	def build(self):
1021	for arch in self.config.archs():
1022	self.build_arch(arch)
1023	log.notice('^ Profile Time %s' % (str(end - start)))
1024	self._finished()
1025
1026	def build_profile(self, profile):
1027	if not self.config.profile_present(profile):
1028	raise error.general('Profile not found: %s' % (profile))
1029	start = datetime.datetime.now()
1030	log.notice(']] Profile: %s' % (profile))
1031	for arch in self.config.profile_archs(profile):
1032	for bsp in self.config.profile_arch_bsps(profile, arch):
1033	self.build_arch_bsp(arch, bsp)
1034	end = datetime.datetime.now()
1035	log.notice('^ Profile Time %s' % (str(end - start)))
1036	self._finished()
1037
1038	def run_args(args):
1039	b = None
1040	ec = 0
1041	try:
1042	#
1043	# On Windows MSYS2 prepends a path to itself to the environment
1044	# path. This means the RTEMS specific automake is not found and which
1045	# breaks the bootstrap. We need to remove the prepended path. Also
1046	# remove any ACLOCAL paths from the environment.
1047	#
1048	if os.name == 'nt':
1049	cspath = os.environ['PATH'].split(os.pathsep)
1050	if 'msys' in cspath[0] and cspath[0].endswith('bin'):
1051	os.environ['PATH'] = os.pathsep.join(cspath[1:])
1052
1053	top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
1054	prefix = '/opt/rtems/%s' % (rtems_version())
1055	tools = prefix
1056	build_dir = 'bsp-builds'
1057	logf = 'bsp-build-%s.txt' % \
1058	(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
1059	config_file = path.join(top, 'share', 'rtems', 'tester',
1060	'rtems', 'rtems-bsps.ini')
1061	if not path.exists(config_file):
1062	config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')
1063
1064	argsp = argparse.ArgumentParser()
1065	argsp.add_argument('--prefix', help = 'Prefix to build the BSP.',
1066	type = str)
1067	argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.',
1068	type = str)
1069	argsp.add_argument('--rtems', help = 'The RTEMS source tree.',
1070	type = str)
1071	argsp.add_argument('--config-report', help = 'Report the configuration.',
1072	action = 'store_true')
1073	argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
1074	type = str, default = None)
1075	argsp.add_argument('--build-path', help = 'Path to build in.',
1076	type = str)
1077	argsp.add_argument('--log', help = 'Log file.', type = str)
1078	argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
1079	action = 'store_true')
1080	argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
1081	action = 'store_true')
1082	argsp.add_argument('--profiles', help = 'Build the listed profiles.',
1083	type = str, default = 'tier-1')
1084	argsp.add_argument('--build', help = 'Build name to build.',
1085	type = str, default='all')
1086	argsp.add_argument('--arch', help = 'Build the specific architecture.',
1087	type = str)
1088	argsp.add_argument('--bsp', help = 'Build the specific BSP.',
1089	type = str)
1090	argsp.add_argument('--jobs', help = 'Number of jobs to run.',
1091	type = int, default = host.cpus())
1092	argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
1093	action = 'store_true')
1094
1095	opts = argsp.parse_args(args[1:])
1096	if opts.log is not None:
1097	logf = opts.log
1098	log.default = log.log([logf])
1099	log.notice(title())
1100	log.output(command_line())
1101	if opts.rtems is None:
1102	raise error.general('No RTEMS source provided on the command line')
1103	if opts.prefix is not None:
1104	prefix = path.shell(opts.prefix)
1105	if opts.rtems_tools is not None:
1106	tools = path.shell(opts.rtems_tools)
1107	if opts.build_path is not None:
1108	build_dir = path.shell(opts.build_path)
1109	if opts.bsp is not None and opts.arch is None:
1110	raise error.general('BSP provided but no architecture')
1111
1112	config = configuration()
1113	config.load(config_file, opts.build)
1114
1115	if opts.config_report:
1116	log.notice('Configuration Report:')
1117	log.notice(config.report())
1118	sys.exit(0)
1119
1120	options = { 'stop-on-error' : opts.stop_on_error,
1121	'no-clean' : opts.no_clean,
1122	'dry-run' : opts.dry_run,
1123	'jobs' : opts.jobs,
1124	'warnings-report' : opts.warnings_report }
1125
1126	b = build(config, rtems_version(), prefix, tools,
1127	path.shell(opts.rtems), build_dir, options)
1128	if opts.arch is not None:
1129	if opts.bsp is not None:
1130	b.build_bsp(opts.arch, opts.bsp)
1131	else:
1132	b.build_arch(opts.arch)
1133	else:
1134	for profile in opts.profiles.split(','):
1135	b.build_profile(profile.strip())
1136
1137	except error.general as gerr:
1138	print(gerr)
1139	print('BSP Build FAILED', file = sys.stderr)
1140	ec = 1
1141	except error.internal as ierr:
1142	print(ierr)
1143	print('BSP Build FAILED', file = sys.stderr)
1144	ec = 1
1145	except error.exit as eerr:
1146	pass
1147	except KeyboardInterrupt:
1148	log.notice('abort: user terminated')
1149	ec = 1
1150	if b is not None:
1151	b.results.report()
1152	sys.exit(ec)
1153
1154	def run():
1155	run_args(sys.argv)
1156
1157	if __name__ == "__main__":
1158	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: