Context Navigation

check.py @ 5d1edd5

Last change on this file since 5d1edd5 was 5d1edd5, checked in by Chris Johns <chrisj@…>, on 04/24/17 at 16:39:56

rtems-bsp-builder: Refactor for better config format, warnings and errors.

Refactor the code to improve the warnings and errors reporting.

Improve the configuration file format to better support any type
of build by separating the flags away from the builds.

Property mode set to 100755

File size: 46.3 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2016-2017 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Redistribution and use in source and binary forms, with or without
9	# modification, are permitted provided that the following conditions are met:
10	#
11	# 1. Redistributions of source code must retain the above copyright notice,
12	# this list of conditions and the following disclaimer.
13	#
14	# 2. Redistributions in binary form must reproduce the above copyright notice,
15	# this list of conditions and the following disclaimer in the documentation
16	# and/or other materials provided with the distribution.
17	#
18	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21	# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22	# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23	# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24	# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25	# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26	# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27	# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28	# POSSIBILITY OF SUCH DAMAGE.
29	#
30
31	from __future__ import print_function
32
33	import argparse
34	import datetime
35	import operator
36	import os
37	import re
38	import sys
39	import textwrap
40
41	import pprint
42
43	try:
44	import configparser
45	except:
46	import ConfigParser as configparser
47
48	from rtemstoolkit import execute
49	from rtemstoolkit import error
50	from rtemstoolkit import host
51	from rtemstoolkit import log
52	from rtemstoolkit import path
53	from rtemstoolkit import textbox
54	from rtemstoolkit import version
55
56	def rtems_version():
57	return version.version()
58
59	def wrap(line, lineend = '', indent = 0):
60	if type(line) is tuple or type(line) is list:
61	if len(line) >= 2:
62	s1 = line[0]
63	else:
64	s1 = ''
65	s2 = line[1:]
66	elif type(line) is str:
67	s1 = ''
68	s2 = [line]
69	else:
70	raise error.internal('line is not a tuple, list or string')
71	s = ''
72	first = True
73	for ss in s2:
74	if type(ss) is not str and type(ss) is not unicode:
75	raise error.internal('text needs to be a string')
76	for l in textwrap.wrap(ss):
77	s += '%s%s%s%s%s' % (' ' * indent, s1, l, lineend, os.linesep)
78	if first and len(s1) > 0:
79	s1 = ' ' * len(s1)
80	if lineend != '':
81	s = s[:0 - len(os.linesep) - 1] + os.linesep
82	return s
83
84	def title():
85	return 'RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str())
86
87	def command_line():
88	return wrap(('command: ', ' '.join(sys.argv)), lineend = '\\')
89
90	class warnings_errors:
91
92	def __init__(self, rtems):
93	self.rtems = path.host(rtems)
94	self.reset()
95	self.groups = { 'groups' : ['Shared', 'BSP', 'Network', 'Tests',
96	'LibCPU', 'CPU Kit'],
97	'exclude' : '.Makefile.',
98	'CPU Kit' : '.cpukit/.',
99	'Network' : '.libnetworking/.',
100	'Tests' : '.testsuites/.',
101	'BSP' : '.libbsp/.',
102	'LibCPU' : '.libcpu/.',
103	'Shared' : '.shared/.' }
104	self.arch = None
105	self.bsp = None
106	self.build = None
107
108	def _opts(self, arch = None, bsp = None, build = None):
109	if arch is None:
110	arch = self.arch
111	if bsp is None:
112	bsp = self.bsp
113	if build is None:
114	build = self.build
115	return arch, bsp, build
116
117	def _key(self, arch, bsp, build):
118	arch, bsp, build = self._opts(arch, bsp, build)
119	return '%s/%s-%s' % (arch, bsp, build)
120
121	def _get_warnings(self, arch = None, bsp = None, build = None):
122	arch, bsp, build = self._opts(arch = arch, bsp = bsp, build = build)
123	if arch is None:
124	arch = '.*'
125	if bsp is None:
126	bsp = '.*'
127	if build is None:
128	build = '.*'
129	selector = re.compile('^%s/%s-%s$' % (arch, bsp, build))
130	warnings = [w for w in self.warnings if selector.match(w)]
131	return sorted(warnings)
132
133	def _total(self, archive):
134	total = 0
135	for a in archive:
136	total += archive[a]
137	return total
138
139	def _analyze(self, warnings, exclude):
140	def _group(data, category, name, warning, count, groups, group_regx):
141	if 'groups' not in data:
142	data['groups'] = { }
143	if category not in data['groups']:
144	data['groups'][category] = { 'totals' : { } }
145	if name not in data['groups'][category]:
146	data['groups'][category][name] = { }
147	for group in groups:
148	if group not in data['groups'][category]['totals']:
149	data['groups'][category]['totals'][group] = 0
150	if group not in data['groups'][category][name]:
151	data['groups'][category][name][group] = 0
152	if group_regx[group].match(warning):
153	data['groups'][category][name][group] += count
154	data['groups'][category]['totals'][group] += count
155	break
156
157	def _update(data, category, name, warning, count, groups, group_regx):
158	if category not in data:
159	data[category] = { }
160	if name not in data[category]:
161	data[category][name] = { }
162	if warning not in data[category][name]:
163	data[category][name][warning] = 0
164	data[category][name][warning] += count
165	_group(data, category, name, w, count, groups, group_regx)
166
167	data = { }
168	group_regx = { }
169	for group in self.groups['groups']:
170	group_regx[group] = re.compile(self.groups[group])
171	exclude_regx = re.compile(exclude)
172	for warning in warnings:
173	arch = warning.split('/', 1)[0]
174	arch_bsp = warning.split('-', 1)[0]
175	build = warning.split('-', 1)[1]
176	for w in self.warnings[warning]:
177	if not exclude_regx.match(w):
178	count = self.warnings[warning][w]
179	_update(data, 'arch', arch, w, count,
180	self.groups['groups'], group_regx)
181	_update(data, 'arch_bsp', arch_bsp, w, count,
182	self.groups['groups'], group_regx)
183	_update(data, 'build', build, w, count,
184	self.groups['groups'], group_regx)
185	for category in ['arch', 'arch_bsp', 'build']:
186	common = {}
187	for name in data[category]:
188	for w in data[category][name]:
189	if w not in common:
190	for other in [n for n in data[category] if n != name]:
191	if w in data[category][other]:
192	common[w] = data[category][name][w]
193	_group(data, category, 'common', w, common[w],
194	self.groups['groups'], group_regx)
195	data[category]['common'] = common
196	return data
197
198	def _report_category(self, label, warnings, group_counts):
199	width = 70
200	cols_1 = [width]
201	cols_2 = [8, width - 8]
202	cols_4 = textbox.even_columns(4, width)
203	cols_2_4 = textbox.merge_columns([cols_2, cols_4])
204	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
205	s += textbox.row(cols_1, [' ' + label], indent = 1)
206	s += textbox.line(cols_1, marker = '+', indent = 1)
207	builds = ['common'] + sorted([b for b in warnings if b != 'common'])
208	common = warnings['common']
209	for build in builds:
210	build_warnings = warnings[build]
211	if build is not 'common':
212	build_warnings = [w for w in build_warnings if w not in common]
213	s += textbox.row(cols_1,
214	[' %s : %d warning(s)' % (build,
215	len(build_warnings))],
216	indent = 1)
217	if len(build_warnings) == 0:
218	s += textbox.line(cols_1, marker = '+', indent = 1)
219	else:
220	s += textbox.line(cols_4, marker = '+', indent = 1)
221	if build not in group_counts:
222	gs = [0 for group in self.groups['groups']]
223	else:
224	gs = []
225	for g in range(0, len(self.groups['groups'])):
226	group = self.groups['groups'][g]
227	gs += ['%*s' % (cols_4[g % 4] - 2,
228	'%s : %4d' % \
229	(group,
230	group_counts[build][group]))]
231	for row in range(0, len(self.groups['groups']), 4):
232	if row + 4 > len(self.groups['groups']):
233	d = gs[row:] + \
234	['' for r in range(row,
235	len(self.groups['groups']))]
236	else:
237	d = gs[row:+4]
238	s += textbox.row(cols_4, d, indent = 1)
239	s += textbox.line(cols_2_4, marker = '+', indent = 1)
240	vw = sorted([(w, warnings[build][w]) for w in build_warnings],
241	key = operator.itemgetter(1),
242	reverse = True)
243	for w in vw:
244	c1 = '%6d' % w[1]
245	for l in textwrap.wrap(' ' + w[0]):
246	s += textbox.row(cols_2, [c1, l], indent = 1)
247	c1 = ' ' * 6
248	s += textbox.line(cols_2, marker = '+', indent = 1)
249	return s
250
251	def _report_warning_map(self):
252	builds = self.messages['warnings']
253	width = 70
254	cols_1 = [width]
255	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
256	s += textbox.row(cols_1, [' Warning Map'], indent = 1)
257	s += textbox.line(cols_1, marker = '+', indent = 1)
258	for build in builds:
259	messages = builds[build]
260	s += textbox.row(cols_1, [' %s : %d' % (build, len(messages))], indent = 1)
261	s += textbox.line(cols_1, marker = '+', indent = 1)
262	for msg in messages:
263	for l in textwrap.wrap(msg, width = width - 3):
264	s += textbox.row(cols_1, [' ' + l], indent = 1)
265	for l in textwrap.wrap(messages[msg], width = width - 3 - 4):
266	s += textbox.row(cols_1, [' ' + l], indent = 1)
267	s += textbox.line(cols_1, marker = '+', indent = 1)
268	return s
269
270	def report(self):
271	arch, bsp, build = self._opts()
272	warnings = self._get_warnings(arch, bsp, build)
273	total = 0
274	for build in warnings:
275	total += self._total(self.warnings[build])
276	if total == 0:
277	s = ' No warnings'
278	else:
279	data = self._analyze(warnings, self.groups['exclude'])
280	s = self._report_category('By Architecture (total : %d)' % (total),
281	data['arch'], data['groups']['arch'])
282	s += os.linesep
283	s += self._report_category('By BSP (total : %d)' % (total),
284	data['arch_bsp'], data['groups']['arch_bsp'])
285	s += os.linesep
286	s += self._report_category('By Build (total : %d)' % (total),
287	data['build'], data['groups']['build'])
288	s += os.linesep
289	s += self._report_warning_map()
290	s += os.linesep
291
292	return s
293
294	def set_build(self, arch, bsp, build):
295	self.arch = arch
296	self.bsp = bsp
297	self.build = build
298	self.build_key = '%s/%s-%s' % (arch, bsp, build)
299	if self.build_key not in self.warnings:
300	self.warnings[self.build_key] = {}
301	if self.build_key not in self.errors:
302	self.errors[self.build_key] = {}
303
304	def clear_build(self):
305	self.arch = None
306	self.bsp = None
307	self.build = None
308	self.build_key = None
309
310	def get_warning_count(self):
311	return self.warning_count
312
313	def get_error_count(self):
314	return self.error_count
315
316	def reset(self):
317	self.warnings = { }
318	self.warning_count = 0
319	self.errors = { }
320	self.error_count = 0
321	self.messages = { 'warnings' : { }, 'errors' : { } }
322
323	def get_warning_messages(self, arch = None, bsp = None, build = None):
324	messages = self.messages['warnings'][self._key(arch, bsp, build)]
325	return ['%s %s' % (m, messages[m]) for m in messages]
326
327	def get_error_messages(self, arch = None, bsp = None, build = None):
328	messages = self.messages['errors'][self._key(arch, bsp, build)]
329	return ['%s %s' % (m, messages[m]) for m in messages]
330
331	def output(self, text):
332	def _line_split(line, source_base):
333	ls = line.split(' ', 1)
334	fname = ls[0].split(':')
335	#
336	# Ignore compiler option warnings.
337	#
338	if len(fname) < 4:
339	return None
340	p = path.abspath(fname[0])
341	p = p.replace(source_base, '')
342	if path.isabspath(p):
343	p = p[1:]
344	return p, fname[1], fname[2], ls[1]
345
346	if self.build_key is not None and \
347	(' warning:' in text or ' error:' in text):
348	for l in text.splitlines():
349	if ' warning:' in l:
350	self.warning_count += 1
351	archive = self.warnings[self.build_key]
352	messages = 'warnings'
353	elif ' error:' in l:
354	self.error_count += 1
355	archive = self.errors[self.build_key]
356	messages = 'errors'
357	else:
358	continue
359	line_parts = _line_split(l, self.rtems)
360	if line_parts is not None:
361	src, line, pos, msg = line_parts
362	where = '%s:%s:%s' % (src, line, pos)
363	if where not in archive:
364	archive[where] = 1
365	else:
366	archive[where] += 1
367	if self.build_key not in self.messages[messages]:
368	self.messages[messages][self.build_key] = { }
369	self.messages[messages][self.build_key][where] = msg
370
371	log.output(text)
372
373	class results:
374
375	def __init__(self):
376	self.passes = []
377	self.fails = []
378
379	def _arch_bsp(self, arch, bsp):
380	return '%s/%s' % (arch, bsp)
381
382	def add(self, good, arch, bsp, configure, warnings, error_messages):
383	if good:
384	self.passes += [(arch, bsp, configure, warnings, None)]
385	else:
386	self.fails += [(arch, bsp, configure, warnings, error_messages)]
387
388	def report(self):
389	log.notice('* Passes: %d Failures: %d' %
390	(len(self.passes), len(self.fails)))
391	log.output()
392	log.output('Build Report')
393	log.output(' Passes: %d Failures: %d' %
394	(len(self.passes), len(self.fails)))
395	log.output(' Failures:')
396	if len(self.fails) == 0:
397	log.output(' None')
398	else:
399	max_col = 0
400	for f in self.fails:
401	arch_bsp = self._arch_bsp(f[0], f[1])
402	if len(arch_bsp) > max_col:
403	max_col = len(arch_bsp)
404	for f in self.fails:
405	config_cmd = f[2]
406	config_at = config_cmd.find('configure')
407	if config_at != -1:
408	config_cmd = config_cmd[config_at:]
409	s1 = ' %*s: ' % (max_col + 2, self._arch_bsp(f[0], f[1]))
410	log.output(wrap([s1, config_cmd], lineend = '\\'))
411	if f[4] is not None:
412	s1 = ' ' * len(s1)
413	for msg in f[4]:
414	log.output(wrap([s1, msg], lineend = '\\'))
415	log.output(' Passes:')
416	if len(self.passes) == 0:
417	log.output(' None')
418	else:
419	max_col = 0
420	for f in self.passes:
421	arch_bsp = self._arch_bsp(f[0], f[1])
422	if len(arch_bsp) > max_col:
423	max_col = len(arch_bsp)
424	for f in self.passes:
425	config_cmd = f[2]
426	config_at = config_cmd.find('configure')
427	if config_at != -1:
428	config_cmd = config_cmd[config_at:]
429	log.output(wrap((' %*s: %5d ' % (max_col + 2,
430	self._arch_bsp(f[0], f[1]),
431	f[3]),
432	config_cmd),
433	lineend = '\\'))
434
435	class configuration:
436
437	def __init__(self):
438	self.config = configparser.ConfigParser()
439	self.name = None
440	self.archs = { }
441	self.builds_ = { }
442	self.profiles = { }
443	self.configurations = { }
444
445	def __str__(self):
446	import pprint
447	s = self.name + os.linesep
448	s += 'Archs:' + os.linesep + \
449	pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
450	s += 'Builds:' + os.linesep + \
451	pprint.pformat(self.builds_, indent = 1, width = 80) + os.linesep
452	s += 'Profiles:' + os.linesep + \
453	pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
454	return s
455
456	def _get_item(self, section, label, err = True):
457	try:
458	rec = self.config.get(section, label).replace(os.linesep, ' ')
459	return rec
460	except:
461	if err:
462	raise error.general('config: no "%s" found in "%s"' % (label, section))
463	return None
464
465	def _get_items(self, section, err = True):
466	try:
467	items = [(name, key.replace(os.linesep, ' ')) for name, key in self.config.items(section)]
468	return items
469	except:
470	if err:
471	raise error.general('config: section "%s" not found' % (section))
472	return []
473
474	def _comma_list(self, section, label, error = True):
475	items = self._get_item(section, label, error)
476	if items is None:
477	return []
478	return sorted(set([a.strip() for a in items.split(',')]))
479
480	def _get_item_names(self, section, err = True):
481	try:
482	return [item[0] for item in self.config.items(section)]
483	except:
484	if err:
485	raise error.general('config: section "%s" not found' % (section))
486	return []
487
488	def _build_options(self, build, nesting = 0):
489	if ':' in build:
490	section, name = build.split(':', 1)
491	opts = [self._get_item(section, name)]
492	return opts
493	builds = self.builds_['builds']
494	if build not in builds:
495	raise error.general('build %s not found' % (build))
496	if nesting > 20:
497	raise error.general('nesting build %s' % (build))
498	options = []
499	for option in self.builds_['builds'][build]:
500	if ':' in option:
501	section, name = option.split(':', 1)
502	opts = [self._get_item(section, name)]
503	else:
504	opts = self._options(option, nesting + 1)
505	for opt in opts:
506	if opt not in options:
507	options += [opt]
508	return options
509
510	def load(self, name, build):
511	if not path.exists(name):
512	raise error.general('config: cannot read configuration: %s' % (name))
513	self.name = name
514	try:
515	self.config.read(name)
516	except configparser.ParsingError as ce:
517	raise error.general('config: %s' % (ce))
518	archs = []
519	self.profiles['profiles'] = self._comma_list('profiles', 'profiles', error = False)
520	if len(self.profiles['profiles']) == 0:
521	self.profiles['profiles'] = ['tier-%d' % (t) for t in range(1,4)]
522	for p in self.profiles['profiles']:
523	profile = {}
524	profile['name'] = p
525	profile['archs'] = self._comma_list(profile['name'], 'archs')
526	archs += profile['archs']
527	for arch in profile['archs']:
528	bsps = 'bsps_%s' % (arch)
529	profile[bsps] = self._comma_list(profile['name'], bsps)
530	self.profiles[profile['name']] = profile
531	for a in set(archs):
532	arch = {}
533	arch['excludes'] = {}
534	for exclude in self._comma_list(a, 'exclude', error = False):
535	arch['excludes'][exclude] = ['all']
536	for i in self._get_items(a, False):
537	if i[0].startswith('exclude_'):
538	exclude = i[0][len('exclude_'):]
539	if exclude not in arch['excludes']:
540	arch['excludes'][exclude] = []
541	arch['excludes'][exclude] += sorted(set([b.strip() for b in i[1].split(',')]))
542	arch['bsps'] = self._comma_list(a, 'bsps', error = False)
543	for b in arch['bsps']:
544	arch[b] = {}
545	arch[b]['bspopts'] = self._comma_list(a, 'bspopts_%s' % (b), error = False)
546	self.archs[a] = arch
547	builds = {}
548	builds['default'] = self._get_item('builds', 'default')
549	if build is None:
550	build = builds['default']
551	builds['config'] = { }
552	for config in self._get_items('config'):
553	builds['config'][config[0]] = config[1]
554	builds['build'] = build
555	builds_ = self._get_item_names('builds')
556	builds['builds'] = {}
557	for build in builds_:
558	build_builds = self._comma_list('builds', build)
559	has_config = False
560	has_build = False
561	for b in build_builds:
562	if ':' in b:
563	if has_build:
564	raise error.general('config and build in build: %s' % (build))
565	has_config = True
566	else:
567	if has_config:
568	raise error.general('config and build in build: %s' % (build))
569	has_build = True
570	builds['builds'][build] = build_builds
571	self.builds_ = builds
572
573	def build(self):
574	return self.builds_['build']
575
576	def builds(self):
577	if self.builds_['build'] in self.builds_['builds']:
578	build = self.builds_['builds'][self.builds_['build']]
579	if ':' in build[0]:
580	return [self.builds_['build']]
581	return build
582	return None
583
584	def build_options(self, build):
585	return ' '.join(self._build_options(build))
586
587	def excludes(self, arch):
588	excludes = self.archs[arch]['excludes'].keys()
589	for exclude in self.archs[arch]['excludes']:
590	if 'all' not in self.archs[arch]['excludes'][exclude]:
591	excludes.remove(exclude)
592	return sorted(excludes)
593
594	def archs(self):
595	return sorted(self.archs.keys())
596
597	def arch_present(self, arch):
598	return arch in self.archs
599
600	def arch_bsps(self, arch):
601	return sorted(self.archs[arch]['bsps'])
602
603	def bsp_present(self, arch, bsp):
604	return bsp in self.archs[arch]['bsps']
605
606	def bsp_excludes(self, arch, bsp):
607	excludes = self.archs[arch]['excludes'].keys()
608	for exclude in self.archs[arch]['excludes']:
609	if bsp not in self.archs[arch]['excludes'][exclude]:
610	excludes.remove(exclude)
611	return sorted(excludes)
612
613	def bspopts(self, arch, bsp):
614	return self.archs[arch][bsp]['bspopts']
615
616	def profile_present(self, profile):
617	return profile in self.profiles
618
619	def profile_archs(self, profile):
620	return self.profiles[profile]['archs']
621
622	def profile_arch_bsps(self, profile, arch):
623	return self.profiles[profile]['bsps_%s' % (arch)]
624
625	def report(self, profiles = True, builds = True, architectures = True):
626	width = 70
627	cols_1 = [width]
628	cols_2 = [10, width - 10]
629	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
630	s1 = ' File'
631	colon = ':'
632	for l in textwrap.wrap(self.name, width = cols_2[1] - 3):
633	s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1)
634	colon = ' '
635	s1 = ' ' * len(s1)
636	s += textbox.line(cols_1, marker = '+', indent = 1)
637	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
638	if profiles:
639	profiles = sorted(self.profiles['profiles'])
640	bsps = 0
641	for profile in profiles:
642	archs = sorted(self.profiles[profile]['archs'])
643	for arch in archs:
644	bsps += len(self.profiles[profile]['bsps_%s' % (arch)])
645	s += textbox.row(cols_1,
646	[' Profiles : %d/%d' % (len(archs), bsps)],
647	indent = 1)
648	for profile in profiles:
649	textbox.row(cols_2,
650	[profile, self.profiles[profile]['name']],
651	indent = 1)
652	s += textbox.line(cols_1, marker = '+', indent = 1)
653	for profile in profiles:
654	s += textbox.row(cols_1, [' %s' % (profile)], indent = 1)
655	profile = self.profiles[profile]
656	archs = sorted(profile['archs'])
657	for arch in archs:
658	s += textbox.line(cols_2, marker = '+', indent = 1)
659	s1 = ' ' + arch
660	for l in textwrap.wrap(', '.join(profile['bsps_%s' % (arch)]),
661	width = cols_2[1] - 2):
662	s += textbox.row(cols_2, [s1, ' ' + l], indent = 1)
663	s1 = ' ' * len(s1)
664	s += textbox.line(cols_2, marker = '+', indent = 1)
665	if builds:
666	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
667	s += textbox.row(cols_1,
668	[' Builds: %s (default)' % (self.builds_['default'])],
669	indent = 1)
670	builds = self.builds_['builds']
671	bsize = 0
672	for build in builds:
673	if len(build) > bsize:
674	bsize = len(build)
675	cols_b = [bsize + 2, width - bsize - 2]
676	s += textbox.line(cols_b, marker = '+', indent = 1)
677	for build in builds:
678	s1 = ' ' + build
679	for l in textwrap.wrap(', '.join(builds[build]),
680	width = cols_b[1] - 2):
681	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
682	s1 = ' ' * len(s1)
683	s += textbox.line(cols_b, marker = '+', indent = 1)
684	configs = self.builds_['config']
685	s += textbox.row(cols_1,
686	[' Configure Options: %d' % (len(configs))],
687	indent = 1)
688	csize = 0
689	for config in configs:
690	if len(config) > csize:
691	csize = len(config)
692	cols_c = [csize + 3, width - csize - 3]
693	s += textbox.line(cols_c, marker = '+', indent = 1)
694	for config in configs:
695	s1 = ' ' + config
696	for l in textwrap.wrap(configs[config], width = cols_c[1] - 3):
697	s += textbox.row(cols_c, [s1, ' ' + l], indent = 1)
698	s1 = ' ' * len(s1)
699	s += textbox.line(cols_c, marker = '+', indent = 1)
700	if architectures:
701	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
702	archs = sorted(self.archs.keys())
703	bsps = 0
704	asize = 0
705	for arch in archs:
706	if len(arch) > asize:
707	asize = len(arch)
708	bsps += len(self.archs[arch]['bsps'])
709	s += textbox.row(cols_1,
710	[' Architectures : %d (bsps: %d)' % (len(archs), bsps)],
711	indent = 1)
712	cols_a = [asize + 2, width - asize - 2]
713	s += textbox.line(cols_a, marker = '+', indent = 1)
714	for arch in archs:
715	s += textbox.row(cols_a,
716	[' ' + arch, ' %d' % (len(self.archs[arch]['bsps']))],
717	indent = 1)
718	s += textbox.line(cols_a, marker = '+', indent = 1)
719	for archn in archs:
720	arch = self.archs[archn]
721	if len(arch['bsps']) > 0:
722	bsize = 0
723	for bsp in arch['bsps']:
724	if len(bsp) > bsize:
725	bsize = len(bsp)
726	cols_b = [bsize + 3, width - bsize - 3]
727	s += textbox.row(cols_1, [' ' + archn + ':'], indent = 1)
728	s += textbox.line(cols_b, marker = '+', indent = 1)
729	for bsp in arch['bsps']:
730	s1 = ' ' + bsp
731	bspopts = ' '.join(arch[bsp]['bspopts'])
732	if len(bspopts):
733	for l in textwrap.wrap('bopt: ' + bspopts,
734	width = cols_b[1] - 3):
735	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
736	s1 = ' ' * len(s1)
737	excludes = []
738	for exclude in arch['excludes']:
739	if bsp in arch['excludes'][exclude]:
740	excludes += [exclude]
741	excludes = ', '.join(excludes)
742	if len(excludes):
743	for l in textwrap.wrap('ex: ' + excludes,
744	width = cols_b[1] - 3):
745	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
746	s1 = ' ' * len(s1)
747	if len(bspopts) == 0 and len(excludes) == 0:
748	s += textbox.row(cols_b, [s1, ' '], indent = 1)
749	s += textbox.line(cols_b, marker = '+', indent = 1)
750	return s
751
752	class build:
753
754	def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
755	self.config = config
756	self.build_dir = build_dir
757	self.rtems_version = version
758	self.prefix = prefix
759	self.tools = tools
760	self.rtems = rtems
761	self.options = options
762	self.errors = { 'configure': 0,
763	'build': 0,
764	'tests': 0,
765	'fails': []}
766	self.counts = { 'h' : 0,
767	'exes' : 0,
768	'objs' : 0,
769	'libs' : 0 }
770	self.warnings_errors = warnings_errors(rtems)
771	self.results = results()
772	if not path.exists(path.join(rtems, 'configure')) or \
773	not path.exists(path.join(rtems, 'Makefile.in')) or \
774	not path.exists(path.join(rtems, 'cpukit')):
775	raise error.general('RTEMS source path does not look like RTEMS')
776
777	def _error_str(self):
778	return 'Status: configure:%d build:%d' % \
779	(self.errors['configure'], self.errors['build'])
780
781	def _path(self, arch, bsp):
782	return path.join(self.build_dir, arch, bsp)
783
784	def _archs(self, build_data):
785	return sorted(build_data.keys())
786
787	def _bsps(self, arch):
788	return self.config.arch_bsps(arch)
789
790	def _build(self):
791	return self.config.build()
792
793	def _builds(self, arch, bsp):
794	builds = self.config.builds()
795	if builds is None:
796	return None
797	for b in self.config.excludes(arch):
798	builds.remove(b)
799	for b in self.config.bsp_excludes(arch, bsp):
800	builds.remove(b)
801	return builds
802
803	def _arch_bsp_dir_make(self, arch, bsp):
804	if not path.exists(self._path(arch, bsp)):
805	path.mkdir(self._path(arch, bsp))
806
807	def _arch_bsp_dir_clean(self, arch, bsp):
808	if path.exists(self._path(arch, bsp)):
809	path.removeall(self._path(arch, bsp))
810
811	def _config_command(self, commands, arch, bsp):
812	if type(commands) is not list:
813	commands = [commands]
814	cmd = [path.join(self.rtems, 'configure')]
815	commands += self.config.bspopts(arch, bsp)
816	for c in commands:
817	c = c.replace('@PREFIX@', self.prefix)
818	c = c.replace('@RTEMS_VERSION@', self.rtems_version)
819	c = c.replace('@ARCH@', arch)
820	c = c.replace('@BSP@', bsp)
821	cmd += [c]
822	return ' '.join(cmd)
823
824	def _build_set(self, builds):
825	build_set = { }
826	for build in builds:
827	build_set[build] = self.config.build_options(build)
828	return build_set
829
830	def _build_dir(self, arch, bsp, build):
831	return path.join(self._path(arch, bsp), build)
832
833	def _count_files(self, arch, bsp, build):
834	counts = { 'h' : 0,
835	'exes' : 0,
836	'objs' : 0,
837	'libs' : 0 }
838	for root, dirs, files in os.walk(self._build_dir(arch, bsp, build)):
839	for file in files:
840	if file.endswith('.exe'):
841	counts['exes'] += 1
842	elif file.endswith('.o'):
843	counts['objs'] += 1
844	elif file.endswith('.a'):
845	counts['libs'] += 1
846	elif file.endswith('.h'):
847	counts['h'] += 1
848	for f in self.counts:
849	if f in counts:
850	self.counts[f] += counts[f]
851	return counts
852
853	def _have_failures(self, fails):
854	return len(fails) != 0
855
856	def _warnings_report(self):
857	if self.options['warnings-report'] is not None:
858	with open(self.options['warnings-report'], 'w') as f:
859	f.write(title() + os.linesep)
860	f.write(os.linesep)
861	f.write('Date: %s%s' % (datetime.date.today().strftime('%c'), os.linesep))
862	f.write(os.linesep)
863	f.write(command_line() + os.linesep)
864	f.write(self.warnings_errors.report())
865
866	def _finished(self):
867	log.notice('+ warnings:%d exes:%d objs:%d libs:%d' % \
868	(self.warnings_errors.get_warning_count(), self.counts['exes'],
869	self.counts['objs'], self.counts['libs']))
870	log.output()
871	log.output('Warnings:')
872	log.output(self.warnings_errors.report())
873	log.output()
874	log.notice('Failures:')
875	log.notice(self.failures_report(self.errors['fails']))
876	self._warnings_report()
877
878	def failures_report(self, fails):
879	if not self._have_failures(fails):
880	return ' No failure(s)'
881	absize = 0
882	bsize = 0
883	ssize = 0
884	for f in fails:
885	arch_bsp = '%s/%s' % (f[1], f[2])
886	if len(arch_bsp) > absize:
887	absize = len(arch_bsp)
888	if len(f[3]) > bsize:
889	bsize = len(f[3])
890	if len(f[0]) > ssize:
891	ssize = len(f[0])
892	fc = 1
893	s = ''
894	for f in fails:
895	fcl = '%3d' % (fc)
896	arch_bsp = '%s/%s' % (f[1], f[2])
897	state = f[0]
898	s1 = '%s %-s %-s %-*s ' % (fcl, bsize, f[3], absize, arch_bsp, ssize, state)
899	s += wrap((s1, f[4]), lineend = '\\')
900	s1 = ' ' * (len(s1) + 2)
901	for e in self.warnings_errors.get_error_messages(f[1], f[2], f[3]):
902	s += wrap([s1, 'error: ' + e])
903	fc += 1
904	return s
905
906	def build_arch_bsp(self, arch, bsp):
907	if not self.config.bsp_present(arch, bsp):
908	raise error.general('BSP not found: %s/%s' % (arch, bsp))
909	log.output('-' * 70)
910	log.notice('] BSP: %s/%s' % (arch, bsp))
911	log.notice('. Creating: %s' % (self._path(arch, bsp)))
912	self._arch_bsp_dir_clean(arch, bsp)
913	self._arch_bsp_dir_make(arch, bsp)
914	builds = self._builds(arch, bsp)
915	if builds is None:
916	raise error.general('build not found: %s' % (self._build()))
917	build_set = self._build_set(builds)
918	bsp_start = datetime.datetime.now()
919	env_path = os.environ['PATH']
920	os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
921	os.pathsep + os.environ['PATH']
922	fails = []
923	for bs in sorted(build_set.keys()):
924	self.warnings_errors.set_build(arch, bsp, bs)
925	start = datetime.datetime.now()
926	log.output('- ' * 35)
927	log.notice('. Configuring: %s' % (bs))
928	try:
929	warnings = self.warnings_errors.get_warning_count()
930	result = '+ Pass'
931	bpath = self._build_dir(arch, bsp, bs)
932	good = True
933	error_messages = None
934	path.mkdir(bpath)
935	config_cmd = self._config_command(build_set[bs], arch, bsp)
936	cmd = config_cmd
937	e = execute.capture_execution(log = self.warnings_errors)
938	log.output(wrap(('run: ', cmd), lineend = '\\'))
939	if self.options['dry-run']:
940	exit_code = 0
941	else:
942	exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
943	if exit_code != 0:
944	result = '- FAIL'
945	failure = ('configure', arch, bsp, bs, config_cmd)
946	fails += [failure]
947	self.errors['configure'] += 1
948	self.errors['fails'] += [failure]
949	log.notice('- Configure failed: %s' % (bs))
950	log.output('cmd failed: %s' % (cmd))
951	good = False
952	else:
953	log.notice('. Building: %s' % (bs))
954	cmd = 'make'
955	if 'jobs' in self.options:
956	cmd += ' -j %s' % (self.options['jobs'])
957	log.output('run: ' + cmd)
958	if self.options['dry-run']:
959	exit_code = 0
960	else:
961	exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
962	if exit_code != 0:
963	error_messages = self.warnings_errors.get_error_messages()
964	result = '- FAIL'
965	failure = ('build', arch, bsp, bs, config_cmd, error_messages)
966	fails += [failure]
967	self.errors['build'] += 1
968	self.errors['fails'] += [failure]
969	log.notice('- FAIL: %s: %s' % (bs, self._error_str()))
970	log.output('cmd failed: %s' % (cmd))
971	good = False
972	files = self._count_files(arch, bsp, bs)
973	log.notice('%s: %s: warnings:%d exes:%d objs:%s libs:%d' % \
974	(result, bs,
975	self.warnings_errors.get_warning_count() - warnings,
976	files['exes'], files['objs'], files['libs']))
977	log.notice(' %s' % (self._error_str()))
978	self.results.add(good, arch, bsp, config_cmd,
979	self.warnings_errors.get_warning_count() - warnings,
980	error_messages)
981	if not good and self.options['stop-on-error']:
982	raise error.general('Configuring %s failed' % (bs))
983	finally:
984	end = datetime.datetime.now()
985	if not self.options['no-clean']:
986	log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs)))
987	path.removeall(self._build_dir(arch, bsp, bs))
988	log.notice('^ Time %s' % (str(end - start)))
989	self.warnings_errors.clear_build()
990	bsp_end = datetime.datetime.now()
991	log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start)))
992	log.output('Failure Report:')
993	log.output(self.failures_report(fails))
994	os.environ['PATH'] = env_path
995
996	def build_bsp(self, arch, bsp):
997	self.build_arch_bsp(arch, bsp)
998	self._finished()
999
1000	def build_arch(self, arch):
1001	start = datetime.datetime.now()
1002	log.output('=' * 70)
1003	log.notice(']] Architecture: %s' % (arch))
1004	if not self.config.arch_present(arch):
1005	raise error.general('Architecture not found: %s' % (arch))
1006	for bsp in self._bsps(arch):
1007	self.build_arch_bsp(arch, bsp)
1008	end = datetime.datetime.now()
1009	log.notice('^ Architecture Time %s' % (str(end - start)))
1010	self._finished()
1011
1012	def build(self):
1013	for arch in self.config.archs():
1014	self.build_arch(arch)
1015	log.notice('^ Profile Time %s' % (str(end - start)))
1016	self._finished()
1017
1018	def build_profile(self, profile):
1019	if not self.config.profile_present(profile):
1020	raise error.general('Profile not found: %s' % (profile))
1021	start = datetime.datetime.now()
1022	log.notice(']] Profile: %s' % (profile))
1023	for arch in self.config.profile_archs(profile):
1024	for bsp in self.config.profile_arch_bsps(profile, arch):
1025	self.build_arch_bsp(arch, bsp)
1026	end = datetime.datetime.now()
1027	log.notice('^ Profile Time %s' % (str(end - start)))
1028	self._finished()
1029
1030	def run_args(args):
1031	b = None
1032	ec = 0
1033	try:
1034	#
1035	# On Windows MSYS2 prepends a path to itself to the environment
1036	# path. This means the RTEMS specific automake is not found and which
1037	# breaks the bootstrap. We need to remove the prepended path. Also
1038	# remove any ACLOCAL paths from the environment.
1039	#
1040	if os.name == 'nt':
1041	cspath = os.environ['PATH'].split(os.pathsep)
1042	if 'msys' in cspath[0] and cspath[0].endswith('bin'):
1043	os.environ['PATH'] = os.pathsep.join(cspath[1:])
1044
1045	top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
1046	prefix = '/opt/rtems/%s' % (rtems_version())
1047	tools = prefix
1048	build_dir = 'bsp-builds'
1049	logf = 'bsp-build-%s.txt' % \
1050	(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
1051	config_file = path.join(top, 'share', 'rtems', 'tester',
1052	'rtems', 'rtems-bsps.ini')
1053	if not path.exists(config_file):
1054	config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')
1055
1056	argsp = argparse.ArgumentParser()
1057	argsp.add_argument('--prefix', help = 'Prefix to build the BSP.',
1058	type = str)
1059	argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.',
1060	type = str)
1061	argsp.add_argument('--rtems', help = 'The RTEMS source tree.',
1062	type = str)
1063	argsp.add_argument('--config-report', help = 'Report the configuration.',
1064	action = 'store_true')
1065	argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
1066	type = str, default = None)
1067	argsp.add_argument('--build-path', help = 'Path to build in.',
1068	type = str)
1069	argsp.add_argument('--log', help = 'Log file.', type = str)
1070	argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
1071	action = 'store_true')
1072	argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
1073	action = 'store_true')
1074	argsp.add_argument('--profiles', help = 'Build the listed profiles.',
1075	type = str, default = 'tier-1')
1076	argsp.add_argument('--build', help = 'Build name to build.',
1077	type = str, default='all')
1078	argsp.add_argument('--arch', help = 'Build the specific architecture.',
1079	type = str)
1080	argsp.add_argument('--bsp', help = 'Build the specific BSP.',
1081	type = str)
1082	argsp.add_argument('--jobs', help = 'Number of jobs to run.',
1083	type = int, default = host.cpus())
1084	argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
1085	action = 'store_true')
1086
1087	opts = argsp.parse_args(args[1:])
1088	if opts.log is not None:
1089	logf = opts.log
1090	log.default = log.log([logf])
1091	log.notice(title())
1092	log.output(command_line())
1093	if opts.rtems is None:
1094	raise error.general('No RTEMS source provided on the command line')
1095	if opts.prefix is not None:
1096	prefix = path.shell(opts.prefix)
1097	if opts.rtems_tools is not None:
1098	tools = path.shell(opts.rtems_tools)
1099	if opts.build_path is not None:
1100	build_dir = path.shell(opts.build_path)
1101	if opts.bsp is not None and opts.arch is None:
1102	raise error.general('BSP provided but no architecture')
1103
1104	config = configuration()
1105	config.load(config_file, opts.build)
1106
1107	if opts.config_report:
1108	log.notice('Configuration Report:')
1109	log.notice(config.report())
1110	sys.exit(0)
1111
1112	options = { 'stop-on-error' : opts.stop_on_error,
1113	'no-clean' : opts.no_clean,
1114	'dry-run' : opts.dry_run,
1115	'jobs' : opts.jobs,
1116	'warnings-report' : opts.warnings_report }
1117
1118	b = build(config, rtems_version(), prefix, tools,
1119	path.shell(opts.rtems), build_dir, options)
1120	if opts.arch is not None:
1121	if opts.bsp is not None:
1122	b.build_bsp(opts.arch, opts.bsp)
1123	else:
1124	b.build_arch(opts.arch)
1125	else:
1126	for profile in opts.profiles.split(','):
1127	b.build_profile(profile.strip())
1128
1129	except error.general as gerr:
1130	print(gerr)
1131	print('BSP Build FAILED', file = sys.stderr)
1132	ec = 1
1133	except error.internal as ierr:
1134	print(ierr)
1135	print('BSP Build FAILED', file = sys.stderr)
1136	ec = 1
1137	except error.exit as eerr:
1138	pass
1139	except KeyboardInterrupt:
1140	log.notice('abort: user terminated')
1141	ec = 1
1142	if b is not None:
1143	b.results.report()
1144	sys.exit(ec)
1145
1146	def run():
1147	run_args(sys.argv)
1148
1149	if __name__ == "__main__":
1150	run()

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: rtems-tools/tester/rt/check.py @ 5d1edd5

Download in other formats: