Context Navigation

source: rtems-tools/tester/rt/check.py @ 005f995

5

Last change on this file since 005f995 was 005f995, checked in by Chris Johns <chrisj@…>, on 01/31/18 at 23:51:54

rtems-bsp-builder: Remove stray %s from the run log message.

Close #3278

Property mode set to 100755

File size: 62.4 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2016-2018 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Redistribution and use in source and binary forms, with or without
9	# modification, are permitted provided that the following conditions are met:
10	#
11	# 1. Redistributions of source code must retain the above copyright notice,
12	# this list of conditions and the following disclaimer.
13	#
14	# 2. Redistributions in binary form must reproduce the above copyright notice,
15	# this list of conditions and the following disclaimer in the documentation
16	# and/or other materials provided with the distribution.
17	#
18	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21	# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22	# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23	# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24	# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25	# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26	# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27	# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28	# POSSIBILITY OF SUCH DAMAGE.
29	#
30
31	from __future__ import print_function
32
33	import argparse
34	import copy
35	import datetime
36	import operator
37	import os
38	import re
39	import sys
40	import textwrap
41	import threading
42	import time
43	import traceback
44
45	import pprint
46
47	from rtemstoolkit import configuration
48	from rtemstoolkit import execute
49	from rtemstoolkit import error
50	from rtemstoolkit import host
51	from rtemstoolkit import log
52	from rtemstoolkit import mailer
53	from rtemstoolkit import path
54	from rtemstoolkit import textbox
55	from rtemstoolkit import version
56
57	#
58	# Group loggin entries together.
59	#
60	log_lock = threading.Lock()
61
62	#
63	# The max build label size in the jobs list.
64	#
65	max_build_label = 0
66
67	def rtems_version():
68	return version.version()
69
70	def wrap(line, lineend = '', indent = 0, width = 75):
71	if type(line) is tuple or type(line) is list:
72	if len(line) >= 2:
73	s1 = line[0]
74	else:
75	s1 = ''
76	s2 = line[1:]
77	elif type(line) is str:
78	s1 = ''
79	s2 = [line]
80	else:
81	raise error.internal('line is not a tuple, list or string')
82	s1len = len(s1)
83	s = ''
84	first = True
85	for ss in s2:
86	if type(ss) is not str and type(ss) is not unicode:
87	raise error.internal('text needs to be a string')
88	for l in textwrap.wrap(ss, width = width - s1len - indent - 1):
89	s += '%s%s%s%s%s' % (' ' * indent, s1, l, lineend, os.linesep)
90	if first and s1len > 0:
91	s1 = ' ' * s1len
92	if lineend != '':
93	s = s[:0 - len(os.linesep) - 1] + os.linesep
94	return s
95
96	def comma_split(options):
97	if options is None:
98	return None
99	return [o.strip() for o in options.split(',')]
100
101	def title():
102	return 'RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str())
103
104	def command_line():
105	return wrap(('command: ', ' '.join(sys.argv)), lineend = '\\')
106
107	def jobs_option_parse(jobs_option):
108	try:
109	if '/' not in jobs_option:
110	return 1, int(jobs_option)
111	jos = jobs_option.split('/')
112	if len(jos) != 2:
113	raise error.general('invalid jobs option: %s' % (jobs_option))
114	return int(jos[0]), int(jos[1])
115	except:
116	pass
117	raise error.general('invalid jobs option: %s' % (jobs_option))
118
119	def arch_bsp_build_parse(build):
120	if type(build) is str:
121	build_key = build
122	else:
123	build_key = build.key()
124	abb = build_key.split('.')
125	if len(abb) != 2:
126	raise error.general('invalid build key: %s' % (build_key))
127	ab = abb[0].split('/')
128	if len(ab) != 2:
129	raise error.general('invalid build key: %s' % (build_key))
130	return ab[0], ab[1], abb[1]
131
132	def set_max_build_label(jobs):
133	global max_build_label
134	for job in jobs:
135	if len(job.build.key()) > max_build_label:
136	max_build_label = len(job.build.key())
137	max_build_label += 2
138
139	class arch_bsp_build:
140
141	def __init__(self, arch, bsp, build, build_config):
142	self.arch = arch
143	self.bsp = bsp
144	self.build = build
145	self.build_config = build_config
146	self.start_time = None
147	self.stop_time = None
148
149	def __str__(self):
150	return self.key() + ': ' + self.build_config
151
152	def key(self):
153	return '%s/%s.%s' % (self.arch, self.bsp, self.build)
154
155	def get_arch_bsp(self):
156	return self.arch, self.bsp
157
158	def start(self):
159	self.start_time = datetime.datetime.now()
160
161	def stop(self):
162	self.stop_time = datetime.datetime.now()
163
164	def duration(self):
165	return self.stop_time - self.start_time
166
167	class output_worker:
168
169	def __init__(self, we, build):
170	self.text = []
171	self.warnings_errors = we
172	self.build = build
173
174	def output(self, text):
175	self.warnings_errors.process_output(text, self.build)
176	self.text += text.splitlines()
177
178	def log_output(self, heading):
179	log_lock.acquire()
180	try:
181	log.output(heading + self.text)
182	except:
183	raise
184	finally:
185	log_lock.release()
186
187	class warnings_errors:
188
189	def __init__(self, source_base, groups):
190	self.lock = threading.Lock()
191	self.source_base = path.host(source_base)
192	self.groups = groups
193	self.reset()
194
195	def _get_warnings(self, build):
196	self.lock.acquire()
197	warnings = [w for w in self.warnings]
198	self.lock.release()
199	return sorted(warnings)
200
201	def _total(self, archive):
202	total = 0
203	for a in archive:
204	total += archive[a]
205	return total
206
207	def _analyze(self, warnings, exclude):
208	def _group(data, category, name, warning, count, groups, group_regx):
209	if 'groups' not in data:
210	data['groups'] = { }
211	if category not in data['groups']:
212	data['groups'][category] = { }
213	if 'totals' not in data['groups'][category]:
214	data['groups'][category]['totals'] = { }
215	if name not in data['groups'][category]:
216	data['groups'][category][name] = { }
217	for group in groups:
218	if group not in data['groups'][category]['totals']:
219	data['groups'][category]['totals'][group] = 0
220	if group not in data['groups'][category][name]:
221	data['groups'][category][name][group] = 0
222	if group_regx[group].match(warning):
223	data['groups'][category][name][group] += count
224	data['groups'][category]['totals'][group] += count
225	break
226
227	def _update(data, category, name, warning, count, groups, group_regx):
228	if category not in data:
229	data[category] = { }
230	if name not in data[category]:
231	data[category][name] = { }
232	if warning not in data[category][name]:
233	data[category][name][warning] = 0
234	data[category][name][warning] += count
235	_group(data, category, name, w, count, groups, group_regx)
236
237	categories = ['arch', 'arch_bsp', 'build']
238	data = { 'groups': { } }
239	for category in categories:
240	data[category] = { }
241	data['groups'][category] = { }
242	group_regx = { }
243	for group in self.groups['groups']:
244	group_regx[group] = re.compile(self.groups[group])
245	exclude_regx = re.compile(exclude)
246	for warning in self.warnings:
247	arch, bsp, build = arch_bsp_build_parse(warning)
248	arch_bsp = '%s/%s' % (arch, bsp)
249	for w in self.warnings[warning]:
250	if not exclude_regx.match(w):
251	count = self.warnings[warning][w]
252	_update(data, 'arch', arch, w, count,
253	self.groups['groups'], group_regx)
254	_update(data, 'arch_bsp', arch_bsp, w, count,
255	self.groups['groups'], group_regx)
256	_update(data, 'build', build, w, count,
257	self.groups['groups'], group_regx)
258	for category in categories:
259	common = {}
260	for name in data[category]:
261	for w in data[category][name]:
262	if w not in common:
263	for other in [n for n in data[category] if n != name]:
264	if w in data[category][other]:
265	common[w] = data[category][name][w]
266	_group(data, category, 'common', w, common[w],
267	self.groups['groups'], group_regx)
268	data[category]['common'] = common
269	return data
270
271	def _report_category(self, label, warnings, group_counts, summary):
272	width = 70
273	cols_1 = [width]
274	cols_2 = [8, width - 8]
275	cols_4 = textbox.even_columns(4, width)
276	cols_2_4 = textbox.merge_columns([cols_2, cols_4])
277	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
278	s += textbox.row(cols_1, [' ' + label], indent = 1)
279	s += textbox.line(cols_1, marker = '+', indent = 1)
280	builds = ['common'] + sorted([b for b in warnings if b != 'common'])
281	common = warnings['common']
282	for build in builds:
283	build_warnings = warnings[build]
284	if build is not 'common':
285	build_warnings = [w for w in build_warnings if w not in common]
286	s += textbox.row(cols_1,
287	[' %s : %d warning(s)' % (build,
288	len(build_warnings))],
289	indent = 1)
290	if len(build_warnings) == 0:
291	s += textbox.line(cols_1, marker = '+', indent = 1)
292	else:
293	s += textbox.line(cols_4, marker = '+', indent = 1)
294	if build not in group_counts:
295	gs = [0 for group in self.groups['groups']]
296	else:
297	gs = []
298	for g in range(0, len(self.groups['groups'])):
299	group = self.groups['groups'][g]
300	if group in group_counts[build]:
301	count = group_counts[build][group]
302	else:
303	count = 0
304	gs += ['%*s' % (cols_4[g % 4] - 2,
305	'%s : %4d' % (group, count))]
306	for row in range(0, len(self.groups['groups']), 4):
307	if row + 4 > len(self.groups['groups']):
308	d = gs[row:] + \
309	['' for r in range(row,
310	len(self.groups['groups']))]
311	else:
312	d = gs[row:+4]
313	s += textbox.row(cols_4, d, indent = 1)
314	s += textbox.line(cols_2_4, marker = '+', indent = 1)
315	if not summary:
316	vw = sorted([(w, warnings[build][w]) for w in build_warnings],
317	key = operator.itemgetter(1),
318	reverse = True)
319	for w in vw:
320	c1 = '%6d' % w[1]
321	for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3):
322	s += textbox.row(cols_2, [c1, l], indent = 1)
323	c1 = ' ' * 6
324	s += textbox.line(cols_2, marker = '+', indent = 1)
325	return s
326
327	def _report_warning_map(self):
328	builds = self.messages['warnings']
329	width = 70
330	cols_1 = [width]
331	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
332	s += textbox.row(cols_1, [' Warning Map'], indent = 1)
333	s += textbox.line(cols_1, marker = '+', indent = 1)
334	for build in builds:
335	messages = builds[build]
336	s += textbox.row(cols_1, [' %s : %d' % (build, len(messages))], indent = 1)
337	s += textbox.line(cols_1, marker = '+', indent = 1)
338	for msg in messages:
339	for l in textwrap.wrap(msg, width = width - 3):
340	s += textbox.row(cols_1, [' ' + l], indent = 1)
341	for l in textwrap.wrap(messages[msg], width = width - 3 - 4):
342	s += textbox.row(cols_1, [' ' + l], indent = 1)
343	s += textbox.line(cols_1, marker = '+', indent = 1)
344	return s
345
346	def warnings_report(self, summary = False):
347	self.lock.acquire()
348	s = ' No warnings' + os.linesep
349	try:
350	total = 0
351	for build in self.warnings:
352	total += self._total(self.warnings[build])
353	if total != 0:
354	data = self._analyze(self.warnings, self.groups['exclude'])
355	s = self._report_category('By Architecture (total : %d)' % (total),
356	data['arch'], data['groups']['arch'],
357	summary)
358	s += os.linesep
359	s += self._report_category('By BSP (total : %d)' % (total),
360	data['arch_bsp'], data['groups']['arch_bsp'],
361	summary)
362	s += os.linesep
363	s += self._report_category('By Build (total : %d)' % (total),
364	data['build'], data['groups']['build'],
365	summary)
366	s += os.linesep
367	if not summary:
368	s += self._report_warning_map()
369	s += os.linesep
370	finally:
371	self.lock.release()
372	return s
373
374	def clear_build(self, build):
375	self.lock.acquire()
376	self.warnings[build.key()] = {}
377	self.errors[build.key()] = {}
378	self.lock.release()
379
380	def get_warning_count(self):
381	self.lock.acquire()
382	count = self.warning_count
383	self.lock.release()
384	return count
385
386	def get_error_count(self):
387	self.lock.acquire()
388	count = self.error_count
389	self.lock.release()
390	return count
391
392	def reset(self):
393	self.lock.acquire()
394	self.warnings = { }
395	self.warning_count = 0
396	self.errors = { }
397	self.error_count = 0
398	self.messages = { 'warnings' : { }, 'errors' : { } }
399	self.lock.release()
400
401	def _get_messages(self, build, key):
402	self.lock.acquire()
403	if type(build) is str:
404	build_key = build
405	else:
406	build_key = build.key()
407	if build_key not in self.messages[key]:
408	messages = []
409	else:
410	messages = self.messages[key][build_key]
411	messages = ['%s %s' % (m, messages[m]) for m in messages]
412	self.lock.release()
413	return messages
414
415	def get_warning_messages(self, build):
416	return self._get_messages(build, 'warning')
417
418	def get_error_messages(self, build):
419	return self._get_messages(build, 'errors')
420
421	def process_output(self, text, build):
422	def _line_split(line, source_base):
423	if line.count(':') < 2:
424	return None
425	ls = line.split(' ', 1)
426	fname = ls[0].strip().split(':', 2)
427	if len(fname) != 3:
428	return None
429	p = path.abspath(fname[0])
430	p = p.replace(source_base, '')
431	if path.isabspath(p):
432	p = p[1:]
433	if len(fname[2]) == 0:
434	pos = None
435	else:
436	pos = fname[2]
437	return p, fname[1], pos, ls[1]
438
439	def _create_build_errors(build, archive):
440	if build.key() not in archive:
441	archive[build.key()] = { }
442	return archive[build.key()]
443
444	#
445	# The GNU linker does not supply 'error' in error messages. There is no
446	# line information which is understandable. Look for `bin/ld:` and
447	# `collect2:` in the output and then create the error when `collect2:`
448	# is seen.
449	#
450	# The order we inspect each line is important.
451	#
452	if ' warning:' in text or \
453	' error:' in text or \
454	' Error:' in text or \
455	'bin/ld:' in text:
456	self.lock.acquire()
457	try:
458	for l in text.splitlines():
459	if 'bin/ld:' in l:
460	archive =_create_build_errors(build, self.errors)
461	if 'linker' not in archive:
462	archive['linker'] = []
463	archive['linker'] += [l.split(':', 1)[1].strip()]
464	elif l.startswith('collect2:'):
465	archive =_create_build_errors(build, self.errors)
466	l = '/ld/collect2:0: error: '
467	if 'linker' not in archive or len(archive['linker']) == 0:
468	l += 'no error message found!'
469	else:
470	l += '; '.join(archive['linker'])
471	archive['linker'] = []
472	messages = 'errors'
473	elif ' warning:' in l:
474	self.warning_count += 1
475	archive = _create_build_errors(build, self.warnings)
476	messages = 'warnings'
477	elif ' error:' in l or ' Error:' in l:
478	self.error_count += 1
479	archive =_create_build_errors(build, self.errors)
480	messages = 'errors'
481	else:
482	continue
483	line_parts = _line_split(l, self.source_base)
484	if line_parts is not None:
485	src, line, pos, msg = line_parts
486	if pos is not None:
487	where = '%s:%s:%s' % (src, line, pos)
488	else:
489	where = '%s:%s' % (src, line)
490	if where not in archive:
491	archive[where] = 1
492	else:
493	archive[where] += 1
494	if build.key() not in self.messages[messages]:
495	self.messages[messages][build.key()] = { }
496	self.messages[messages][build.key()][where] = msg
497	finally:
498	self.lock.release()
499
500	class results:
501
502	def __init__(self, source_base, groups):
503	self.lock = threading.Lock()
504	self.errors = { 'pass': 0,
505	'configure': 0,
506	'build': 0,
507	'tests': 0,
508	'passes': { },
509	'fails': { } }
510	self.counts = { 'h' : 0,
511	'exes' : 0,
512	'objs' : 0,
513	'libs' : 0 }
514	self.warnings_errors = warnings_errors(source_base, groups)
515
516	def _arch_bsp(self, arch, bsp):
517	return '%s/%s' % (arch, bsp)
518
519	def _arch_bsp_passes(self, build):
520	if build.key() not in self.errors['passes']:
521	self.errors['passes'][build.key()] = []
522	return self.errors['passes'][build.key()]
523
524	def _arch_bsp_fails(self, build):
525	if build.key() not in self.errors['fails']:
526	self.errors['fails'][build.key()] = []
527	return self.errors['fails'][build.key()]
528
529	def _count(self, label):
530	count = 0
531	for build in self.errors[label]:
532	count += len(self.errors[label][build])
533	return count
534
535	def _max_col(self, label):
536	max_col = 0
537	for build in self.errors[label]:
538	arch, bsp, build_config = arch_bsp_build_parse(build)
539	arch_bsp = self._arch_bsp(arch, bsp)
540	if len(arch_bsp) > max_col:
541	max_col = len(arch_bsp)
542	return max_col
543
544	def get_warning_count(self):
545	return self.warnings_errors.get_warning_count()
546
547	def get_error_count(self):
548	return self.warnings_errors.get_error_count()
549
550	def get_warning_messages(self, build):
551	return self.warnings_errors.get_warning_messages(build)
552
553	def get_error_messages(self, build):
554	return self.warnings_errors.get_error_messages(build)
555
556	def status(self):
557	self.lock.acquire()
558	try:
559	s = 'Pass: %4d Fail: %4d (configure:%d build:%d)' % \
560	(self.errors['pass'],
561	self.errors['configure'] + self.errors['build'],
562	self.errors['configure'], self.errors['build'])
563	except:
564	raise
565	finally:
566	self.lock.release()
567	return s;
568
569	def add_fail(self, phase, build, configure, warnings, error_messages):
570	fails = self._arch_bsp_fails(build)
571	self.lock.acquire()
572	try:
573	self.errors[phase] += 1
574	fails += [(phase, build.build_config, configure, error_messages)]
575	finally:
576	self.lock.release()
577
578	def add_pass(self, build, configure, warnings):
579	passes = self._arch_bsp_passes(build)
580	self.lock.acquire()
581	try:
582	self.errors['pass'] += 1
583	passes += [(build.build_config, configure, warnings, None)]
584	finally:
585	self.lock.release()
586
587	def pass_count(self):
588	return self._count('passes')
589
590	def fail_count(self):
591	return self._count('fails')
592
593	def _failures_report(self, build, count):
594	if type(build) is str:
595	build_key = build
596	else:
597	build_key = build.key()
598	if build_key not in self.errors['fails'] or \
599	len(self.errors['fails'][build_key]) == 0:
600	return count, 0, ' No failures'
601	absize = 0
602	bsize = 0
603	ssize = 0
604	arch, bsp, build_set = arch_bsp_build_parse(build_key)
605	arch_bsp = self._arch_bsp(arch, bsp)
606	fails = self.errors['fails'][build_key]
607	for f in fails:
608	if len(f[0]) > ssize:
609	ssize = len(f[0])
610	s = ''
611	for f in fails:
612	count += 1
613	fcl = ' %3d' % (count)
614	state = f[0]
615	s += '%s %s %s %-*s:%s' % \
616	(fcl, build_set, arch_bsp, ssize, state, os.linesep)
617	s1 = ' ' * 6
618	s += wrap((s1, 'configure: ' + f[2]), lineend = '\\', width = 75)
619	s1 = ' ' * 5
620	for e in self.warnings_errors.get_error_messages(build):
621	s += wrap([s1 + 'error: ', e])
622	return count, len(fails), s
623
624	def failures_report(self, build = None):
625	s = ''
626	count = 0
627	if build is not None:
628	count, build_fails, bs = self._failures_report(build, count)
629	if build_fails > 0:
630	s += bs + os.linesep
631	else:
632	self.lock.acquire()
633	builds = sorted(self.errors['fails'].keys())
634	self.lock.release()
635	for build in builds:
636	count, build_fails, bs = self._failures_report(build, count)
637	if build_fails > 0:
638	s += bs + os.linesep
639	if count == 0:
640	s = ' No failures' + os.linesep
641	return s
642
643	def warnings_report(self, summary = False):
644	return self.warnings_errors.warnings_report(summary)
645
646	def report(self):
647	self.lock.acquire()
648	log_lock.acquire()
649	passes = self.pass_count()
650	fails = self.fail_count()
651	log.notice('Passes: %d Failures: %d' % (passes, fails))
652	log.output()
653	log.output('Build Report')
654	log.output(' Passes: %d Failures: %d' % (passes, fails))
655	log.output(' Failures:')
656	if fails == 0:
657	log.output(' None')
658	else:
659	max_col = self._max_col('fails')
660	for build in self.errors['fails']:
661	arch, bsp, build_config = arch_bsp_build_parse(build)
662	arch_bsp = self._arch_bsp(arch, bsp)
663	for f in self.errors['fails'][build]:
664	config_cmd = f[2]
665	config_at = config_cmd.find('configure')
666	if config_at != -1:
667	config_cmd = config_cmd[config_at:]
668	log.output(' %*s:' % (max_col + 2, arch_bsp))
669	s1 = ' ' * 6
670	log.output(wrap([s1, config_cmd], lineend = '\\', width = 75))
671	if f[3] is not None:
672	s1 = ' ' * len(s1)
673	for msg in f[3]:
674	log.output(wrap([s1, msg], lineend = '\\'))
675	log.output(' Passes:')
676	if passes == 0:
677	log.output(' None')
678	else:
679	max_col = self._max_col('passes')
680	for build in self.errors['passes']:
681	arch, bsp, build_config = arch_bsp_build_parse(build)
682	arch_bsp = self._arch_bsp(arch, bsp)
683	for f in self.errors['passes'][build]:
684	config_cmd = f[1]
685	config_at = config_cmd.find('configure')
686	if config_at != -1:
687	config_cmd = config_cmd[config_at:]
688	log.output(' %s (%5d):' % (arch_bsp, f[2]))
689	log.output(wrap([' ' * 6, config_cmd], lineend = '\\', width = 75))
690	log_lock.release()
691	self.lock.release()
692
693	class arch_bsp_builder:
694
695	def __init__(self, results_, build, commands, build_dir, tag):
696	self.lock = threading.Lock()
697	self.state = 'ready'
698	self.thread = None
699	self.proc = None
700	self.results = results_
701	self.build = build
702	self.commands = commands
703	self.build_dir = build_dir
704	self.tag = tag
705	self.output = output_worker(results_.warnings_errors, build)
706	self.counts = { 'h' : 0,
707	'exes' : 0,
708	'objs' : 0,
709	'libs' : 0 }
710
711	def _notice(self, text):
712	global max_build_label
713	arch, bsp, build_set = arch_bsp_build_parse(self.build.key())
714	label = '%s/%s (%s)' % (arch, bsp, build_set)
715	log.notice('[%s] %-*s %s' % (self.tag, max_build_label, label, text))
716
717	def _build_dir(self):
718	return path.join(self.build_dir, self.build.key())
719
720	def _make_build_dir(self):
721	if not path.exists(self._build_dir()):
722	path.mkdir(self._build_dir())
723
724	def _count_files(self):
725	for root, dirs, files in os.walk(self._build_dir()):
726	for file in files:
727	if file.endswith('.exe'):
728	self.counts['exes'] += 1
729	elif file.endswith('.o'):
730	self.counts['objs'] += 1
731	elif file.endswith('.a'):
732	self.counts['libs'] += 1
733	elif file.endswith('.h'):
734	self.counts['h'] += 1
735
736	def _execute(self, phase):
737	exit_code = 0
738	cmd = self.commands[phase]
739	try:
740	# This should locked; not sure how to do that
741	self.proc = execute.capture_execution(log = self.output)
742	log.output(wrap(('run:', self.build.key(), cmd), lineend = '\\'))
743	if not self.commands['dry-run']:
744	exit_code, proc, output = self.proc.shell(cmd,
745	cwd = path.host(self._build_dir()))
746	except:
747	traceback.print_exc()
748	self.lock.acquire()
749	if self.proc is not None:
750	self.proc.kill()
751	self.lock.release()
752	exit_code = 1
753	self.lock.acquire()
754	self.proc = None
755	self.lock.release()
756	return exit_code == 0
757
758	def _configure(self):
759	return self._execute('configure')
760
761	def _make(self):
762	return self._execute('build')
763
764	def _worker(self):
765	self.lock.acquire()
766	self.state = 'running'
767	self.lock.release()
768	self.build.start()
769	warnings = self.results.get_warning_count()
770	ok = False
771	try:
772	log_lock.acquire()
773	try:
774	self._notice('Start')
775	self._notice('Creating: %s' % (self._build_dir()))
776	except:
777	raise
778	finally:
779	log_lock.release()
780	self._make_build_dir()
781	self._notice('Configuring')
782	ok = self._configure()
783	if not ok:
784	warnings = self.results.get_warning_count() - warnings
785	self.results.add_fail('configure',
786	self.build,
787	self.commands['configure'],
788	warnings,
789	self.results.get_error_messages(self.build))
790	self.lock.acquire()
791	if self.state == 'killing':
792	ok = False
793	self.lock.release()
794	if ok:
795	self._notice('Building')
796	ok = self._make()
797	if not ok:
798	warnings = self.results.get_warning_count() - warnings
799	self.results.add_fail('build',
800	self.build,
801	self.commands['configure'],
802	warnings,
803	self.results.get_error_messages(self.build))
804	if ok:
805	warnings = self.results.get_warning_count() - warnings
806	self.results.add_pass(self.build,
807	self.commands['configure'],
808	warnings)
809	except:
810	ok = False
811	self._notice('Build Exception')
812	traceback.print_exc()
813	self.build.stop()
814	log_lock.acquire()
815	try:
816	self._count_files()
817	if ok:
818	self._notice('PASS')
819	else:
820	self._notice('FAIL')
821	self._notice('Warnings:%d exes:%d objs:%d libs:%d' % \
822	(warnings, self.counts['exes'],
823	self.counts['objs'], self.counts['libs']))
824	log.output(' %s: Failure Report:' % (self.build.key()))
825	log.output(self.results.failures_report(self.build))
826	self._notice('Finished (duration:%s)' % (str(self.build.duration())))
827	self._notice('Status: %s' % (self.results.status()))
828	except:
829	self._notice('Build Exception:')
830	traceback.print_exc()
831	finally:
832	log_lock.release()
833	self.lock.acquire()
834	self.state = 'finished'
835	self.lock.release()
836
837	def get_file_counts(self):
838	return self.counts
839
840	def run(self):
841	self.lock.acquire()
842	try:
843	if self.state != 'ready':
844	raise error.general('builder already run')
845	self.state = 'starting'
846	self.thread = threading.Thread(target = self._worker)
847	self.thread.start()
848	except:
849	raise
850	finally:
851	self.lock.release()
852
853	def kill(self):
854	self.lock.acquire()
855	if self.thread is not None:
856	self.state = 'killing'
857	if self.proc is not None:
858	try:
859	self.proc.kill()
860	except:
861	pass
862	self.lock.release()
863	self.thread.join(5)
864	self.lock.acquire()
865	self.state = 'finished'
866	self.lock.release()
867
868	def current_state(self):
869	self.lock.acquire()
870	state = self.state
871	self.lock.release()
872	return state
873
874	def log_output(self):
875	self.output.log_output(['-' * 79, '] %s: Build output:' % (self.build.key())])
876
877	def clean(self):
878	if not self.commands['no-clean']:
879	self._notice('Cleaning: %s' % (self._build_dir()))
880	path.removeall(self._build_dir())
881
882	class configuration_:
883
884	def __init__(self):
885	self.config = configuration.configuration()
886	self.archs = { }
887	self.builds_ = { }
888	self.profiles = { }
889
890	def __str__(self):
891	s = self.name + os.linesep
892	s += 'Archs:' + os.linesep + \
893	pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
894	s += 'Builds:' + os.linesep + \
895	pprint.pformat(self.builds_, indent = 1, width = 80) + os.linesep
896	s += 'Profiles:' + os.linesep + \
897	pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
898	return s
899
900	def _build_options(self, build, nesting = 0):
901	if ':' in build:
902	section, name = build.split(':', 1)
903	opts = [self.config.get_item(section, name)]
904	return opts
905	builds = self.builds_['builds']
906	if build not in builds:
907	raise error.general('build %s not found' % (build))
908	if nesting > 20:
909	raise error.general('nesting build %s' % (build))
910	options = []
911	for option in self.builds_['builds'][build]:
912	if ':' in option:
913	section, name = option.split(':', 1)
914	opts = [self.config.get_item(section, name)]
915	else:
916	opts = self._build_options(option, nesting + 1)
917	for opt in opts:
918	if opt not in options:
919	options += [opt]
920	return options
921
922	def load(self, name, build):
923	self.config.load(name)
924	archs = []
925	self.profiles['profiles'] = \
926	self.config.comma_list('profiles', 'profiles', err = False)
927	if len(self.profiles['profiles']) == 0:
928	self.profiles['profiles'] = ['tier-%d' % (t) for t in range(1,4)]
929	for p in self.profiles['profiles']:
930	profile = {}
931	profile['name'] = p
932	profile['archs'] = self.config.comma_list(profile['name'], 'archs', err = False)
933	archs += profile['archs']
934	for arch in profile['archs']:
935	bsps = 'bsps_%s' % (arch)
936	profile[bsps] = self.config.comma_list(profile['name'], bsps)
937	self.profiles[profile['name']] = profile
938	invalid_chars = re.compile(r'[^a-zA-Z0-9_-]')
939	for a in set(archs):
940	if len(invalid_chars.findall(a)) != 0:
941	raise error.general('invalid character(s) in arch name: %s' % (a))
942	arch = {}
943	arch['excludes'] = {}
944	for exclude in self.config.comma_list(a, 'exclude', err = False):
945	arch['excludes'][exclude] = ['all']
946	for i in self.config.get_items(a, False):
947	if i[0].startswith('exclude-'):
948	exclude = i[0][len('exclude-'):]
949	if exclude not in arch['excludes']:
950	arch['excludes'][exclude] = []
951	arch['excludes'][exclude] += \
952	sorted(set([b.strip() for b in i[1].split(',')]))
953	arch['bsps'] = self.config.comma_list(a, 'bsps', err = False)
954	for b in arch['bsps']:
955	if len(invalid_chars.findall(b)) != 0:
956	raise error.general('invalid character(s) in BSP name: %s' % (b))
957	arch[b] = {}
958	arch[b]['bspopts'] = \
959	self.config.comma_list(a, 'bspopts_%s' % (b), err = False)
960	self.archs[a] = arch
961	builds = {}
962	builds['default'] = self.config.get_item('builds', 'default')
963	if build is None:
964	build = builds['default']
965	builds['config'] = { }
966	for config in self.config.get_items('config'):
967	builds['config'][config[0]] = config[1]
968	builds['build'] = build
969	builds_ = self.config.get_item_names('builds')
970	builds['builds'] = {}
971	for build in builds_:
972	build_builds = self.config.comma_list('builds', build)
973	has_config = False
974	has_build = False
975	for b in build_builds:
976	if ':' in b:
977	if has_build:
978	raise error.general('config and build in build: %s' % (build))
979	has_config = True
980	else:
981	if has_config:
982	raise error.general('config and build in build: %s' % (build))
983	has_build = True
984	builds['builds'][build] = build_builds
985	self.builds_ = builds
986
987	def build(self):
988	return self.builds_['build']
989
990	def builds(self):
991	if self.builds_['build'] in self.builds_['builds']:
992	build = copy.copy(self.builds_['builds'][self.builds_['build']])
993	if ':' in build[0]:
994	return [self.builds_['build']]
995	return build
996	return None
997
998	def build_options(self, build):
999	return ' '.join(self._build_options(build))
1000
1001	def excludes(self, arch):
1002	excludes = self.archs[arch]['excludes'].keys()
1003	for exclude in self.archs[arch]['excludes']:
1004	if 'all' not in self.archs[arch]['excludes'][exclude]:
1005	excludes.remove(exclude)
1006	return sorted(excludes)
1007
1008	def archs(self):
1009	return sorted(self.archs.keys())
1010
1011	def arch_present(self, arch):
1012	return arch in self.archs
1013
1014	def arch_bsps(self, arch):
1015	return sorted(self.archs[arch]['bsps'])
1016
1017	def bsp_present(self, arch, bsp):
1018	return bsp in self.archs[arch]['bsps']
1019
1020	def bsp_excludes(self, arch, bsp):
1021	excludes = self.archs[arch]['excludes'].keys()
1022	for exclude in self.archs[arch]['excludes']:
1023	if 'all' not in self.archs[arch]['excludes'][exclude] and \
1024	bsp not in self.archs[arch]['excludes'][exclude]:
1025	excludes.remove(exclude)
1026	return sorted(excludes)
1027
1028	def bspopts(self, arch, bsp):
1029	if arch not in self.archs:
1030	raise error.general('invalid architecture: %s' % (arch))
1031	if bsp not in self.archs[arch]:
1032	raise error.general('invalid BSP: %s' % (bsp))
1033	return self.archs[arch][bsp]['bspopts']
1034
1035	def profile_present(self, profile):
1036	return profile in self.profiles
1037
1038	def profile_archs(self, profile):
1039	if profile not in self.profiles:
1040	raise error.general('invalid profile: %s' % (profile))
1041	return self.profiles[profile]['archs']
1042
1043	def profile_arch_bsps(self, profile, arch):
1044	if profile not in self.profiles:
1045	raise error.general('invalid profile: %s' % (profile))
1046	if 'bsps_%s' % (arch) not in self.profiles[profile]:
1047	raise error.general('invalid profile arch: %s' % (arch))
1048	return ['%s/%s' % (arch, bsp) for bsp in self.profiles[profile]['bsps_%s' % (arch)]]
1049
1050	def report(self, profiles = True, builds = True, architectures = True):
1051	width = 70
1052	cols_1 = [width]
1053	cols_2 = [10, width - 10]
1054	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
1055	s1 = ' File(s)'
1056	for f in self.config.files():
1057	colon = ':'
1058	for l in textwrap.wrap(f, width = cols_2[1] - 3):
1059	s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1)
1060	colon = ' '
1061	s1 = ' ' * len(s1)
1062	s += textbox.line(cols_1, marker = '+', indent = 1)
1063	s += os.linesep
1064	if profiles:
1065	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
1066	profiles = sorted(self.profiles['profiles'])
1067	archs = []
1068	bsps = []
1069	for profile in profiles:
1070	archs += self.profiles[profile]['archs']
1071	for arch in sorted(self.profiles[profile]['archs']):
1072	bsps += self.profiles[profile]['bsps_%s' % (arch)]
1073	archs = len(set(archs))
1074	bsps = len(set(bsps))
1075	s += textbox.row(cols_1,
1076	[' Profiles : %d (archs:%d, bsps:%d)' % \
1077	(len(profiles), archs, bsps)],
1078	indent = 1)
1079	for profile in profiles:
1080	textbox.row(cols_2,
1081	[profile, self.profiles[profile]['name']],
1082	indent = 1)
1083	s += textbox.line(cols_1, marker = '+', indent = 1)
1084	for profile in profiles:
1085	s += textbox.row(cols_1, [' %s' % (profile)], indent = 1)
1086	profile = self.profiles[profile]
1087	archs = sorted(profile['archs'])
1088	for arch in archs:
1089	arch_bsps = ', '.join(profile['bsps_%s' % (arch)])
1090	if len(arch_bsps) > 0:
1091	s += textbox.line(cols_2, marker = '+', indent = 1)
1092	s1 = ' ' + arch
1093	for l in textwrap.wrap(arch_bsps,
1094	width = cols_2[1] - 3):
1095	s += textbox.row(cols_2, [s1, ' ' + l], indent = 1)
1096	s1 = ' ' * len(s1)
1097	s += textbox.line(cols_2, marker = '+', indent = 1)
1098	s += os.linesep
1099	if builds:
1100	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
1101	s += textbox.row(cols_1,
1102	[' Builds: %s (default)' % (self.builds_['default'])],
1103	indent = 1)
1104	builds = self.builds_['builds']
1105	bsize = 0
1106	for build in builds:
1107	if len(build) > bsize:
1108	bsize = len(build)
1109	cols_b = [bsize + 2, width - bsize - 2]
1110	s += textbox.line(cols_b, marker = '+', indent = 1)
1111	for build in builds:
1112	s1 = ' ' + build
1113	for l in textwrap.wrap(', '.join(builds[build]),
1114	width = cols_b[1] - 3):
1115	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
1116	s1 = ' ' * len(s1)
1117	s += textbox.line(cols_b, marker = '+', indent = 1)
1118	configs = self.builds_['config']
1119	s += textbox.row(cols_1,
1120	[' Configure Options: %d' % (len(configs))],
1121	indent = 1)
1122	csize = 0
1123	for config in configs:
1124	if len(config) > csize:
1125	csize = len(config)
1126	cols_c = [csize + 3, width - csize - 3]
1127	s += textbox.line(cols_c, marker = '+', indent = 1)
1128	for config in configs:
1129	s1 = ' ' + config
1130	for l in textwrap.wrap(configs[config], width = cols_c[1] - 3):
1131	s += textbox.row(cols_c, [s1, ' ' + l], indent = 1)
1132	s1 = ' ' * len(s1)
1133	s += textbox.line(cols_c, marker = '+', indent = 1)
1134	s += os.linesep
1135	if architectures:
1136	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
1137	archs = sorted(self.archs.keys())
1138	bsps = 0
1139	asize = 0
1140	for arch in archs:
1141	if len(arch) > asize:
1142	asize = len(arch)
1143	bsps += len(self.archs[arch]['bsps'])
1144	s += textbox.row(cols_1,
1145	[' Architectures : %d (bsps: %d)' % (len(archs), bsps)],
1146	indent = 1)
1147	cols_a = [asize + 2, width - asize - 2]
1148	s += textbox.line(cols_a, marker = '+', indent = 1)
1149	for arch in archs:
1150	s += textbox.row(cols_a,
1151	[' ' + arch, ' %d' % (len(self.archs[arch]['bsps']))],
1152	indent = 1)
1153	s += textbox.line(cols_a, marker = '+', indent = 1)
1154	for archn in archs:
1155	arch = self.archs[archn]
1156	if len(arch['bsps']) > 0:
1157	bsize = 0
1158	for bsp in arch['bsps']:
1159	if len(bsp) > bsize:
1160	bsize = len(bsp)
1161	cols_b = [bsize + 3, width - bsize - 3]
1162	s += textbox.row(cols_1, [' ' + archn + ':'], indent = 1)
1163	s += textbox.line(cols_b, marker = '+', indent = 1)
1164	for bsp in arch['bsps']:
1165	s1 = ' ' + bsp
1166	bspopts = ' '.join(arch[bsp]['bspopts'])
1167	if len(bspopts):
1168	for l in textwrap.wrap('bopt: ' + bspopts,
1169	width = cols_b[1] - 3):
1170	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
1171	s1 = ' ' * len(s1)
1172	excludes = []
1173	for exclude in arch['excludes']:
1174	if 'all' in arch['excludes'][exclude] or \
1175	bsp in arch['excludes'][exclude]:
1176	excludes += [exclude]
1177	excludes = ', '.join(excludes)
1178	if len(excludes):
1179	for l in textwrap.wrap('ex: ' + excludes,
1180	width = cols_b[1] - 3):
1181	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
1182	s1 = ' ' * len(s1)
1183	if len(bspopts) == 0 and len(excludes) == 0:
1184	s += textbox.row(cols_b, [s1, ' '], indent = 1)
1185	s += textbox.line(cols_b, marker = '+', indent = 1)
1186	s += os.linesep
1187	return s
1188
1189	class build_jobs:
1190
1191	def __init__(self, config, arch, bsp):
1192	self.arch = arch
1193	self.bsp = bsp
1194	self.builds = config.builds()
1195	if self.builds is None:
1196	raise error.general('build not found: %s' % (config.build()))
1197	excludes = set(config.excludes(self.arch) +
1198	config.bsp_excludes(self.arch, self.bsp))
1199	remove = []
1200	for e in excludes:
1201	remove += [b for b in self.builds if e in b]
1202	for b in remove:
1203	self.builds.remove(b)
1204	self.build_set = { }
1205	for build in self.builds:
1206	self.build_set[build] = config.build_options(build)
1207
1208	def jobs(self):
1209	return [arch_bsp_build(self.arch, self.bsp, b, self.build_set[b]) \
1210	for b in sorted(self.build_set.keys())]
1211
1212	class builder:
1213
1214	def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
1215	self.config = config
1216	self.build_dir = build_dir
1217	self.rtems_version = version
1218	self.prefix = prefix
1219	self.tools = tools
1220	self.rtems = rtems
1221	self.options = options
1222	self.counts = { 'h' : 0,
1223	'exes' : 0,
1224	'objs' : 0,
1225	'libs' : 0 }
1226	self.results = results(rtems,
1227	{ 'groups' : ['Shared', 'BSP', 'Network', 'Tests',
1228	'LibCPU', 'CPU Kit'],
1229	'exclude' : '.Makefile.',
1230	'CPU Kit' : '.cpukit/.',
1231	'Network' : '.libnetworking/.\|.librpc/.',
1232	'Tests' : '.testsuites/.',
1233	'BSP' : '.libbsp/.',
1234	'LibCPU' : '.libcpu/.',
1235	'Shared' : '.shared/.' })
1236	if not path.exists(path.join(rtems, 'configure')) or \
1237	not path.exists(path.join(rtems, 'Makefile.in')) or \
1238	not path.exists(path.join(rtems, 'cpukit')):
1239	raise error.general('RTEMS source path does not look like RTEMS')
1240
1241	def _bsps(self, arch):
1242	return self.config.arch_bsps(arch)
1243
1244	def _create_build_jobs(self, jobs, build_job_count):
1245	max_job_size = len('%d' % len(jobs))
1246	build_jobs = []
1247	job_index = 1
1248	for job in jobs:
1249	tag = '%*d/%d' % (max_job_size, job_index, len(jobs))
1250	build_jobs += [arch_bsp_builder(self.results,
1251	job,
1252	self._commands(job, build_job_count),
1253	self.build_dir,
1254	tag)]
1255	job_index += 1
1256	set_max_build_label(build_jobs)
1257	return build_jobs
1258
1259	def _commands(self, build, build_jobs):
1260	commands = { 'dry-run' : self.options['dry-run'],
1261	'no-clean' : self.options['no-clean'],
1262	'configure': None,
1263	'build' : None }
1264	cmds = build.build_config.split()
1265	cmds += self.config.bspopts(build.arch, build.bsp)
1266	cmd = [path.join(self.rtems, 'configure')]
1267	for c in cmds:
1268	c = c.replace('@PREFIX@', self.prefix)
1269	c = c.replace('@RTEMS_VERSION@', self.rtems_version)
1270	c = c.replace('@ARCH@', build.arch)
1271	c = c.replace('@BSP@', build.bsp)
1272	cmd += [c]
1273	commands['configure'] = ' '.join(cmd)
1274	cmd = 'make -j %s' % (build_jobs)
1275	commands['build'] = cmd
1276	return commands
1277
1278	def _update_file_counts(self, counts):
1279	for f in self.counts:
1280	if f in counts:
1281	self.counts[f] += counts[f]
1282	return counts
1283
1284	def _warnings_report(self):
1285	if self.options['warnings-report'] is not None:
1286	with open(self.options['warnings-report'], 'w') as f:
1287	f.write(title() + os.linesep)
1288	f.write(os.linesep)
1289	f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'),
1290	os.linesep))
1291	f.write(os.linesep)
1292	f.write(command_line() + os.linesep)
1293	f.write(self.results.warnings_errors.warnings_report())
1294
1295	def _failures_report(self):
1296	if self.options['failures-report'] is not None:
1297	with open(self.options['failures-report'], 'w') as f:
1298	f.write(title() + os.linesep)
1299	f.write(os.linesep)
1300	f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'),
1301	os.linesep))
1302	f.write(os.linesep)
1303	f.write(command_line() + os.linesep)
1304	f.write(self.results.failures_report())
1305
1306	def _finished(self):
1307	log.notice('Total: Warnings:%d exes:%d objs:%d libs:%d' % \
1308	(self.results.get_warning_count(), self.counts['exes'],
1309	self.counts['objs'], self.counts['libs']))
1310	log.output()
1311	log.output('Warnings:')
1312	log.output(self.results.warnings_report())
1313	log.output()
1314	log.notice('Failures:')
1315	log.notice(self.results.failures_report())
1316	self._warnings_report()
1317	self._failures_report()
1318
1319	def run_jobs(self, jobs):
1320	if path.exists(self.build_dir):
1321	log.notice('Cleaning: %s' % (self.build_dir))
1322	path.removeall(self.build_dir)
1323	self.start = datetime.datetime.now()
1324	self.end = datetime.datetime.now()
1325	self.duration = self.end - self.start
1326	self.average = self.duration
1327	env_path = os.environ['PATH']
1328	os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
1329	os.pathsep + os.environ['PATH']
1330	job_count, build_job_count = jobs_option_parse(self.options['jobs'])
1331	builds = self._create_build_jobs(jobs, build_job_count)
1332	active_jobs = []
1333	self.jobs_completed = 0
1334	try:
1335	while len(builds) > 0 or len(active_jobs) > 0:
1336	new_jobs = job_count - len(active_jobs)
1337	if new_jobs > 0:
1338	active_jobs += builds[:new_jobs]
1339	builds = builds[new_jobs:]
1340	finished_jobs = []
1341	for job in active_jobs:
1342	state = job.current_state()
1343	if state == 'ready':
1344	job.run()
1345	elif state != 'running':
1346	finished_jobs += [job]
1347	for job in finished_jobs:
1348	self._update_file_counts(job.get_file_counts())
1349	job.log_output()
1350	job.clean()
1351	active_jobs.remove(job)
1352	self.jobs_completed += 1
1353	time.sleep(0.250)
1354	except:
1355	for job in active_jobs:
1356	try:
1357	job.kill()
1358	except:
1359	pass
1360	raise
1361	self.end = datetime.datetime.now()
1362	os.environ['PATH'] = env_path
1363	self.duration = self.end - self.start
1364	if self.jobs_completed == 0:
1365	self.jobs_completed = 1
1366	self._finished()
1367	self.average = self.duration / self.jobs_completed
1368	log.notice('Average BSP Build Time: %s' % (str(self.average)))
1369	log.notice('Total Time %s' % (str(self.duration)))
1370
1371	def arch_bsp_jobs(self, arch, bsps):
1372	jobs = []
1373	for bsp in bsps:
1374	jobs += build_jobs(self.config, arch, bsp).jobs()
1375	return jobs
1376
1377	def bsp_jobs(self, bsps):
1378	jobs = []
1379	for bsp in bsps:
1380	if bsp.count('/') != 1:
1381	raise error.general('invalid bsp format (use: arch/bsp): %s' % (bsp))
1382	arch, bsp = bsp.split('/')
1383	jobs += build_jobs(self.config, arch, bsp).jobs()
1384	return jobs
1385
1386	def arch_jobs(self, archs):
1387	jobs = []
1388	for arch in archs:
1389	if not self.config.arch_present(arch):
1390	raise error.general('Architecture not found: %s' % (arch))
1391	jobs += self.arch_bsp_jobs(arch, self._bsps(arch))
1392	return jobs
1393
1394	def profile_jobs(self, profiles):
1395	jobs = []
1396	for profile in profiles:
1397	if not self.config.profile_present(profile):
1398	raise error.general('Profile not found: %s' % (profile))
1399	for arch in self.config.profile_archs(profile):
1400	jobs += self.bsp_jobs(self.config.profile_arch_bsps(profile, arch))
1401	return jobs
1402
1403	def build_bsps(self, bsps):
1404	log.notice('BSPS(s): %s' % (', '.join(bsps)))
1405	self.run_jobs(self.bsp_jobs(bsps))
1406
1407	def build_archs(self, archs):
1408	log.notice('Architecture(s): %s' % (', '.join(archs)))
1409	self.run_jobs(self.arch_jobs(archs))
1410
1411	def build_profiles(self, profiles):
1412	log.notice('Profile(s): %s' % (', '.join(profiles)))
1413	self.run_jobs(self.profile_jobs(profiles))
1414
1415	def run_args(args):
1416	b = None
1417	ec = 0
1418	try:
1419	#
1420	# On Windows MSYS2 prepends a path to itself to the environment
1421	# path. This means the RTEMS specific automake is not found and which
1422	# breaks the bootstrap. We need to remove the prepended path. Also
1423	# remove any ACLOCAL paths from the environment.
1424	#
1425	if os.name == 'nt':
1426	cspath = os.environ['PATH'].split(os.pathsep)
1427	if 'msys' in cspath[0] and cspath[0].endswith('bin'):
1428	os.environ['PATH'] = os.pathsep.join(cspath[1:])
1429
1430	start = datetime.datetime.now()
1431	top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
1432	prefix = '/opt/rtems/%s' % (rtems_version())
1433	tools = prefix
1434	build_dir = 'bsp-builds'
1435	logf = 'bsp-build-%s.txt' % \
1436	(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
1437	config_file = path.join(top, 'share', 'rtems', 'tester',
1438	'rtems', 'rtems-bsps.ini')
1439	if not path.exists(config_file):
1440	config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')
1441
1442	argsp = argparse.ArgumentParser()
1443	argsp.add_argument('--prefix', help = 'Prefix to build the BSP.',
1444	type = str)
1445	argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.',
1446	type = str)
1447	argsp.add_argument('--rtems', help = 'The RTEMS source tree.',
1448	type = str)
1449	argsp.add_argument('--build-path', help = 'Path to build in.',
1450	type = str)
1451	argsp.add_argument('--log', help = 'Log file.', type = str)
1452	argsp.add_argument('--config-report', help = 'Report the configuration.',
1453	type = str, default = None,
1454	choices = ['all', 'profiles', 'builds', 'archs'])
1455	argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
1456	type = str, default = None)
1457	argsp.add_argument('--failures-report', help = 'Report the failures to a file.',
1458	type = str, default = None)
1459	argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
1460	action = 'store_true')
1461	argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
1462	action = 'store_true')
1463	argsp.add_argument('--profiles', help = 'Build the listed profiles (profile,profile,..).',
1464	type = str, default = 'tier-1')
1465	argsp.add_argument('--arch', help = 'Build the architectures (arch,arch,..).',
1466	type = str)
1467	argsp.add_argument('--bsp', help = 'Build the BSPs (arch/bsp,arch/bsp,..).',
1468	type = str)
1469	argsp.add_argument('--build', help = 'Build name to build (see --config-report).',
1470	type = str, default='all')
1471	argsp.add_argument('--jobs', help = 'Number of jobs to run.',
1472	type = str, default = '1/%d' % (host.cpus()))
1473	argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
1474	action = 'store_true')
1475	mailer.add_arguments(argsp)
1476
1477	opts = argsp.parse_args(args[1:])
1478	mail = None
1479	if opts.mail:
1480	mail = mailer.mail(opts)
1481	# Request these now to generate any errors.
1482	from_addr = mail.from_address()
1483	smtp_host = mail.smtp_host()
1484	if 'mail_to' in opts and opts.mail_to is not None:
1485	to_addr = opts.mail_to
1486	else:
1487	to_addr = 'build@rtems.org'
1488	if opts.log is not None:
1489	logf = opts.log
1490	log.default = log.log([logf])
1491	log.notice(title())
1492	log.output(command_line())
1493	if mail:
1494	log.notice('Mail: from:%s to:%s smtp:%s' % (from_addr,
1495	to_addr,
1496	smtp_host))
1497
1498	config = configuration_()
1499	config.load(config_file, opts.build)
1500
1501	if opts.config_report:
1502	log.notice('Configuration Report: %s' % (opts.config_report))
1503	c_profiles = False
1504	c_builds = False
1505	c_archs = False
1506	if opts.config_report == 'all':
1507	c_profiles = True
1508	c_builds = True
1509	c_archs = True
1510	elif opts.config_report == 'profiles':
1511	c_profiles = True
1512	elif opts.config_report == 'builds':
1513	c_builds = True
1514	elif opts.config_report == 'archs':
1515	c_archs = True
1516	log.notice(config.report(c_profiles, c_builds, c_archs))
1517	sys.exit(0)
1518
1519	if opts.rtems is None:
1520	raise error.general('No RTEMS source provided on the command line')
1521	if opts.prefix is not None:
1522	prefix = path.shell(opts.prefix)
1523	if opts.rtems_tools is not None:
1524	tools = path.shell(opts.rtems_tools)
1525	if opts.build_path is not None:
1526	build_dir = path.shell(opts.build_path)
1527
1528	options = { 'stop-on-error' : opts.stop_on_error,
1529	'no-clean' : opts.no_clean,
1530	'dry-run' : opts.dry_run,
1531	'jobs' : opts.jobs,
1532	'warnings-report' : opts.warnings_report,
1533	'failures-report' : opts.failures_report }
1534
1535	b = builder(config, rtems_version(), prefix, tools,
1536	path.shell(opts.rtems), build_dir, options)
1537
1538	profiles = comma_split(opts.profiles)
1539	archs = comma_split(opts.arch)
1540	bsps = comma_split(opts.bsp)
1541
1542	#
1543	# The default is build a profile.
1544	#
1545	if bsps is not None:
1546	if archs is not None:
1547	raise error.general('--arch supplied with --bsp;' \
1548	' use --bsp=arch/bsp,arch/bsp,..')
1549	what = 'BSPs: %s' % (' '.join(bsps))
1550	b.build_bsps(bsps)
1551	elif archs is not None:
1552	what = 'Archs: %s' % (' '.join(archs))
1553	b.build_archs(archs)
1554	else:
1555	what = 'Profile(s): %s' % (' '.join(profiles))
1556	b.build_profiles(profiles)
1557	end = datetime.datetime.now()
1558
1559	#
1560	# Email the results of the build.
1561	#
1562	if mail is not None:
1563	subject = '[rtems-bsp-builder] %s: %s' % (str(start).split('.')[0],
1564	what)
1565	t = title()
1566	body = t + os.linesep
1567	body += '=' * len(t) + os.linesep
1568	body += os.linesep
1569	body += 'Host: %s' % (os.uname()[3]) + os.linesep
1570	body += os.linesep
1571	body += command_line()
1572	body += os.linesep
1573	body += 'Total Time : %s for %d completed job(s)' % \
1574	(str(b.duration), b.jobs_completed)
1575	body += os.linesep
1576	body += 'Average BSP Build Time: %s' % (str(b.average))
1577	body += os.linesep + os.linesep
1578	body += 'Builds' + os.linesep
1579	body += '======' + os.linesep
1580	body += os.linesep.join([' ' + cb for cb in config.builds()])
1581	body += os.linesep + os.linesep
1582	body += 'Failures Report' + os.linesep
1583	body += '===============' + os.linesep
1584	body += b.results.failures_report()
1585	body += os.linesep
1586	body += 'Warnings Report' + os.linesep
1587	body += '===============' + os.linesep
1588	body += b.results.warnings_report(summary = True)
1589	mail.send(to_addr, subject, body)
1590
1591	except error.general as gerr:
1592	print(gerr)
1593	print('BSP Build FAILED', file = sys.stderr)
1594	ec = 1
1595	except error.internal as ierr:
1596	print(ierr)
1597	print('BSP Build FAILED', file = sys.stderr)
1598	ec = 1
1599	except error.exit as eerr:
1600	pass
1601	except KeyboardInterrupt:
1602	log.notice('abort: user terminated')
1603	ec = 1
1604	if b is not None:
1605	b.results.report()
1606	sys.exit(ec)
1607
1608	def run():
1609	run_args(sys.argv)
1610
1611	if __name__ == "__main__":
1612	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: