Context Navigation

source: rtems-tools/tester/rt/check.py @ 5075e8e

5

Last change on this file since 5075e8e was 5075e8e, checked in by Chris Johns <chrisj@…>, on 09/08/18 at 05:36:31

tester/bsp-builder: Yield the job control thread when in a dry-run.

This stops a dry-run blocking while the job control thread spins.

Property mode set to 100755

File size: 63.5 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2016-2018 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Redistribution and use in source and binary forms, with or without
9	# modification, are permitted provided that the following conditions are met:
10	#
11	# 1. Redistributions of source code must retain the above copyright notice,
12	# this list of conditions and the following disclaimer.
13	#
14	# 2. Redistributions in binary form must reproduce the above copyright notice,
15	# this list of conditions and the following disclaimer in the documentation
16	# and/or other materials provided with the distribution.
17	#
18	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21	# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22	# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23	# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24	# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25	# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26	# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27	# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28	# POSSIBILITY OF SUCH DAMAGE.
29	#
30
31	from __future__ import print_function
32
33	import argparse
34	import copy
35	import datetime
36	import operator
37	import os
38	import re
39	import sys
40	import textwrap
41	import threading
42	import time
43	import traceback
44
45	import pprint
46
47	from rtemstoolkit import configuration
48	from rtemstoolkit import execute
49	from rtemstoolkit import error
50	from rtemstoolkit import host
51	from rtemstoolkit import log
52	from rtemstoolkit import mailer
53	from rtemstoolkit import path
54	from rtemstoolkit import textbox
55	from rtemstoolkit import version
56
57	#
58	# Group loggin entries together.
59	#
60	log_lock = threading.Lock()
61
62	#
63	# The max build label size in the jobs list.
64	#
65	max_build_label = 0
66
67	def rtems_version():
68	return version.version()
69
70	def wrap(line, lineend = '', indent = 0, width = 75):
71	if type(line) is tuple or type(line) is list:
72	if len(line) >= 2:
73	s1 = line[0]
74	else:
75	s1 = ''
76	s2 = line[1:]
77	elif type(line) is str:
78	s1 = ''
79	s2 = [line]
80	else:
81	raise error.internal('line is not a tuple, list or string')
82	s1len = len(s1)
83	s = ''
84	first = True
85	for ss in s2:
86	if type(ss) is not str and type(ss) is not unicode:
87	raise error.internal('text needs to be a string')
88	for l in textwrap.wrap(ss, width = width - s1len - indent - 1):
89	s += '%s%s%s%s%s' % (' ' * indent, s1, l, lineend, os.linesep)
90	if first and s1len > 0:
91	s1 = ' ' * s1len
92	if lineend != '':
93	s = s[:0 - len(os.linesep) - 1] + os.linesep
94	return s
95
96	def comma_split(options):
97	if options is None:
98	return None
99	return [o.strip() for o in options.split(',')]
100
101	def title():
102	return 'RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str())
103
104	def command_line():
105	return wrap(('command: ', ' '.join(sys.argv)), lineend = '\\')
106
107	def jobs_option_parse(jobs_option):
108	try:
109	if '/' not in jobs_option:
110	return 1, int(jobs_option)
111	jos = jobs_option.split('/')
112	if len(jos) != 2:
113	raise error.general('invalid jobs option: %s' % (jobs_option))
114	return int(jos[0]), int(jos[1])
115	except:
116	pass
117	raise error.general('invalid jobs option: %s' % (jobs_option))
118
119	def arch_bsp_build_parse(build):
120	if type(build) is str:
121	build_key = build
122	else:
123	build_key = build.key()
124	abb = build_key.split('.')
125	if len(abb) != 2:
126	raise error.general('invalid build key: %s' % (build_key))
127	ab = abb[0].split('/')
128	if len(ab) != 2:
129	raise error.general('invalid build key: %s' % (build_key))
130	return ab[0], ab[1], abb[1]
131
132	def set_max_build_label(jobs):
133	global max_build_label
134	for job in jobs:
135	if len(job.build.key()) > max_build_label:
136	max_build_label = len(job.build.key())
137	max_build_label += 2
138
139	class arch_bsp_build:
140
141	def __init__(self, arch, bsp, build, build_config):
142	self.arch = arch
143	self.bsp = bsp
144	self.build = build
145	self.build_config = build_config
146	self.start_time = None
147	self.stop_time = None
148
149	def __str__(self):
150	return self.key() + ': ' + self.build_config
151
152	def key(self):
153	return '%s/%s.%s' % (self.arch, self.bsp, self.build)
154
155	def get_arch_bsp(self):
156	return self.arch, self.bsp
157
158	def start(self):
159	self.start_time = datetime.datetime.now()
160
161	def stop(self):
162	self.stop_time = datetime.datetime.now()
163
164	def duration(self):
165	return self.stop_time - self.start_time
166
167	class output_worker:
168
169	def __init__(self, we, build):
170	self.text = []
171	self.warnings_errors = we
172	self.build = build
173
174	def output(self, text):
175	self.warnings_errors.process_output(text, self.build)
176	self.text += text.splitlines()
177
178	def log_output(self, heading):
179	log_lock.acquire()
180	try:
181	log.output(heading + self.text)
182	except:
183	raise
184	finally:
185	log_lock.release()
186
187	class warnings_errors:
188
189	def __init__(self, source_base, groups):
190	self.lock = threading.Lock()
191	self.source_base = path.host(source_base)
192	self.groups = groups
193	self.reset()
194
195	def _get_warnings(self, build):
196	self.lock.acquire()
197	warnings = [w for w in self.warnings]
198	self.lock.release()
199	return sorted(warnings)
200
201	def _total(self, archive):
202	total = 0
203	for a in archive:
204	total += archive[a]
205	return total
206
207	def _analyze(self, warnings, exclude):
208	def _group(data, category, name, warning, count, groups, group_regx):
209	if 'groups' not in data:
210	data['groups'] = { }
211	if category not in data['groups']:
212	data['groups'][category] = { }
213	if 'totals' not in data['groups'][category]:
214	data['groups'][category]['totals'] = { }
215	if name not in data['groups'][category]:
216	data['groups'][category][name] = { }
217	for group in groups:
218	if group not in data['groups'][category]['totals']:
219	data['groups'][category]['totals'][group] = 0
220	if group not in data['groups'][category][name]:
221	data['groups'][category][name][group] = 0
222	if group_regx[group].match(warning):
223	data['groups'][category][name][group] += count
224	data['groups'][category]['totals'][group] += count
225	break
226
227	def _update(data, category, name, warning, count, groups, group_regx):
228	if category not in data:
229	data[category] = { }
230	if name not in data[category]:
231	data[category][name] = { }
232	if warning not in data[category][name]:
233	data[category][name][warning] = 0
234	data[category][name][warning] += count
235	_group(data, category, name, w, count, groups, group_regx)
236
237	categories = ['arch', 'arch_bsp', 'build']
238	data = { 'groups': { } }
239	for category in categories:
240	data[category] = { }
241	data['groups'][category] = { }
242	group_regx = { }
243	for group in self.groups['groups']:
244	group_regx[group] = re.compile(self.groups[group])
245	exclude_regx = re.compile(exclude)
246	for warning in self.warnings:
247	arch, bsp, build = arch_bsp_build_parse(warning)
248	arch_bsp = '%s/%s' % (arch, bsp)
249	for w in self.warnings[warning]:
250	if not exclude_regx.match(w):
251	count = self.warnings[warning][w]
252	_update(data, 'arch', arch, w, count,
253	self.groups['groups'], group_regx)
254	_update(data, 'arch_bsp', arch_bsp, w, count,
255	self.groups['groups'], group_regx)
256	_update(data, 'build', build, w, count,
257	self.groups['groups'], group_regx)
258	for category in categories:
259	common = {}
260	for name in data[category]:
261	for w in data[category][name]:
262	if w not in common:
263	for other in [n for n in data[category] if n != name]:
264	if w in data[category][other]:
265	common[w] = data[category][name][w]
266	_group(data, category, 'common', w, common[w],
267	self.groups['groups'], group_regx)
268	data[category]['common'] = common
269	return data
270
271	def _report_category(self, label, warnings, group_counts, summary):
272	width = 70
273	cols_1 = [width]
274	cols_2 = [8, width - 8]
275	cols_4 = textbox.even_columns(4, width)
276	cols_2_4 = textbox.merge_columns([cols_2, cols_4])
277	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
278	s += textbox.row(cols_1, [' ' + label], indent = 1)
279	s += textbox.line(cols_1, marker = '+', indent = 1)
280	builds = ['common'] + sorted([b for b in warnings if b != 'common'])
281	common = warnings['common']
282	for build in builds:
283	build_warnings = warnings[build]
284	if build is not 'common':
285	build_warnings = [w for w in build_warnings if w not in common]
286	s += textbox.row(cols_1,
287	[' %s : %d warning(s)' % (build,
288	len(build_warnings))],
289	indent = 1)
290	if len(build_warnings) == 0:
291	s += textbox.line(cols_1, marker = '+', indent = 1)
292	else:
293	s += textbox.line(cols_4, marker = '+', indent = 1)
294	if build not in group_counts:
295	gs = [0 for group in self.groups['groups']]
296	else:
297	gs = []
298	for g in range(0, len(self.groups['groups'])):
299	group = self.groups['groups'][g]
300	if group in group_counts[build]:
301	count = group_counts[build][group]
302	else:
303	count = 0
304	gs += ['%*s' % (cols_4[g % 4] - 2,
305	'%s : %4d' % (group, count))]
306	for row in range(0, len(self.groups['groups']), 4):
307	if row + 4 > len(self.groups['groups']):
308	d = gs[row:] + \
309	['' for r in range(row,
310	len(self.groups['groups']))]
311	else:
312	d = gs[row:+4]
313	s += textbox.row(cols_4, d, indent = 1)
314	s += textbox.line(cols_2_4, marker = '+', indent = 1)
315	if not summary:
316	vw = sorted([(w, warnings[build][w]) for w in build_warnings],
317	key = operator.itemgetter(1),
318	reverse = True)
319	for w in vw:
320	c1 = '%6d' % w[1]
321	for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3):
322	s += textbox.row(cols_2, [c1, l], indent = 1)
323	c1 = ' ' * 6
324	s += textbox.line(cols_2, marker = '+', indent = 1)
325	return s
326
327	def _report_warning_map(self):
328	builds = self.messages['warnings']
329	width = 70
330	cols_1 = [width]
331	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
332	s += textbox.row(cols_1, [' Warning Map'], indent = 1)
333	s += textbox.line(cols_1, marker = '+', indent = 1)
334	for build in builds:
335	messages = builds[build]
336	s += textbox.row(cols_1, [' %s : %d' % (build, len(messages))], indent = 1)
337	s += textbox.line(cols_1, marker = '+', indent = 1)
338	for msg in messages:
339	for l in textwrap.wrap(msg, width = width - 3):
340	s += textbox.row(cols_1, [' ' + l], indent = 1)
341	for l in textwrap.wrap(messages[msg], width = width - 3 - 4):
342	s += textbox.row(cols_1, [' ' + l], indent = 1)
343	s += textbox.line(cols_1, marker = '+', indent = 1)
344	return s
345
346	def warnings_report(self, summary = False):
347	self.lock.acquire()
348	s = ' No warnings' + os.linesep
349	try:
350	total = 0
351	for build in self.warnings:
352	total += self._total(self.warnings[build])
353	if total != 0:
354	data = self._analyze(self.warnings, self.groups['exclude'])
355	s = self._report_category('By Architecture (total : %d)' % (total),
356	data['arch'], data['groups']['arch'],
357	summary)
358	s += os.linesep
359	s += self._report_category('By BSP (total : %d)' % (total),
360	data['arch_bsp'], data['groups']['arch_bsp'],
361	summary)
362	s += os.linesep
363	s += self._report_category('By Build (total : %d)' % (total),
364	data['build'], data['groups']['build'],
365	summary)
366	s += os.linesep
367	if not summary:
368	s += self._report_warning_map()
369	s += os.linesep
370	finally:
371	self.lock.release()
372	return s
373
374	def clear_build(self, build):
375	self.lock.acquire()
376	self.warnings[build.key()] = {}
377	self.errors[build.key()] = {}
378	self.lock.release()
379
380	def get_warning_count(self):
381	self.lock.acquire()
382	count = self.warning_count
383	self.lock.release()
384	return count
385
386	def get_error_count(self):
387	self.lock.acquire()
388	count = self.error_count
389	self.lock.release()
390	return count
391
392	def reset(self):
393	self.lock.acquire()
394	self.warnings = { }
395	self.warning_count = 0
396	self.errors = { }
397	self.error_count = 0
398	self.messages = { 'warnings' : { }, 'errors' : { } }
399	self.lock.release()
400
401	def _get_messages(self, build, key):
402	self.lock.acquire()
403	if type(build) is str:
404	build_key = build
405	else:
406	build_key = build.key()
407	if build_key not in self.messages[key]:
408	messages = []
409	else:
410	messages = self.messages[key][build_key]
411	messages = ['%s %s' % (m, messages[m]) for m in messages]
412	self.lock.release()
413	return messages
414
415	def get_warning_messages(self, build):
416	return self._get_messages(build, 'warning')
417
418	def get_error_messages(self, build):
419	return self._get_messages(build, 'errors')
420
421	def process_output(self, text, build):
422	def _line_split(line, source_base):
423	if line.count(':') < 2:
424	return None
425	ls = line.split(' ', 1)
426	fname = ls[0].strip().split(':', 2)
427	if len(fname) != 3:
428	return None
429	p = path.abspath(fname[0])
430	p = p.replace(source_base, '')
431	if path.isabspath(p):
432	p = p[1:]
433	if len(fname[2]) == 0:
434	pos = None
435	else:
436	pos = fname[2]
437	return p, fname[1], pos, ls[1]
438
439	def _create_build_errors(build, archive):
440	if build.key() not in archive:
441	archive[build.key()] = { }
442	return archive[build.key()]
443
444	#
445	# The GNU linker does not supply 'error' in error messages. There is no
446	# line information which is understandable. Look for `bin/ld:` and
447	# `collect2:` in the output and then create the error when `collect2:`
448	# is seen.
449	#
450	# The order we inspect each line is important.
451	#
452	if ' warning:' in text or \
453	' error:' in text or \
454	' Error:' in text or \
455	'bin/ld:' in text:
456	self.lock.acquire()
457	try:
458	for l in text.splitlines():
459	if 'bin/ld:' in l:
460	archive =_create_build_errors(build, self.errors)
461	if 'linker' not in archive:
462	archive['linker'] = []
463	archive['linker'] += [l.split(':', 1)[1].strip()]
464	messages = 'errors'
465	elif l.startswith('collect2:'):
466	archive =_create_build_errors(build, self.errors)
467	l = '/ld/collect2:0: error: '
468	if 'linker' not in archive or len(archive['linker']) == 0:
469	l += 'no error message found!'
470	else:
471	l += '; '.join(archive['linker'])
472	archive['linker'] = []
473	messages = 'errors'
474	elif ' warning:' in l:
475	self.warning_count += 1
476	archive = _create_build_errors(build, self.warnings)
477	messages = 'warnings'
478	elif ' error:' in l or ' Error:' in l:
479	self.error_count += 1
480	archive =_create_build_errors(build, self.errors)
481	messages = 'errors'
482	else:
483	continue
484	line_parts = _line_split(l, self.source_base)
485	if line_parts is not None:
486	src, line, pos, msg = line_parts
487	if pos is not None:
488	where = '%s:%s:%s' % (src, line, pos)
489	else:
490	where = '%s:%s' % (src, line)
491	if where not in archive:
492	archive[where] = 1
493	else:
494	archive[where] += 1
495	if build.key() not in self.messages[messages]:
496	self.messages[messages][build.key()] = { }
497	self.messages[messages][build.key()][where] = msg
498	finally:
499	self.lock.release()
500
501	class results:
502
503	def __init__(self, source_base, groups):
504	self.lock = threading.Lock()
505	self.errors = { 'pass': 0,
506	'configure': 0,
507	'build': 0,
508	'tests': 0,
509	'passes': { },
510	'fails': { } }
511	self.counts = { 'h' : 0,
512	'exes' : 0,
513	'objs' : 0,
514	'libs' : 0 }
515	self.warnings_errors = warnings_errors(source_base, groups)
516
517	def _arch_bsp(self, arch, bsp):
518	return '%s/%s' % (arch, bsp)
519
520	def _arch_bsp_passes(self, build):
521	if build.key() not in self.errors['passes']:
522	self.errors['passes'][build.key()] = []
523	return self.errors['passes'][build.key()]
524
525	def _arch_bsp_fails(self, build):
526	if build.key() not in self.errors['fails']:
527	self.errors['fails'][build.key()] = []
528	return self.errors['fails'][build.key()]
529
530	def _count(self, label):
531	count = 0
532	for build in self.errors[label]:
533	count += len(self.errors[label][build])
534	return count
535
536	def _max_col(self, label):
537	max_col = 0
538	for build in self.errors[label]:
539	arch, bsp, build_config = arch_bsp_build_parse(build)
540	arch_bsp = self._arch_bsp(arch, bsp)
541	if len(arch_bsp) > max_col:
542	max_col = len(arch_bsp)
543	return max_col
544
545	def get_warning_count(self):
546	return self.warnings_errors.get_warning_count()
547
548	def get_error_count(self):
549	return self.warnings_errors.get_error_count()
550
551	def get_warning_messages(self, build):
552	return self.warnings_errors.get_warning_messages(build)
553
554	def get_error_messages(self, build):
555	return self.warnings_errors.get_error_messages(build)
556
557	def status(self):
558	self.lock.acquire()
559	try:
560	s = 'Pass: %4d Fail: %4d (configure:%d build:%d)' % \
561	(self.errors['pass'],
562	self.errors['configure'] + self.errors['build'],
563	self.errors['configure'], self.errors['build'])
564	except:
565	raise
566	finally:
567	self.lock.release()
568	return s;
569
570	def add_fail(self, phase, build, configure, warnings, error_messages):
571	fails = self._arch_bsp_fails(build)
572	self.lock.acquire()
573	try:
574	self.errors[phase] += 1
575	fails += [(phase, build.build_config, configure, error_messages)]
576	finally:
577	self.lock.release()
578
579	def add_pass(self, build, configure, warnings):
580	passes = self._arch_bsp_passes(build)
581	self.lock.acquire()
582	try:
583	self.errors['pass'] += 1
584	passes += [(build.build_config, configure, warnings, None)]
585	finally:
586	self.lock.release()
587
588	def pass_count(self):
589	return self._count('passes')
590
591	def fail_count(self):
592	return self._count('fails')
593
594	def _failures_report(self, build, count):
595	if type(build) is str:
596	build_key = build
597	else:
598	build_key = build.key()
599	if build_key not in self.errors['fails'] or \
600	len(self.errors['fails'][build_key]) == 0:
601	return count, 0, ' No failures'
602	absize = 0
603	bsize = 0
604	ssize = 0
605	arch, bsp, build_set = arch_bsp_build_parse(build_key)
606	arch_bsp = self._arch_bsp(arch, bsp)
607	fails = self.errors['fails'][build_key]
608	for f in fails:
609	if len(f[0]) > ssize:
610	ssize = len(f[0])
611	s = ''
612	for f in fails:
613	count += 1
614	fcl = ' %3d' % (count)
615	state = f[0]
616	s += '%s %s %s %-*s:%s' % \
617	(fcl, build_set, arch_bsp, ssize, state, os.linesep)
618	s1 = ' ' * 6
619	s += wrap((s1, 'configure: ' + f[2]), lineend = '\\', width = 75)
620	s1 = ' ' * 5
621	for e in self.warnings_errors.get_error_messages(build):
622	s += wrap([s1 + 'error: ', e])
623	return count, len(fails), s
624
625	def failures_report(self, build = None):
626	s = ''
627	count = 0
628	if build is not None:
629	count, build_fails, bs = self._failures_report(build, count)
630	if build_fails > 0:
631	s += bs + os.linesep
632	else:
633	self.lock.acquire()
634	builds = sorted(self.errors['fails'].keys())
635	self.lock.release()
636	for build in builds:
637	count, build_fails, bs = self._failures_report(build, count)
638	if build_fails > 0:
639	s += bs + os.linesep
640	if count == 0:
641	s = ' No failures' + os.linesep
642	return s
643
644	def warnings_report(self, summary = False):
645	return self.warnings_errors.warnings_report(summary)
646
647	def report(self):
648	self.lock.acquire()
649	log_lock.acquire()
650	passes = self.pass_count()
651	fails = self.fail_count()
652	log.notice('Passes: %d Failures: %d' % (passes, fails))
653	log.output()
654	log.output('Build Report')
655	log.output(' Passes: %d Failures: %d' % (passes, fails))
656	log.output(' Failures:')
657	if fails == 0:
658	log.output(' None')
659	else:
660	max_col = self._max_col('fails')
661	for build in self.errors['fails']:
662	arch, bsp, build_config = arch_bsp_build_parse(build)
663	arch_bsp = self._arch_bsp(arch, bsp)
664	for f in self.errors['fails'][build]:
665	config_cmd = f[2]
666	config_at = config_cmd.find('configure')
667	if config_at != -1:
668	config_cmd = config_cmd[config_at:]
669	log.output(' %*s:' % (max_col + 2, arch_bsp))
670	s1 = ' ' * 6
671	log.output(wrap([s1, config_cmd], lineend = '\\', width = 75))
672	if f[3] is not None:
673	s1 = ' ' * len(s1)
674	for msg in f[3]:
675	log.output(wrap([s1, msg], lineend = '\\'))
676	log.output(' Passes:')
677	if passes == 0:
678	log.output(' None')
679	else:
680	max_col = self._max_col('passes')
681	for build in self.errors['passes']:
682	arch, bsp, build_config = arch_bsp_build_parse(build)
683	arch_bsp = self._arch_bsp(arch, bsp)
684	for f in self.errors['passes'][build]:
685	config_cmd = f[1]
686	config_at = config_cmd.find('configure')
687	if config_at != -1:
688	config_cmd = config_cmd[config_at:]
689	log.output(' %s (%5d):' % (arch_bsp, f[2]))
690	log.output(wrap([' ' * 6, config_cmd], lineend = '\\', width = 75))
691	log_lock.release()
692	self.lock.release()
693
694	class arch_bsp_builder:
695
696	def __init__(self, results_, build, commands, build_dir, tag):
697	self.lock = threading.Lock()
698	self.state = 'ready'
699	self.thread = None
700	self.proc = None
701	self.results = results_
702	self.build = build
703	self.commands = commands
704	self.build_dir = build_dir
705	self.tag = tag
706	self.output = output_worker(results_.warnings_errors, build)
707	self.counts = { 'h' : 0,
708	'exes' : 0,
709	'objs' : 0,
710	'libs' : 0 }
711
712	def _notice(self, text):
713	global max_build_label
714	arch, bsp, build_set = arch_bsp_build_parse(self.build.key())
715	label = '%s/%s (%s)' % (arch, bsp, build_set)
716	log.notice('[%s] %-*s %s' % (self.tag, max_build_label, label, text))
717
718	def _build_dir(self):
719	return path.join(self.build_dir, self.build.key())
720
721	def _make_build_dir(self):
722	if not path.exists(self._build_dir()):
723	path.mkdir(self._build_dir())
724
725	def _count_files(self):
726	for root, dirs, files in os.walk(self._build_dir()):
727	for file in files:
728	if file.endswith('.exe'):
729	self.counts['exes'] += 1
730	elif file.endswith('.o'):
731	self.counts['objs'] += 1
732	elif file.endswith('.a'):
733	self.counts['libs'] += 1
734	elif file.endswith('.h'):
735	self.counts['h'] += 1
736
737	def _execute(self, phase):
738	exit_code = 0
739	cmd = self.commands[phase]
740	try:
741	# This should locked; not sure how to do that
742	self.proc = execute.capture_execution(log = self.output)
743	log.output(wrap(('run:', self.build.key(), cmd), lineend = '\\'))
744	if not self.commands['dry-run']:
745	exit_code, proc, output = self.proc.shell(cmd,
746	cwd = path.host(self._build_dir()))
747	except:
748	traceback.print_exc()
749	self.lock.acquire()
750	if self.proc is not None:
751	self.proc.kill()
752	self.lock.release()
753	exit_code = 1
754	self.lock.acquire()
755	self.proc = None
756	self.lock.release()
757	return exit_code == 0
758
759	def _configure(self):
760	return self._execute('configure')
761
762	def _make(self):
763	return self._execute('build')
764
765	def _worker(self):
766	self.lock.acquire()
767	self.state = 'running'
768	self.lock.release()
769	self.build.start()
770	warnings = self.results.get_warning_count()
771	ok = False
772	try:
773	log_lock.acquire()
774	try:
775	self._notice('Start')
776	self._notice('Creating: %s' % (self._build_dir()))
777	except:
778	raise
779	finally:
780	log_lock.release()
781	self._make_build_dir()
782	self._notice('Configuring')
783	ok = self._configure()
784	if not ok:
785	warnings = self.results.get_warning_count() - warnings
786	self.results.add_fail('configure',
787	self.build,
788	self.commands['configure'],
789	warnings,
790	self.results.get_error_messages(self.build))
791	self.lock.acquire()
792	if self.state == 'killing':
793	ok = False
794	self.lock.release()
795	if ok:
796	self._notice('Building')
797	ok = self._make()
798	if not ok:
799	warnings = self.results.get_warning_count() - warnings
800	self.results.add_fail('build',
801	self.build,
802	self.commands['configure'],
803	warnings,
804	self.results.get_error_messages(self.build))
805	if ok:
806	warnings = self.results.get_warning_count() - warnings
807	self.results.add_pass(self.build,
808	self.commands['configure'],
809	warnings)
810	except:
811	ok = False
812	self._notice('Build Exception')
813	traceback.print_exc()
814	self.build.stop()
815	log_lock.acquire()
816	try:
817	self._count_files()
818	if ok:
819	self._notice('PASS')
820	else:
821	self._notice('FAIL')
822	self._notice('Warnings:%d exes:%d objs:%d libs:%d' % \
823	(warnings, self.counts['exes'],
824	self.counts['objs'], self.counts['libs']))
825	log.output(' %s: Failure Report:' % (self.build.key()))
826	log.output(self.results.failures_report(self.build))
827	self._notice('Finished (duration:%s)' % (str(self.build.duration())))
828	self._notice('Status: %s' % (self.results.status()))
829	except:
830	self._notice('Build Exception:')
831	traceback.print_exc()
832	finally:
833	log_lock.release()
834	self.lock.acquire()
835	self.state = 'finished'
836	self.lock.release()
837
838	def get_file_counts(self):
839	return self.counts
840
841	def run(self):
842	self.lock.acquire()
843	try:
844	if self.state != 'ready':
845	raise error.general('builder already run')
846	self.state = 'starting'
847	self.thread = threading.Thread(target = self._worker)
848	self.thread.start()
849	except:
850	raise
851	finally:
852	self.lock.release()
853
854	def kill(self):
855	self.lock.acquire()
856	if self.thread is not None:
857	self.state = 'killing'
858	if self.proc is not None:
859	try:
860	self.proc.kill()
861	except:
862	pass
863	self.lock.release()
864	self.thread.join(5)
865	self.lock.acquire()
866	self.state = 'finished'
867	self.lock.release()
868
869	def current_state(self):
870	self.lock.acquire()
871	state = self.state
872	self.lock.release()
873	return state
874
875	def log_output(self):
876	self.output.log_output(['-' * 79, '] %s: Build output:' % (self.build.key())])
877
878	def clean(self):
879	if not self.commands['no-clean']:
880	self._notice('Cleaning: %s' % (self._build_dir()))
881	path.removeall(self._build_dir())
882
883	class configuration_:
884
885	def __init__(self):
886	self.config = configuration.configuration()
887	self.archs = { }
888	self.builds_ = { }
889	self.profiles = { }
890
891	def __str__(self):
892	s = self.name + os.linesep
893	s += 'Archs:' + os.linesep + \
894	pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
895	s += 'Builds:' + os.linesep + \
896	pprint.pformat(self.builds_, indent = 1, width = 80) + os.linesep
897	s += 'Profiles:' + os.linesep + \
898	pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
899	return s
900
901	def _build_options(self, build, nesting = 0):
902	if ':' in build:
903	section, name = build.split(':', 1)
904	opts = [self.config.get_item(section, name)]
905	return opts
906	builds = self.builds_['builds']
907	if build not in builds:
908	raise error.general('build %s not found' % (build))
909	if nesting > 20:
910	raise error.general('nesting build %s' % (build))
911	options = []
912	for option in self.builds_['builds'][build]:
913	if ':' in option:
914	section, name = option.split(':', 1)
915	opts = [self.config.get_item(section, name)]
916	else:
917	opts = self._build_options(option, nesting + 1)
918	for opt in opts:
919	if opt not in options:
920	options += [opt]
921	return options
922
923	def load(self, name, build):
924	self.config.load(name)
925	archs = []
926	self.profiles['profiles'] = \
927	self.config.comma_list('profiles', 'profiles', err = False)
928	if len(self.profiles['profiles']) == 0:
929	self.profiles['profiles'] = ['tier-%d' % (t) for t in range(1,4)]
930	for p in self.profiles['profiles']:
931	profile = {}
932	profile['name'] = p
933	profile['archs'] = self.config.comma_list(profile['name'], 'archs', err = False)
934	archs += profile['archs']
935	for arch in profile['archs']:
936	bsps = 'bsps_%s' % (arch)
937	profile[bsps] = self.config.comma_list(profile['name'], bsps)
938	self.profiles[profile['name']] = profile
939	invalid_chars = re.compile(r'[^a-zA-Z0-9_-]')
940	for a in set(archs):
941	if len(invalid_chars.findall(a)) != 0:
942	raise error.general('invalid character(s) in arch name: %s' % (a))
943	arch = {}
944	arch['excludes'] = {}
945	for exclude in self.config.comma_list(a, 'exclude', err = False):
946	arch['excludes'][exclude] = ['all']
947	for i in self.config.get_items(a, False):
948	if i[0].startswith('exclude-'):
949	exclude = i[0][len('exclude-'):]
950	if exclude not in arch['excludes']:
951	arch['excludes'][exclude] = []
952	arch['excludes'][exclude] += \
953	sorted(set([b.strip() for b in i[1].split(',')]))
954	arch['bsps'] = self.config.comma_list(a, 'bsps', err = False)
955	for b in arch['bsps']:
956	if len(invalid_chars.findall(b)) != 0:
957	raise error.general('invalid character(s) in BSP name: %s' % (b))
958	arch[b] = {}
959	arch[b]['bspopts'] = \
960	self.config.comma_list(a, 'bspopts_%s' % (b), err = False)
961	self.archs[a] = arch
962	builds = {}
963	builds['default'] = self.config.get_item('builds', 'default')
964	if build is None:
965	build = builds['default']
966	builds['config'] = { }
967	for config in self.config.get_items('config'):
968	builds['config'][config[0]] = config[1]
969	builds['build'] = build
970	builds_ = self.config.get_item_names('builds')
971	builds['builds'] = {}
972	for build in builds_:
973	build_builds = self.config.comma_list('builds', build)
974	has_config = False
975	has_build = False
976	for b in build_builds:
977	if ':' in b:
978	if has_build:
979	raise error.general('config and build in build: %s' % (build))
980	has_config = True
981	else:
982	if has_config:
983	raise error.general('config and build in build: %s' % (build))
984	has_build = True
985	builds['builds'][build] = build_builds
986	self.builds_ = builds
987
988	def configs(self):
989	return sorted(list(self.builds_['config'].keys()))
990
991	def config_flags(self, config):
992	if config not in self.builds_['config']:
993	raise error.general('config entry not found: %s' % (config))
994	return self.builds_['config'][config]
995
996	def build(self):
997	return self.builds_['build']
998
999	def builds(self):
1000	if self.builds_['build'] in self.builds_['builds']:
1001	build = copy.copy(self.builds_['builds'][self.builds_['build']])
1002	if ':' in build[0]:
1003	return [self.builds_['build']]
1004	return build
1005	return None
1006
1007	def build_options(self, build):
1008	return ' '.join(self._build_options(build))
1009
1010	def excludes(self, arch, bsp):
1011	return list(set(self.arch_excludes(arch) + self.bsp_excludes(arch, bsp)))
1012
1013	def exclude_options(self, arch, bsp):
1014	return ' '.join([self.config_flags('no-' + e) for e in self.excludes(arch, bsp)])
1015
1016	def archs(self):
1017	return sorted(self.archs.keys())
1018
1019	def arch_present(self, arch):
1020	return arch in self.archs
1021
1022	def arch_excludes(self, arch):
1023	excludes = self.archs[arch]['excludes'].keys()
1024	for exclude in self.archs[arch]['excludes']:
1025	if 'all' not in self.archs[arch]['excludes'][exclude]:
1026	excludes.remove(exclude)
1027	return sorted(excludes)
1028
1029	def arch_bsps(self, arch):
1030	return sorted(self.archs[arch]['bsps'])
1031
1032	def bsp_present(self, arch, bsp):
1033	return bsp in self.archs[arch]['bsps']
1034
1035	def bsp_excludes(self, arch, bsp):
1036	excludes = self.archs[arch]['excludes'].keys()
1037	for exclude in self.archs[arch]['excludes']:
1038	if 'all' not in self.archs[arch]['excludes'][exclude] and \
1039	bsp not in self.archs[arch]['excludes'][exclude]:
1040	excludes.remove(exclude)
1041	return sorted(excludes)
1042
1043	def bspopts(self, arch, bsp):
1044	if arch not in self.archs:
1045	raise error.general('invalid architecture: %s' % (arch))
1046	if bsp not in self.archs[arch]:
1047	raise error.general('invalid BSP: %s' % (bsp))
1048	return self.archs[arch][bsp]['bspopts']
1049
1050	def profile_present(self, profile):
1051	return profile in self.profiles
1052
1053	def profile_archs(self, profile):
1054	if profile not in self.profiles:
1055	raise error.general('invalid profile: %s' % (profile))
1056	return self.profiles[profile]['archs']
1057
1058	def profile_arch_bsps(self, profile, arch):
1059	if profile not in self.profiles:
1060	raise error.general('invalid profile: %s' % (profile))
1061	if 'bsps_%s' % (arch) not in self.profiles[profile]:
1062	raise error.general('invalid profile arch: %s' % (arch))
1063	return ['%s/%s' % (arch, bsp) for bsp in self.profiles[profile]['bsps_%s' % (arch)]]
1064
1065	def report(self, profiles = True, builds = True, architectures = True):
1066	width = 70
1067	cols_1 = [width]
1068	cols_2 = [10, width - 10]
1069	s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
1070	s1 = ' File(s)'
1071	for f in self.config.files():
1072	colon = ':'
1073	for l in textwrap.wrap(f, width = cols_2[1] - 3):
1074	s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1)
1075	colon = ' '
1076	s1 = ' ' * len(s1)
1077	s += textbox.line(cols_1, marker = '+', indent = 1)
1078	s += os.linesep
1079	if profiles:
1080	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
1081	profiles = sorted(self.profiles['profiles'])
1082	archs = []
1083	bsps = []
1084	for profile in profiles:
1085	archs += self.profiles[profile]['archs']
1086	for arch in sorted(self.profiles[profile]['archs']):
1087	bsps += self.profiles[profile]['bsps_%s' % (arch)]
1088	archs = len(set(archs))
1089	bsps = len(set(bsps))
1090	s += textbox.row(cols_1,
1091	[' Profiles : %d (archs:%d, bsps:%d)' % \
1092	(len(profiles), archs, bsps)],
1093	indent = 1)
1094	for profile in profiles:
1095	textbox.row(cols_2,
1096	[profile, self.profiles[profile]['name']],
1097	indent = 1)
1098	s += textbox.line(cols_1, marker = '+', indent = 1)
1099	for profile in profiles:
1100	s += textbox.row(cols_1, [' %s' % (profile)], indent = 1)
1101	profile = self.profiles[profile]
1102	archs = sorted(profile['archs'])
1103	for arch in archs:
1104	arch_bsps = ', '.join(profile['bsps_%s' % (arch)])
1105	if len(arch_bsps) > 0:
1106	s += textbox.line(cols_2, marker = '+', indent = 1)
1107	s1 = ' ' + arch
1108	for l in textwrap.wrap(arch_bsps,
1109	width = cols_2[1] - 3):
1110	s += textbox.row(cols_2, [s1, ' ' + l], indent = 1)
1111	s1 = ' ' * len(s1)
1112	s += textbox.line(cols_2, marker = '+', indent = 1)
1113	s += os.linesep
1114	if builds:
1115	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
1116	s += textbox.row(cols_1,
1117	[' Builds: %s (default)' % (self.builds_['default'])],
1118	indent = 1)
1119	builds = self.builds_['builds']
1120	bsize = 0
1121	for build in builds:
1122	if len(build) > bsize:
1123	bsize = len(build)
1124	cols_b = [bsize + 2, width - bsize - 2]
1125	s += textbox.line(cols_b, marker = '+', indent = 1)
1126	for build in builds:
1127	s1 = ' ' + build
1128	for l in textwrap.wrap(', '.join(builds[build]),
1129	width = cols_b[1] - 3):
1130	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
1131	s1 = ' ' * len(s1)
1132	s += textbox.line(cols_b, marker = '+', indent = 1)
1133	configs = self.builds_['config']
1134	s += textbox.row(cols_1,
1135	[' Configure Options: %d' % (len(configs))],
1136	indent = 1)
1137	csize = 0
1138	for config in configs:
1139	if len(config) > csize:
1140	csize = len(config)
1141	cols_c = [csize + 3, width - csize - 3]
1142	s += textbox.line(cols_c, marker = '+', indent = 1)
1143	for config in configs:
1144	s1 = ' ' + config
1145	for l in textwrap.wrap(configs[config], width = cols_c[1] - 3):
1146	s += textbox.row(cols_c, [s1, ' ' + l], indent = 1)
1147	s1 = ' ' * len(s1)
1148	s += textbox.line(cols_c, marker = '+', indent = 1)
1149	s += os.linesep
1150	if architectures:
1151	s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
1152	archs = sorted(self.archs.keys())
1153	bsps = 0
1154	asize = 0
1155	for arch in archs:
1156	if len(arch) > asize:
1157	asize = len(arch)
1158	bsps += len(self.archs[arch]['bsps'])
1159	s += textbox.row(cols_1,
1160	[' Architectures : %d (bsps: %d)' % (len(archs), bsps)],
1161	indent = 1)
1162	cols_a = [asize + 2, width - asize - 2]
1163	s += textbox.line(cols_a, marker = '+', indent = 1)
1164	for arch in archs:
1165	s += textbox.row(cols_a,
1166	[' ' + arch, ' %d' % (len(self.archs[arch]['bsps']))],
1167	indent = 1)
1168	s += textbox.line(cols_a, marker = '+', indent = 1)
1169	for archn in archs:
1170	arch = self.archs[archn]
1171	if len(arch['bsps']) > 0:
1172	bsize = 0
1173	for bsp in arch['bsps']:
1174	if len(bsp) > bsize:
1175	bsize = len(bsp)
1176	cols_b = [bsize + 3, width - bsize - 3]
1177	s += textbox.row(cols_1, [' ' + archn + ':'], indent = 1)
1178	s += textbox.line(cols_b, marker = '+', indent = 1)
1179	for bsp in arch['bsps']:
1180	s1 = ' ' + bsp
1181	bspopts = ' '.join(arch[bsp]['bspopts'])
1182	if len(bspopts):
1183	for l in textwrap.wrap('bopt: ' + bspopts,
1184	width = cols_b[1] - 3):
1185	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
1186	s1 = ' ' * len(s1)
1187	excludes = []
1188	for exclude in arch['excludes']:
1189	if 'all' in arch['excludes'][exclude] or \
1190	bsp in arch['excludes'][exclude]:
1191	excludes += [exclude]
1192	excludes = ', '.join(excludes)
1193	if len(excludes):
1194	for l in textwrap.wrap('ex: ' + excludes,
1195	width = cols_b[1] - 3):
1196	s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
1197	s1 = ' ' * len(s1)
1198	if len(bspopts) == 0 and len(excludes) == 0:
1199	s += textbox.row(cols_b, [s1, ' '], indent = 1)
1200	s += textbox.line(cols_b, marker = '+', indent = 1)
1201	s += os.linesep
1202	return s
1203
1204	class build_jobs:
1205
1206	def __init__(self, config, arch, bsp):
1207	self.arch = arch
1208	self.bsp = bsp
1209	self.builds = config.builds()
1210	if self.builds is None:
1211	raise error.general('build not found: %s' % (config.build()))
1212	valid_configs = config.configs()
1213	excludes = config.excludes(self.arch, self.bsp)
1214	for e in excludes:
1215	if e.startswith('no-'):
1216	raise error.general('excludes cannot start with "no-": %s' % (e))
1217	if e not in valid_configs:
1218	raise error.general('invalid exclude: %s' % (e))
1219	#
1220	# The build can be in the build string delimited by '-'.
1221	#
1222	remove = []
1223	for e in excludes:
1224	remove += [b for b in self.builds if e in b.split('-')]
1225	self.builds = [b for b in self.builds if b not in remove]
1226	self.build_set = { }
1227	exclude_options = ' ' + config.exclude_options(self.arch, self.bsp)
1228	for build in self.builds:
1229	self.build_set[build] = config.build_options(build) + exclude_options
1230
1231	def jobs(self):
1232	return [arch_bsp_build(self.arch, self.bsp, b, self.build_set[b]) \
1233	for b in sorted(self.build_set.keys())]
1234
1235	class builder:
1236
1237	def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
1238	self.config = config
1239	self.build_dir = build_dir
1240	self.rtems_version = version
1241	self.prefix = prefix
1242	self.tools = tools
1243	self.rtems = rtems
1244	self.options = options
1245	self.counts = { 'h' : 0,
1246	'exes' : 0,
1247	'objs' : 0,
1248	'libs' : 0 }
1249	self.results = results(rtems,
1250	{ 'groups' : ['Shared', 'BSP', 'Network', 'Tests',
1251	'LibCPU', 'CPU Kit'],
1252	'exclude' : '.Makefile.',
1253	'CPU Kit' : '.cpukit/.',
1254	'Network' : '.libnetworking/.\|.librpc/.',
1255	'Tests' : '.testsuites/.',
1256	'BSP' : '.libbsp/.',
1257	'LibCPU' : '.libcpu/.',
1258	'Shared' : '.shared/.' })
1259	if not path.exists(path.join(rtems, 'configure')) or \
1260	not path.exists(path.join(rtems, 'Makefile.in')) or \
1261	not path.exists(path.join(rtems, 'cpukit')):
1262	raise error.general('RTEMS source path does not look like RTEMS')
1263
1264	def _bsps(self, arch):
1265	return self.config.arch_bsps(arch)
1266
1267	def _create_build_jobs(self, jobs, build_job_count):
1268	max_job_size = len('%d' % len(jobs))
1269	build_jobs = []
1270	job_index = 1
1271	for job in jobs:
1272	tag = '%*d/%d' % (max_job_size, job_index, len(jobs))
1273	build_jobs += [arch_bsp_builder(self.results,
1274	job,
1275	self._commands(job, build_job_count),
1276	self.build_dir,
1277	tag)]
1278	job_index += 1
1279	set_max_build_label(build_jobs)
1280	return build_jobs
1281
1282	def _commands(self, build, build_jobs):
1283	commands = { 'dry-run' : self.options['dry-run'],
1284	'no-clean' : self.options['no-clean'],
1285	'configure': None,
1286	'build' : None }
1287	cmds = build.build_config.split()
1288	cmds += self.config.bspopts(build.arch, build.bsp)
1289	cmd = [path.join(self.rtems, 'configure')]
1290	for c in cmds:
1291	c = c.replace('@PREFIX@', self.prefix)
1292	c = c.replace('@RTEMS_VERSION@', self.rtems_version)
1293	c = c.replace('@ARCH@', build.arch)
1294	c = c.replace('@BSP@', build.bsp)
1295	cmd += [c]
1296	commands['configure'] = ' '.join(cmd)
1297	cmd = 'make -j %s' % (build_jobs)
1298	commands['build'] = cmd
1299	return commands
1300
1301	def _update_file_counts(self, counts):
1302	for f in self.counts:
1303	if f in counts:
1304	self.counts[f] += counts[f]
1305	return counts
1306
1307	def _warnings_report(self):
1308	if self.options['warnings-report'] is not None:
1309	with open(self.options['warnings-report'], 'w') as f:
1310	f.write(title() + os.linesep)
1311	f.write(os.linesep)
1312	f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'),
1313	os.linesep))
1314	f.write(os.linesep)
1315	f.write(command_line() + os.linesep)
1316	f.write(self.results.warnings_errors.warnings_report())
1317
1318	def _failures_report(self):
1319	if self.options['failures-report'] is not None:
1320	with open(self.options['failures-report'], 'w') as f:
1321	f.write(title() + os.linesep)
1322	f.write(os.linesep)
1323	f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'),
1324	os.linesep))
1325	f.write(os.linesep)
1326	f.write(command_line() + os.linesep)
1327	f.write(self.results.failures_report())
1328
1329	def _finished(self):
1330	log.notice('Total: Warnings:%d exes:%d objs:%d libs:%d' % \
1331	(self.results.get_warning_count(), self.counts['exes'],
1332	self.counts['objs'], self.counts['libs']))
1333	log.output()
1334	log.output('Warnings:')
1335	log.output(self.results.warnings_report())
1336	log.output()
1337	log.notice('Failures:')
1338	log.notice(self.results.failures_report())
1339	self._warnings_report()
1340	self._failures_report()
1341
1342	def run_jobs(self, jobs):
1343	if path.exists(self.build_dir):
1344	log.notice('Cleaning: %s' % (self.build_dir))
1345	path.removeall(self.build_dir)
1346	self.start = datetime.datetime.now()
1347	self.end = datetime.datetime.now()
1348	self.duration = self.end - self.start
1349	self.average = self.duration
1350	env_path = os.environ['PATH']
1351	os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
1352	os.pathsep + os.environ['PATH']
1353	job_count, build_job_count = jobs_option_parse(self.options['jobs'])
1354	builds = self._create_build_jobs(jobs, build_job_count)
1355	active_jobs = []
1356	self.jobs_completed = 0
1357	try:
1358	while len(builds) > 0 or len(active_jobs) > 0:
1359	new_jobs = job_count - len(active_jobs)
1360	if new_jobs > 0:
1361	active_jobs += builds[:new_jobs]
1362	builds = builds[new_jobs:]
1363	finished_jobs = []
1364	for job in active_jobs:
1365	state = job.current_state()
1366	if state == 'ready':
1367	job.run()
1368	elif state != 'running':
1369	finished_jobs += [job]
1370	for job in finished_jobs:
1371	self._update_file_counts(job.get_file_counts())
1372	job.log_output()
1373	job.clean()
1374	active_jobs.remove(job)
1375	self.jobs_completed += 1
1376	if self.options['dry-run']:
1377	time.sleep(0)
1378	else:
1379	time.sleep(0.100)
1380	except:
1381	for job in active_jobs:
1382	try:
1383	job.kill()
1384	except:
1385	pass
1386	raise
1387	self.end = datetime.datetime.now()
1388	os.environ['PATH'] = env_path
1389	self.duration = self.end - self.start
1390	if self.jobs_completed == 0:
1391	self.jobs_completed = 1
1392	self._finished()
1393	self.average = self.duration / self.jobs_completed
1394	log.notice('Average BSP Build Time: %s' % (str(self.average)))
1395	log.notice('Total Time %s' % (str(self.duration)))
1396
1397	def arch_bsp_jobs(self, arch, bsps):
1398	jobs = []
1399	for bsp in bsps:
1400	jobs += build_jobs(self.config, arch, bsp).jobs()
1401	return jobs
1402
1403	def bsp_jobs(self, bsps):
1404	jobs = []
1405	for bsp in bsps:
1406	if bsp.count('/') != 1:
1407	raise error.general('invalid bsp format (use: arch/bsp): %s' % (bsp))
1408	arch, bsp = bsp.split('/')
1409	jobs += build_jobs(self.config, arch, bsp).jobs()
1410	return jobs
1411
1412	def arch_jobs(self, archs):
1413	jobs = []
1414	for arch in archs:
1415	if not self.config.arch_present(arch):
1416	raise error.general('Architecture not found: %s' % (arch))
1417	jobs += self.arch_bsp_jobs(arch, self._bsps(arch))
1418	return jobs
1419
1420	def profile_jobs(self, profiles):
1421	jobs = []
1422	for profile in profiles:
1423	if not self.config.profile_present(profile):
1424	raise error.general('Profile not found: %s' % (profile))
1425	for arch in self.config.profile_archs(profile):
1426	jobs += self.bsp_jobs(self.config.profile_arch_bsps(profile, arch))
1427	return jobs
1428
1429	def build_bsps(self, bsps):
1430	log.notice('BSPS(s): %s' % (', '.join(bsps)))
1431	self.run_jobs(self.bsp_jobs(bsps))
1432
1433	def build_archs(self, archs):
1434	log.notice('Architecture(s): %s' % (', '.join(archs)))
1435	self.run_jobs(self.arch_jobs(archs))
1436
1437	def build_profiles(self, profiles):
1438	log.notice('Profile(s): %s' % (', '.join(profiles)))
1439	self.run_jobs(self.profile_jobs(profiles))
1440
1441	def run_args(args):
1442	b = None
1443	ec = 0
1444	try:
1445	#
1446	# On Windows MSYS2 prepends a path to itself to the environment
1447	# path. This means the RTEMS specific automake is not found and which
1448	# breaks the bootstrap. We need to remove the prepended path. Also
1449	# remove any ACLOCAL paths from the environment.
1450	#
1451	if os.name == 'nt':
1452	cspath = os.environ['PATH'].split(os.pathsep)
1453	if 'msys' in cspath[0] and cspath[0].endswith('bin'):
1454	os.environ['PATH'] = os.pathsep.join(cspath[1:])
1455
1456	start = datetime.datetime.now()
1457	top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
1458	prefix = '/opt/rtems/%s' % (rtems_version())
1459	tools = prefix
1460	build_dir = 'bsp-builds'
1461	logf = 'bsp-build-%s.txt' % \
1462	(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
1463	config_file = path.join(top, 'share', 'rtems', 'tester',
1464	'rtems', 'rtems-bsps.ini')
1465	if not path.exists(config_file):
1466	config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')
1467
1468	argsp = argparse.ArgumentParser()
1469	argsp.add_argument('--prefix', help = 'Prefix to build the BSP.',
1470	type = str)
1471	argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.',
1472	type = str)
1473	argsp.add_argument('--rtems', help = 'The RTEMS source tree.',
1474	type = str)
1475	argsp.add_argument('--build-path', help = 'Path to build in.',
1476	type = str)
1477	argsp.add_argument('--log', help = 'Log file.', type = str)
1478	argsp.add_argument('--config-report', help = 'Report the configuration.',
1479	type = str, default = None,
1480	choices = ['all', 'profiles', 'builds', 'archs'])
1481	argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
1482	type = str, default = None)
1483	argsp.add_argument('--failures-report', help = 'Report the failures to a file.',
1484	type = str, default = None)
1485	argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
1486	action = 'store_true')
1487	argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
1488	action = 'store_true')
1489	argsp.add_argument('--profiles', help = 'Build the listed profiles (profile,profile,..).',
1490	type = str, default = 'tier-1')
1491	argsp.add_argument('--arch', help = 'Build the architectures (arch,arch,..).',
1492	type = str)
1493	argsp.add_argument('--bsp', help = 'Build the BSPs (arch/bsp,arch/bsp,..).',
1494	type = str)
1495	argsp.add_argument('--build', help = 'Build name to build (see --config-report).',
1496	type = str, default='all')
1497	argsp.add_argument('--jobs', help = 'Number of jobs to run.',
1498	type = str, default = '1/%d' % (host.cpus()))
1499	argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
1500	action = 'store_true')
1501	mailer.add_arguments(argsp)
1502
1503	opts = argsp.parse_args(args[1:])
1504	mail = None
1505	if opts.mail:
1506	mail = mailer.mail(opts)
1507	# Request these now to generate any errors.
1508	from_addr = mail.from_address()
1509	smtp_host = mail.smtp_host()
1510	if 'mail_to' in opts and opts.mail_to is not None:
1511	to_addr = opts.mail_to
1512	else:
1513	to_addr = 'build@rtems.org'
1514	if opts.log is not None:
1515	logf = opts.log
1516	log.default = log.log([logf])
1517	log.notice(title())
1518	log.output(command_line())
1519	if mail:
1520	log.notice('Mail: from:%s to:%s smtp:%s' % (from_addr,
1521	to_addr,
1522	smtp_host))
1523
1524	config = configuration_()
1525	config.load(config_file, opts.build)
1526
1527	if opts.config_report:
1528	log.notice('Configuration Report: %s' % (opts.config_report))
1529	c_profiles = False
1530	c_builds = False
1531	c_archs = False
1532	if opts.config_report == 'all':
1533	c_profiles = True
1534	c_builds = True
1535	c_archs = True
1536	elif opts.config_report == 'profiles':
1537	c_profiles = True
1538	elif opts.config_report == 'builds':
1539	c_builds = True
1540	elif opts.config_report == 'archs':
1541	c_archs = True
1542	log.notice(config.report(c_profiles, c_builds, c_archs))
1543	sys.exit(0)
1544
1545	if opts.rtems is None:
1546	raise error.general('No RTEMS source provided on the command line')
1547	if opts.prefix is not None:
1548	prefix = path.shell(opts.prefix)
1549	if opts.rtems_tools is not None:
1550	tools = path.shell(opts.rtems_tools)
1551	if opts.build_path is not None:
1552	build_dir = path.shell(opts.build_path)
1553
1554	options = { 'stop-on-error' : opts.stop_on_error,
1555	'no-clean' : opts.no_clean,
1556	'dry-run' : opts.dry_run,
1557	'jobs' : opts.jobs,
1558	'warnings-report' : opts.warnings_report,
1559	'failures-report' : opts.failures_report }
1560
1561	b = builder(config, rtems_version(), prefix, tools,
1562	path.shell(opts.rtems), build_dir, options)
1563
1564	profiles = comma_split(opts.profiles)
1565	archs = comma_split(opts.arch)
1566	bsps = comma_split(opts.bsp)
1567
1568	#
1569	# The default is build a profile.
1570	#
1571	if bsps is not None:
1572	if archs is not None:
1573	raise error.general('--arch supplied with --bsp;' \
1574	' use --bsp=arch/bsp,arch/bsp,..')
1575	what = 'BSPs: %s' % (' '.join(bsps))
1576	b.build_bsps(bsps)
1577	elif archs is not None:
1578	what = 'Archs: %s' % (' '.join(archs))
1579	b.build_archs(archs)
1580	else:
1581	what = 'Profile(s): %s' % (' '.join(profiles))
1582	b.build_profiles(profiles)
1583	end = datetime.datetime.now()
1584
1585	#
1586	# Email the results of the build.
1587	#
1588	if mail is not None:
1589	subject = '[rtems-bsp-builder] %s: %s' % (str(start).split('.')[0],
1590	what)
1591	t = title()
1592	body = t + os.linesep
1593	body += '=' * len(t) + os.linesep
1594	body += os.linesep
1595	body += 'Host: %s' % (os.uname()[3]) + os.linesep
1596	body += os.linesep
1597	body += command_line()
1598	body += os.linesep
1599	body += 'Total Time : %s for %d completed job(s)' % \
1600	(str(b.duration), b.jobs_completed)
1601	body += os.linesep
1602	body += 'Average BSP Build Time: %s' % (str(b.average))
1603	body += os.linesep + os.linesep
1604	body += 'Builds' + os.linesep
1605	body += '======' + os.linesep
1606	body += os.linesep.join([' ' + cb for cb in config.builds()])
1607	body += os.linesep + os.linesep
1608	body += 'Failures Report' + os.linesep
1609	body += '===============' + os.linesep
1610	body += b.results.failures_report()
1611	body += os.linesep
1612	body += 'Warnings Report' + os.linesep
1613	body += '===============' + os.linesep
1614	body += b.results.warnings_report(summary = True)
1615	mail.send(to_addr, subject, body)
1616
1617	except error.general as gerr:
1618	print(gerr)
1619	print('BSP Build FAILED', file = sys.stderr)
1620	ec = 1
1621	except error.internal as ierr:
1622	print(ierr)
1623	print('BSP Build FAILED', file = sys.stderr)
1624	ec = 1
1625	except error.exit as eerr:
1626	pass
1627	except KeyboardInterrupt:
1628	log.notice('abort: user terminated')
1629	ec = 1
1630	if b is not None:
1631	b.results.report()
1632	sys.exit(ec)
1633
1634	def run():
1635	run_args(sys.argv)
1636
1637	if __name__ == "__main__":
1638	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: