Context Navigation

source: rtems-source-builder/source-builder/sb/setbuilder.py @ a293ddc

5

Last change on this file since a293ddc was 4dc9bf3, checked in by Chris Johns <chrisj@…>, on 10/31/17 at 00:57:21
sb: Add platform reporting for better host reviewing in posted email reports.
Property mode set to `100644`
File size: 23.1 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Permission to use, copy, modify, and/or distribute this software for any
9	# purpose with or without fee is hereby granted, provided that the above
10	# copyright notice and this permission notice appear in all copies.
11	#
12	# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13	# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14	# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15	# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16	# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17	# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18	# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20	#
21	# This code builds a package compiler tool suite given a tool set. A tool
22	# set lists the various tools. These are specific tool configurations.
23	#
24
25	from __future__ import print_function
26
27	import copy
28	import datetime
29	import glob
30	import operator
31	import os
32	import sys
33	import textwrap
34
35	try:
36	import build
37	import check
38	import error
39	import log
40	import mailer
41	import options
42	import path
43	import reports
44	import sources
45	import version
46	except KeyboardInterrupt:
47	print('abort: user terminated', file = sys.stderr)
48	sys.exit(1)
49	except:
50	print('error: unknown application load error', file = sys.stderr)
51	sys.exit(1)
52
53	class log_capture(object):
54	def __init__(self):
55	self.log = []
56	log.capture = self.capture
57
58	def __str__(self):
59	return os.linesep.join(self.log)
60
61	def capture(self, text):
62	self.log += [l for l in text.replace(chr(13), '').splitlines()]
63
64	def get(self):
65	return self.log
66
67	def clear(self):
68	self.log = []
69
70	class buildset:
71	"""Build a set builds a set of packages."""
72
73	def __init__(self, bset, _configs, opts, macros = None):
74	log.trace('_bset: %s: init' % (bset))
75	self.configs = _configs
76	self.opts = opts
77	if macros is None:
78	self.macros = copy.copy(opts.defaults)
79	else:
80	self.macros = copy.copy(macros)
81	log.trace('_bset: %s: macro defaults' % (bset))
82	log.trace(str(self.macros))
83	self.bset = bset
84	_target = self.macros.expand('%{_target}')
85	if len(_target):
86	pkg_prefix = _target
87	else:
88	pkg_prefix = self.macros.expand('%{_host}')
89	self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset)
90	self.mail_header = ''
91	self.mail_report = ''
92	self.mail_report_0subject = ''
93	self.build_failure = None
94
95	def write_mail_header(self, text = '', prepend = False):
96	if type(text) is list:
97	text = os.linesep.join(text)
98	text = text.replace('\r', '').replace('\n', os.linesep)
99	if len(text) == 0 or text[-1] != os.linesep:
100	text += os.linesep
101	if prepend:
102	self.mail_header = text + self.mail_header
103	else:
104	self.mail_header += text
105
106	def get_mail_header(self):
107	return self.mail_header
108
109	def write_mail_report(self, text, prepend = False):
110	if type(text) is list:
111	text = os.linesep.join(text)
112	text = text.replace('\r', '').replace('\n', os.linesep)
113	if len(text) == 0 or text[-1] != os.linesep:
114	text += os.linesep
115	if prepend:
116	self.mail_report = text + self.mail_report
117	else:
118	self.mail_report += text
119
120	def get_mail_report(self):
121	return self.mail_report
122
123	def copy(self, src, dst):
124	log.output('copy: %s => %s' % (path.host(src), path.host(dst)))
125	if not self.opts.dry_run():
126	path.copy_tree(src, dst)
127
128	def report(self, _config, _build, opts, macros, format = None, mail = None):
129	if len(_build.main_package().name()) > 0 \
130	and not _build.macros.get('%{_disable_reporting}') \
131	and (not _build.opts.get_arg('--no-report') \
132	or _build.opts.get_arg('--mail')):
133	if format is None:
134	format = _build.opts.get_arg('--report-format')
135	if format is not None:
136	if len(format) != 2:
137	raise error.general('invalid report format option: %s' % \
138	('='.join(format)))
139	format = format[1]
140	if format is None:
141	format = 'text'
142	if format == 'text':
143	ext = '.txt'
144	elif format == 'asciidoc':
145	ext = '.txt'
146	elif format == 'html':
147	ext = '.html'
148	elif format == 'xml':
149	ext = '.xml'
150	elif format == 'ini':
151	ext = '.ini'
152	else:
153	raise error.general('invalid report format: %s' % (format))
154	buildroot = _build.config.abspath('%{buildroot}')
155	prefix = _build.macros.expand('%{_prefix}')
156	name = _build.main_package().name() + ext
157	log.notice('reporting: %s -> %s' % (_config, name))
158	if not _build.opts.get_arg('--no-report'):
159	outpath = path.host(path.join(buildroot, prefix, 'share', 'rtems', 'rsb'))
160	if not _build.opts.dry_run():
161	outname = path.host(path.join(outpath, name))
162	else:
163	outname = None
164	r = reports.report(format, self.configs,
165	copy.copy(opts), copy.copy(macros))
166	r.introduction(_build.config.file_name())
167	r.generate(_build.config.file_name())
168	r.epilogue(_build.config.file_name())
169	if not _build.opts.dry_run():
170	_build.mkdir(outpath)
171	r.write(outname)
172	del r
173	if mail:
174	r = reports.report('text', self.configs,
175	copy.copy(opts), copy.copy(macros))
176	r.introduction(_build.config.file_name())
177	r.generate(_build.config.file_name())
178	r.epilogue(_build.config.file_name())
179	self.write_mail_report(r.get_output())
180	del r
181
182	def root_copy(self, src, dst):
183	what = '%s -> %s' % \
184	(os.path.relpath(path.host(src)), os.path.relpath(path.host(dst)))
185	log.trace('_bset: %s: collecting: %s' % (self.bset, what))
186	self.copy(src, dst)
187
188	def install(self, name, buildroot, prefix):
189	dst = prefix
190	src = path.join(buildroot, prefix)
191	log.notice('installing: %s -> %s' % (name, path.host(dst)))
192	self.copy(src, dst)
193
194	def canadian_cross(self, _build):
195	log.trace('_bset: Cxc for build machine: _build => _host')
196	macros_to_copy = [('%{_host}', '%{_build}'),
197	('%{_host_alias}', '%{_build_alias}'),
198	('%{_host_arch}', '%{_build_arch}'),
199	('%{_host_cpu}', '%{_build_cpu}'),
200	('%{_host_os}', '%{_build_os}'),
201	('%{_host_vendor}', '%{_build_vendor}'),
202	('%{_tmproot}', '%{_tmpcxcroot}'),
203	('%{buildroot}', '%{buildcxcroot}'),
204	('%{_builddir}', '%{_buildcxcdir}')]
205	cxc_macros = _build.copy_init_macros()
206	for m in macros_to_copy:
207	log.trace('_bset: Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
208	cxc_macros[m[0]] = cxc_macros[m[1]]
209	_build.set_macros(cxc_macros)
210	_build.reload()
211	_build.make()
212	if not _build.macros.get('%{_disable_collecting}'):
213	self.root_copy(_build.config.expand('%{buildroot}'),
214	_build.config.expand('%{_tmproot}'))
215	_build.set_macros(_build.copy_init_macros())
216	_build.reload()
217
218	def build_package(self, _config, _build):
219	if not _build.disabled():
220	if _build.canadian_cross():
221	self.canadian_cross(_build)
222	_build.make()
223	if not _build.macros.get('%{_disable_collecting}'):
224	self.root_copy(_build.config.expand('%{buildroot}'),
225	_build.config.expand('%{_tmproot}'))
226
227	def bset_tar(self, _build):
228	tardir = _build.config.expand('%{_tardir}')
229	if (self.opts.get_arg('--bset-tar-file') or self.opts.canadian_cross()) \
230	and not _build.macros.get('%{_disable_packaging}'):
231	path.mkdir(tardir)
232	tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (_build.main_package().name())))
233	log.notice('tarball: %s' % (os.path.relpath(path.host(tar))))
234	if not self.opts.dry_run():
235	tmproot = _build.config.expand('%{_tmproot}')
236	cmd = _build.config.expand('"cd ' + tmproot + \
237	' && %{__tar} -cf - . \| %{__bzip2} > ' + tar + '"')
238	_build.run(cmd, shell_opts = '-c', cwd = tmproot)
239
240	def parse(self, bset):
241
242	def _clean(line):
243	line = line[0:-1]
244	b = line.find('#')
245	if b >= 0:
246	line = line[1:b]
247	return line.strip()
248
249	bsetname = bset
250
251	if not path.exists(bsetname):
252	for cp in self.macros.expand('%{_configdir}').split(':'):
253	configdir = path.abspath(cp)
254	bsetname = path.join(configdir, bset)
255	if path.exists(bsetname):
256	break
257	bsetname = None
258	if bsetname is None:
259	raise error.general('no build set file found: %s' % (bset))
260	try:
261	log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
262	bset = open(path.host(bsetname), 'r')
263	except IOError as err:
264	raise error.general('error opening bset file: %s' % (bsetname))
265
266	configs = []
267
268	try:
269	lc = 0
270	for l in bset:
271	lc += 1
272	l = _clean(l)
273	if len(l) == 0:
274	continue
275	log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l))
276	ls = l.split()
277	if ls[0][-1] == ':' and ls[0][:-1] == 'package':
278	self.bset_pkg = ls[1].strip()
279	self.macros['package'] = self.bset_pkg
280	elif ls[0][0] == '%':
281	def err(msg):
282	raise error.general('%s:%d: %s' % (self.bset, lc, msg))
283	if ls[0] == '%define':
284	if len(ls) > 2:
285	self.macros.define(ls[1].strip(),
286	' '.join([f.strip() for f in ls[2:]]))
287	else:
288	self.macros.define(ls[1].strip())
289	elif ls[0] == '%undefine':
290	if len(ls) > 2:
291	raise error.general('%s:%d: %undefine requires just the name' % \
292	(self.bset, lc))
293	self.macros.undefine(ls[1].strip())
294	elif ls[0] == '%include':
295	configs += self.parse(ls[1].strip())
296	elif ls[0] in ['%patch', '%source']:
297	sources.process(ls[0][1:], ls[1:], self.macros, err)
298	elif ls[0] == '%hash':
299	sources.hash(ls[1:], self.macros, err)
300	else:
301	l = l.strip()
302	c = build.find_config(l, self.configs)
303	if c is None:
304	raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l))
305	configs += [c]
306	except:
307	bset.close()
308	raise
309
310	bset.close()
311
312	return configs
313
314	def load(self):
315
316	#
317	# If the build set file ends with .cfg the user has passed to the
318	# buildset builder a configuration so we just return it.
319	#
320	if self.bset.endswith('.cfg'):
321	configs = [self.bset]
322	else:
323	exbset = self.macros.expand(self.bset)
324	self.macros['_bset'] = exbset
325	root, ext = path.splitext(exbset)
326	if exbset.endswith('.bset'):
327	bset = exbset
328	else:
329	bset = '%s.bset' % (exbset)
330	configs = self.parse(bset)
331	return configs
332
333	def build(self, deps = None, nesting_count = 0, mail = None):
334
335	build_error = False
336
337	nesting_count += 1
338
339	if mail:
340	mail['output'].clear()
341
342	log.trace('_bset: %s: make' % (self.bset))
343	log.notice('Build Set: %s' % (self.bset))
344
345	mail_subject = '%s on %s' % (self.bset,
346	self.macros.expand('%{_host}'))
347
348	current_path = os.environ['PATH']
349
350	start = datetime.datetime.now()
351
352	mail_report = False
353	have_errors = False
354
355	if mail:
356	mail['output'].clear()
357
358	try:
359	configs = self.load()
360
361	log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
362
363	builds = []
364	for s in range(0, len(configs)):
365	b = None
366	try:
367	#
368	# Each section of the build set gets a separate set of
369	# macros so we do not contaminate one configuration with
370	# another.
371	#
372	opts = copy.copy(self.opts)
373	macros = copy.copy(self.macros)
374	if configs[s].endswith('.bset'):
375	log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
376	bs = buildset(configs[s], self.configs, opts, macros)
377	bs.build(deps, nesting_count, mail)
378	del bs
379	elif configs[s].endswith('.cfg'):
380	if mail:
381	mail_report = True
382	log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
383	try:
384	b = build.build(configs[s],
385	self.opts.get_arg('--pkg-tar-files'),
386	opts,
387	macros)
388	except:
389	build_error = True
390	raise
391	if b.macros.get('%{_disable_reporting}'):
392	mail_report = False
393	if deps is None:
394	self.build_package(configs[s], b)
395	self.report(configs[s], b,
396	copy.copy(self.opts),
397	copy.copy(self.macros),
398	mail = mail)
399	# Always produce an XML report.
400	self.report(configs[s], b,
401	copy.copy(self.opts),
402	copy.copy(self.macros),
403	format = 'xml',
404	mail = mail)
405	if s == len(configs) - 1 and not have_errors:
406	self.bset_tar(b)
407	else:
408	deps += b.config.includes()
409	builds += [b]
410	#
411	# Dump post build macros.
412	#
413	log.trace('_bset: macros post-build')
414	log.trace(str(macros))
415	else:
416	raise error.general('invalid config type: %s' % (configs[s]))
417	except error.general as gerr:
418	have_errors = True
419	if b is not None:
420	if self.build_failure is None:
421	self.build_failure = b.name()
422	self.write_mail_header('')
423	self.write_mail_header('= ' * 40)
424	self.write_mail_header('Build FAILED: %s' % (b.name()))
425	self.write_mail_header('- ' * 40)
426	self.write_mail_header(str(log.default))
427	self.write_mail_header('- ' * 40)
428	if self.opts.keep_going():
429	log.notice(str(gerr))
430	if self.opts.always_clean():
431	builds += [b]
432	else:
433	raise
434	else:
435	raise
436	#
437	# Installing ...
438	#
439	log.trace('_bset: installing: deps:%r no-install:%r' % \
440	(deps is None, self.opts.no_install()))
441	if deps is None \
442	and not self.opts.no_install() \
443	and not have_errors:
444	for b in builds:
445	log.trace('_bset: installing: %r' % b.installable())
446	if b.installable():
447	self.install(b.name(),
448	b.config.expand('%{buildroot}'),
449	b.config.expand('%{_prefix}'))
450
451	if deps is None and \
452	(not self.opts.no_clean() or self.opts.always_clean()):
453	for b in builds:
454	if not b.disabled():
455	log.notice('cleaning: %s' % (b.name()))
456	b.cleanup()
457	for b in builds:
458	del b
459	except error.general as gerr:
460	if not build_error:
461	log.stderr(str(gerr))
462	raise
463	except KeyboardInterrupt:
464	mail_report = False
465	raise
466	except:
467	self.build_failure = 'RSB general failure'
468	raise
469	finally:
470	end = datetime.datetime.now()
471	os.environ['PATH'] = current_path
472	build_time = str(end - start)
473	if mail_report and not self.macros.defined('mail_disable'):
474	self.write_mail_header('Build Time: %s' % (build_time), True)
475	self.write_mail_header('', True)
476	if self.build_failure is not None:
477	mail_subject = 'FAILED %s (%s)' % \
478	(mail_subject, self.build_failure)
479	else:
480	mail_subject = 'PASSED %s' % (mail_subject)
481	mail_subject = 'Build %s: %s' % (reports.platform(mode = 'system'),
482	mail_subject)
483	self.write_mail_header(mail['header'], True)
484	self.write_mail_header('')
485	log.notice('Mailing report: %s' % (mail['to']))
486	body = self.get_mail_header()
487	body += 'Output' + os.linesep
488	body += '======' + os.linesep + os.linesep
489	body += os.linesep.join(mail['output'].get())
490	body += os.linesep + os.linesep
491	body += 'Report' + os.linesep
492	body += '======' + os.linesep + os.linesep
493	body += self.get_mail_report()
494	if not opts.dry_run():
495	mail['mail'].send(mail['to'], mail_subject, body)
496	log.notice('Build Set: Time %s' % (build_time))
497
498	def list_bset_cfg_files(opts, configs):
499	if opts.get_arg('--list-configs') or opts.get_arg('--list-bsets'):
500	if opts.get_arg('--list-configs'):
501	ext = '.cfg'
502	else:
503	ext = '.bset'
504	for p in configs['paths']:
505	print('Examining: %s' % (os.path.relpath(p)))
506	for c in configs['files']:
507	if c.endswith(ext):
508	print(' %s' % (c))
509	return True
510	return False
511
512	def run():
513	import sys
514	ec = 0
515	setbuilder_error = False
516	mail = None
517	try:
518	optargs = { '--list-configs': 'List available configurations',
519	'--list-bsets': 'List available build sets',
520	'--list-deps': 'List the dependent files.',
521	'--bset-tar-file': 'Create a build set tar file',
522	'--pkg-tar-files': 'Create package tar files',
523	'--no-report': 'Do not create a package report.',
524	'--report-format': 'The report format (text, html, asciidoc).' }
525	mailer.append_options(optargs)
526	opts = options.load(sys.argv, optargs)
527	if opts.get_arg('--mail'):
528	mail = { 'mail' : mailer.mail(opts),
529	'output': log_capture() }
530	to_addr = opts.get_arg('--mail-to')
531	if to_addr is not None:
532	mail['to'] = to_addr[1]
533	else:
534	mail['to'] = opts.defaults.expand('%{_mail_tools_to}')
535	mail['from'] = mail['mail'].from_address()
536	log.notice('RTEMS Source Builder - Set Builder, %s' % (version.str()))
537	opts.log_info()
538	if not check.host_setup(opts):
539	raise error.general('host build environment is not set up correctly')
540	if mail:
541	mail['header'] = os.linesep.join(mail['output'].get()) + os.linesep
542	mail['header'] += os.linesep
543	mail['header'] += 'Host: ' + reports.platform('compact') + os.linesep
544	indent = ' '
545	for l in textwrap.wrap(reports.platform('extended'),
546	width = 80 - len(indent)):
547	mail['header'] += indent + l + os.linesep
548	configs = build.get_configs(opts)
549	if opts.get_arg('--list-deps'):
550	deps = []
551	else:
552	deps = None
553	if not list_bset_cfg_files(opts, configs):
554	prefix = opts.defaults.expand('%{_prefix}')
555	if opts.canadian_cross():
556	opts.disable_install()
557
558	if not opts.dry_run() and \
559	not opts.canadian_cross() and \
560	not opts.no_install() and \
561	not path.ispathwritable(prefix):
562	raise error.general('prefix is not writable: %s' % (path.host(prefix)))
563
564	for bset in opts.params():
565	setbuilder_error = True
566	b = buildset(bset, configs, opts)
567	b.build(deps, mail = mail)
568	b = None
569	setbuilder_error = False
570
571	if deps is not None:
572	c = 0
573	for d in sorted(set(deps)):
574	c += 1
575	print('dep[%d]: %s' % (c, d))
576	except error.general as gerr:
577	if not setbuilder_error:
578	log.stderr(str(gerr))
579	log.stderr('Build FAILED')
580	ec = 1
581	except error.internal as ierr:
582	if not setbuilder_error:
583	log.stderr(str(ierr))
584	log.stderr('Internal Build FAILED')
585	ec = 1
586	except error.exit as eerr:
587	pass
588	except KeyboardInterrupt:
589	log.notice('abort: user terminated')
590	ec = 1
591	except:
592	raise
593	log.notice('abort: unknown error')
594	ec = 1
595	sys.exit(ec)
596
597	if __name__ == "__main__":
598	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: