Context Navigation

source: rtems-source-builder/source-builder/sb/setbuilder.py @ 0f97375

5

Last change on this file since 0f97375 was 0f97375, checked in by Chris Johns <chrisj@…>, on 10/27/17 at 06:25:45

sb: Provide a more detail email message.

Close #3210.

Property mode set to 100644

File size: 22.7 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Permission to use, copy, modify, and/or distribute this software for any
9	# purpose with or without fee is hereby granted, provided that the above
10	# copyright notice and this permission notice appear in all copies.
11	#
12	# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13	# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14	# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15	# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16	# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17	# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18	# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20	#
21	# This code builds a package compiler tool suite given a tool set. A tool
22	# set lists the various tools. These are specific tool configurations.
23	#
24
25	from __future__ import print_function
26
27	import copy
28	import datetime
29	import glob
30	import operator
31	import os
32	import sys
33
34	try:
35	import build
36	import check
37	import error
38	import log
39	import mailer
40	import options
41	import path
42	import reports
43	import sources
44	import version
45	except KeyboardInterrupt:
46	print('abort: user terminated', file = sys.stderr)
47	sys.exit(1)
48	except:
49	print('error: unknown application load error', file = sys.stderr)
50	sys.exit(1)
51
52	class log_capture(object):
53	def __init__(self):
54	self.log = []
55	log.capture = self.capture
56
57	def __str__(self):
58	return os.linesep.join(self.log)
59
60	def capture(self, text):
61	self.log += [l for l in text.replace(chr(13), '').splitlines()]
62
63	def get(self):
64	return self.log
65
66	def clear(self):
67	self.log = []
68
69	class buildset:
70	"""Build a set builds a set of packages."""
71
72	def __init__(self, bset, _configs, opts, macros = None):
73	log.trace('_bset: %s: init' % (bset))
74	self.configs = _configs
75	self.opts = opts
76	if macros is None:
77	self.macros = copy.copy(opts.defaults)
78	else:
79	self.macros = copy.copy(macros)
80	log.trace('_bset: %s: macro defaults' % (bset))
81	log.trace(str(self.macros))
82	self.bset = bset
83	_target = self.macros.expand('%{_target}')
84	if len(_target):
85	pkg_prefix = _target
86	else:
87	pkg_prefix = self.macros.expand('%{_host}')
88	self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset)
89	self.mail_header = ''
90	self.mail_report = ''
91	self.mail_report_0subject = ''
92	self.build_failure = None
93
94	def write_mail_header(self, text = '', prepend = False):
95	if type(text) is list:
96	text = os.linesep.join(text)
97	text = text.replace('\r', '').replace('\n', os.linesep)
98	if len(text) == 0 or text[-1] != os.linesep:
99	text += os.linesep
100	if prepend:
101	self.mail_header = text + self.mail_header
102	else:
103	self.mail_header += text
104
105	def get_mail_header(self):
106	return self.mail_header
107
108	def write_mail_report(self, text, prepend = False):
109	if type(text) is list:
110	text = os.linesep.join(text)
111	text = text.replace('\r', '').replace('\n', os.linesep)
112	if len(text) == 0 or text[-1] != os.linesep:
113	text += os.linesep
114	if prepend:
115	self.mail_report = text + self.mail_report
116	else:
117	self.mail_report += text
118
119	def get_mail_report(self):
120	return self.mail_report
121
122	def copy(self, src, dst):
123	log.output('copy: %s => %s' % (path.host(src), path.host(dst)))
124	if not self.opts.dry_run():
125	path.copy_tree(src, dst)
126
127	def report(self, _config, _build, opts, macros, format = None, mail = None):
128	if len(_build.main_package().name()) > 0 \
129	and not _build.macros.get('%{_disable_reporting}') \
130	and (not _build.opts.get_arg('--no-report') \
131	or _build.opts.get_arg('--mail')):
132	if format is None:
133	format = _build.opts.get_arg('--report-format')
134	if format is not None:
135	if len(format) != 2:
136	raise error.general('invalid report format option: %s' % \
137	('='.join(format)))
138	format = format[1]
139	if format is None:
140	format = 'text'
141	if format == 'text':
142	ext = '.txt'
143	elif format == 'asciidoc':
144	ext = '.txt'
145	elif format == 'html':
146	ext = '.html'
147	elif format == 'xml':
148	ext = '.xml'
149	elif format == 'ini':
150	ext = '.ini'
151	else:
152	raise error.general('invalid report format: %s' % (format))
153	buildroot = _build.config.abspath('%{buildroot}')
154	prefix = _build.macros.expand('%{_prefix}')
155	name = _build.main_package().name() + ext
156	log.notice('reporting: %s -> %s' % (_config, name))
157	if not _build.opts.get_arg('--no-report'):
158	outpath = path.host(path.join(buildroot, prefix, 'share', 'rtems', 'rsb'))
159	if not _build.opts.dry_run():
160	outname = path.host(path.join(outpath, name))
161	else:
162	outname = None
163	r = reports.report(format, self.configs,
164	copy.copy(opts), copy.copy(macros))
165	r.introduction(_build.config.file_name())
166	r.generate(_build.config.file_name())
167	r.epilogue(_build.config.file_name())
168	if not _build.opts.dry_run():
169	_build.mkdir(outpath)
170	r.write(outname)
171	del r
172	if mail:
173	r = reports.report('text', self.configs,
174	copy.copy(opts), copy.copy(macros))
175	r.introduction(_build.config.file_name())
176	r.generate(_build.config.file_name())
177	r.epilogue(_build.config.file_name())
178	self.write_mail_report(r.get_output())
179	del r
180
181	def root_copy(self, src, dst):
182	what = '%s -> %s' % \
183	(os.path.relpath(path.host(src)), os.path.relpath(path.host(dst)))
184	log.trace('_bset: %s: collecting: %s' % (self.bset, what))
185	self.copy(src, dst)
186
187	def install(self, name, buildroot, prefix):
188	dst = prefix
189	src = path.join(buildroot, prefix)
190	log.notice('installing: %s -> %s' % (name, path.host(dst)))
191	self.copy(src, dst)
192
193	def canadian_cross(self, _build):
194	log.trace('_bset: Cxc for build machine: _build => _host')
195	macros_to_copy = [('%{_host}', '%{_build}'),
196	('%{_host_alias}', '%{_build_alias}'),
197	('%{_host_arch}', '%{_build_arch}'),
198	('%{_host_cpu}', '%{_build_cpu}'),
199	('%{_host_os}', '%{_build_os}'),
200	('%{_host_vendor}', '%{_build_vendor}'),
201	('%{_tmproot}', '%{_tmpcxcroot}'),
202	('%{buildroot}', '%{buildcxcroot}'),
203	('%{_builddir}', '%{_buildcxcdir}')]
204	cxc_macros = _build.copy_init_macros()
205	for m in macros_to_copy:
206	log.trace('_bset: Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
207	cxc_macros[m[0]] = cxc_macros[m[1]]
208	_build.set_macros(cxc_macros)
209	_build.reload()
210	_build.make()
211	if not _build.macros.get('%{_disable_collecting}'):
212	self.root_copy(_build.config.expand('%{buildroot}'),
213	_build.config.expand('%{_tmproot}'))
214	_build.set_macros(_build.copy_init_macros())
215	_build.reload()
216
217	def build_package(self, _config, _build):
218	if not _build.disabled():
219	if _build.canadian_cross():
220	self.canadian_cross(_build)
221	_build.make()
222	if not _build.macros.get('%{_disable_collecting}'):
223	self.root_copy(_build.config.expand('%{buildroot}'),
224	_build.config.expand('%{_tmproot}'))
225
226	def bset_tar(self, _build):
227	tardir = _build.config.expand('%{_tardir}')
228	if (self.opts.get_arg('--bset-tar-file') or self.opts.canadian_cross()) \
229	and not _build.macros.get('%{_disable_packaging}'):
230	path.mkdir(tardir)
231	tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (_build.main_package().name())))
232	log.notice('tarball: %s' % (os.path.relpath(path.host(tar))))
233	if not self.opts.dry_run():
234	tmproot = _build.config.expand('%{_tmproot}')
235	cmd = _build.config.expand('"cd ' + tmproot + \
236	' && %{__tar} -cf - . \| %{__bzip2} > ' + tar + '"')
237	_build.run(cmd, shell_opts = '-c', cwd = tmproot)
238
239	def parse(self, bset):
240
241	def _clean(line):
242	line = line[0:-1]
243	b = line.find('#')
244	if b >= 0:
245	line = line[1:b]
246	return line.strip()
247
248	bsetname = bset
249
250	if not path.exists(bsetname):
251	for cp in self.macros.expand('%{_configdir}').split(':'):
252	configdir = path.abspath(cp)
253	bsetname = path.join(configdir, bset)
254	if path.exists(bsetname):
255	break
256	bsetname = None
257	if bsetname is None:
258	raise error.general('no build set file found: %s' % (bset))
259	try:
260	log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
261	bset = open(path.host(bsetname), 'r')
262	except IOError as err:
263	raise error.general('error opening bset file: %s' % (bsetname))
264
265	configs = []
266
267	try:
268	lc = 0
269	for l in bset:
270	lc += 1
271	l = _clean(l)
272	if len(l) == 0:
273	continue
274	log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l))
275	ls = l.split()
276	if ls[0][-1] == ':' and ls[0][:-1] == 'package':
277	self.bset_pkg = ls[1].strip()
278	self.macros['package'] = self.bset_pkg
279	elif ls[0][0] == '%':
280	def err(msg):
281	raise error.general('%s:%d: %s' % (self.bset, lc, msg))
282	if ls[0] == '%define':
283	if len(ls) > 2:
284	self.macros.define(ls[1].strip(),
285	' '.join([f.strip() for f in ls[2:]]))
286	else:
287	self.macros.define(ls[1].strip())
288	elif ls[0] == '%undefine':
289	if len(ls) > 2:
290	raise error.general('%s:%d: %undefine requires just the name' % \
291	(self.bset, lc))
292	self.macros.undefine(ls[1].strip())
293	elif ls[0] == '%include':
294	configs += self.parse(ls[1].strip())
295	elif ls[0] in ['%patch', '%source']:
296	sources.process(ls[0][1:], ls[1:], self.macros, err)
297	elif ls[0] == '%hash':
298	sources.hash(ls[1:], self.macros, err)
299	else:
300	l = l.strip()
301	c = build.find_config(l, self.configs)
302	if c is None:
303	raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l))
304	configs += [c]
305	except:
306	bset.close()
307	raise
308
309	bset.close()
310
311	return configs
312
313	def load(self):
314
315	#
316	# If the build set file ends with .cfg the user has passed to the
317	# buildset builder a configuration so we just return it.
318	#
319	if self.bset.endswith('.cfg'):
320	configs = [self.bset]
321	else:
322	exbset = self.macros.expand(self.bset)
323	self.macros['_bset'] = exbset
324	root, ext = path.splitext(exbset)
325	if exbset.endswith('.bset'):
326	bset = exbset
327	else:
328	bset = '%s.bset' % (exbset)
329	configs = self.parse(bset)
330	return configs
331
332	def build(self, deps = None, nesting_count = 0, mail = None):
333
334	build_error = False
335
336	nesting_count += 1
337
338	if mail:
339	mail['output'].clear()
340
341	log.trace('_bset: %s: make' % (self.bset))
342	log.notice('Build Set: %s' % (self.bset))
343
344	mail_subject = '%s on %s' % (self.bset, self.macros.expand('%{_host}'))
345
346	current_path = os.environ['PATH']
347
348	start = datetime.datetime.now()
349
350	mail_report = False
351	have_errors = False
352
353	if mail:
354	mail['output'].clear()
355
356	try:
357	configs = self.load()
358
359	log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
360
361	builds = []
362	for s in range(0, len(configs)):
363	b = None
364	try:
365	#
366	# Each section of the build set gets a separate set of
367	# macros so we do not contaminate one configuration with
368	# another.
369	#
370	opts = copy.copy(self.opts)
371	macros = copy.copy(self.macros)
372	if configs[s].endswith('.bset'):
373	log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
374	bs = buildset(configs[s], self.configs, opts, macros)
375	bs.build(deps, nesting_count, mail)
376	del bs
377	elif configs[s].endswith('.cfg'):
378	if mail:
379	mail_report = True
380	log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
381	try:
382	b = build.build(configs[s],
383	self.opts.get_arg('--pkg-tar-files'),
384	opts,
385	macros)
386	except:
387	build_error = True
388	raise
389	if b.macros.get('%{_disable_reporting}'):
390	mail_report = False
391	if deps is None:
392	self.build_package(configs[s], b)
393	self.report(configs[s], b,
394	copy.copy(self.opts),
395	copy.copy(self.macros),
396	mail = mail)
397	# Always produce an XML report.
398	self.report(configs[s], b,
399	copy.copy(self.opts),
400	copy.copy(self.macros),
401	format = 'xml',
402	mail = mail)
403	if s == len(configs) - 1 and not have_errors:
404	self.bset_tar(b)
405	else:
406	deps += b.config.includes()
407	builds += [b]
408	#
409	# Dump post build macros.
410	#
411	log.trace('_bset: macros post-build')
412	log.trace(str(macros))
413	else:
414	raise error.general('invalid config type: %s' % (configs[s]))
415	except error.general as gerr:
416	have_errors = True
417	if b is not None:
418	if self.build_failure is None:
419	self.build_failure = b.name()
420	self.write_mail_header('')
421	self.write_mail_header('= ' * 40)
422	self.write_mail_header('Build FAILED: %s' % (b.name()))
423	self.write_mail_header('- ' * 40)
424	self.write_mail_header(str(log.default))
425	self.write_mail_header('- ' * 40)
426	if self.opts.keep_going():
427	log.notice(str(gerr))
428	if self.opts.always_clean():
429	builds += [b]
430	else:
431	raise
432	else:
433	raise
434	#
435	# Installing ...
436	#
437	log.trace('_bset: installing: deps:%r no-install:%r' % \
438	(deps is None, self.opts.no_install()))
439	if deps is None \
440	and not self.opts.no_install() \
441	and not have_errors:
442	for b in builds:
443	log.trace('_bset: installing: %r' % b.installable())
444	if b.installable():
445	self.install(b.name(),
446	b.config.expand('%{buildroot}'),
447	b.config.expand('%{_prefix}'))
448
449	if deps is None and \
450	(not self.opts.no_clean() or self.opts.always_clean()):
451	for b in builds:
452	if not b.disabled():
453	log.notice('cleaning: %s' % (b.name()))
454	b.cleanup()
455	for b in builds:
456	del b
457	except error.general as gerr:
458	if not build_error:
459	log.stderr(str(gerr))
460	raise
461	except KeyboardInterrupt:
462	mail_report = False
463	raise
464	except:
465	self.build_failure = 'RSB general failure'
466	raise
467	finally:
468	end = datetime.datetime.now()
469	os.environ['PATH'] = current_path
470	build_time = str(end - start)
471	if mail_report and not self.macros.defined('mail_disable'):
472	self.write_mail_header('Build Time: %s' % (build_time), True)
473	self.write_mail_header('', True)
474	if self.build_failure is not None:
475	mail_subject = 'Build : FAILED %s (%s)' % \
476	(mail_subject, self.build_failure)
477	else:
478	mail_subject = 'Build : PASSED %s' % (mail_subject)
479	self.write_mail_header(mail['header'], True)
480	self.write_mail_header('')
481	log.notice('Mailing report: %s' % (mail['to']))
482	body = self.get_mail_header()
483	body += 'Output' + os.linesep
484	body += '======' + os.linesep + os.linesep
485	body += os.linesep.join(mail['output'].get())
486	body += os.linesep + os.linesep
487	body += 'Report' + os.linesep
488	body += '======' + os.linesep + os.linesep
489	body += self.get_mail_report()
490	if not opts.dry_run():
491	mail['mail'].send(mail['to'], mail_subject, body)
492	log.notice('Build Set: Time %s' % (build_time))
493
494	def list_bset_cfg_files(opts, configs):
495	if opts.get_arg('--list-configs') or opts.get_arg('--list-bsets'):
496	if opts.get_arg('--list-configs'):
497	ext = '.cfg'
498	else:
499	ext = '.bset'
500	for p in configs['paths']:
501	print('Examining: %s' % (os.path.relpath(p)))
502	for c in configs['files']:
503	if c.endswith(ext):
504	print(' %s' % (c))
505	return True
506	return False
507
508	def run():
509	import sys
510	ec = 0
511	setbuilder_error = False
512	mail = None
513	try:
514	optargs = { '--list-configs': 'List available configurations',
515	'--list-bsets': 'List available build sets',
516	'--list-deps': 'List the dependent files.',
517	'--bset-tar-file': 'Create a build set tar file',
518	'--pkg-tar-files': 'Create package tar files',
519	'--no-report': 'Do not create a package report.',
520	'--report-format': 'The report format (text, html, asciidoc).' }
521	mailer.append_options(optargs)
522	opts = options.load(sys.argv, optargs)
523	if opts.get_arg('--mail'):
524	mail = { 'mail' : mailer.mail(opts),
525	'output': log_capture() }
526	to_addr = opts.get_arg('--mail-to')
527	if to_addr is not None:
528	mail['to'] = to_addr[1]
529	else:
530	mail['to'] = pts.defaults.expand('%{_mail_tools_to}')
531	mail['from'] = mail['mail'].from_address()
532	log.notice('RTEMS Source Builder - Set Builder, %s' % (version.str()))
533	opts.log_info()
534	if not check.host_setup(opts):
535	raise error.general('host build environment is not set up correctly')
536	if mail:
537	mail['header'] = os.linesep.join(mail['output'].get())
538	mail['header'] += os.linesep
539	mail['header'] += opts.info() + os.linesep
540	configs = build.get_configs(opts)
541	if opts.get_arg('--list-deps'):
542	deps = []
543	else:
544	deps = None
545	if not list_bset_cfg_files(opts, configs):
546	prefix = opts.defaults.expand('%{_prefix}')
547	if opts.canadian_cross():
548	opts.disable_install()
549
550	if not opts.dry_run() and \
551	not opts.canadian_cross() and \
552	not opts.no_install() and \
553	not path.ispathwritable(prefix):
554	raise error.general('prefix is not writable: %s' % (path.host(prefix)))
555
556	for bset in opts.params():
557	setbuilder_error = True
558	b = buildset(bset, configs, opts)
559	b.build(deps, mail = mail)
560	b = None
561	setbuilder_error = False
562
563	if deps is not None:
564	c = 0
565	for d in sorted(set(deps)):
566	c += 1
567	print('dep[%d]: %s' % (c, d))
568	except error.general as gerr:
569	if not setbuilder_error:
570	log.stderr(str(gerr))
571	log.stderr('Build FAILED')
572	ec = 1
573	except error.internal as ierr:
574	if not setbuilder_error:
575	log.stderr(str(ierr))
576	log.stderr('Internal Build FAILED')
577	ec = 1
578	except error.exit as eerr:
579	pass
580	except KeyboardInterrupt:
581	log.notice('abort: user terminated')
582	ec = 1
583	except:
584	raise
585	log.notice('abort: unknown error')
586	ec = 1
587	sys.exit(ec)
588
589	if __name__ == "__main__":
590	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: