Context Navigation

source: rtems-source-builder/source-builder/sb/setbuilder.py @ 4bd058e

4.104.114.95

Last change on this file since 4bd058e was 4bd058e, checked in by Chris Johns <chrisj@…>, on 05/22/15 at 02:20:41
sb: Always generate an XML report.
Property mode set to `100644`
File size: 20.0 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Permission to use, copy, modify, and/or distribute this software for any
9	# purpose with or without fee is hereby granted, provided that the above
10	# copyright notice and this permission notice appear in all copies.
11	#
12	# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13	# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14	# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15	# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16	# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17	# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18	# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20	#
21	# This code builds a package compiler tool suite given a tool set. A tool
22	# set lists the various tools. These are specific tool configurations.
23	#
24
25	import copy
26	import datetime
27	import glob
28	import operator
29	import os
30	import sys
31
32	try:
33	import build
34	import check
35	import error
36	import log
37	import mailer
38	import options
39	import path
40	import reports
41	import sources
42	import version
43	except KeyboardInterrupt:
44	print 'abort: user terminated'
45	sys.exit(1)
46	except:
47	print 'error: unknown application load error'
48	sys.exit(1)
49
50	class buildset:
51	"""Build a set builds a set of packages."""
52
53	def __init__(self, bset, _configs, opts, macros = None):
54	log.trace('_bset: %s: init' % (bset))
55	self.configs = _configs
56	self.opts = opts
57	if macros is None:
58	self.macros = copy.copy(opts.defaults)
59	else:
60	self.macros = copy.copy(macros)
61	self.bset = bset
62	_target = self.macros.expand('%{_target}')
63	if len(_target):
64	pkg_prefix = _target
65	else:
66	pkg_prefix = self.macros.expand('%{_host}')
67	self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset)
68	self.mail_header = ''
69	self.mail_report = ''
70	self.build_failure = None
71
72	def write_mail_header(self, text, prepend = False):
73	if len(text) == 0 or text[-1] != '\n' or text[-1] != '\r':
74	text += os.linesep
75	if prepend:
76	self.mail_header = text + self.mail_header
77	else:
78	self.mail_header += text
79
80	def write_mail_report(self, text, prepend = False):
81	if len(text) == 0 or text[-1] != '\n' or text[-1] != '\r':
82	text += os.linesep
83	if prepend:
84	self.mail_report = text + self.mail_report
85	else:
86	self.mail_report += text
87
88	def copy(self, src, dst):
89	log.output('copy: %s => %s' % (path.host(src), path.host(dst)))
90	if not self.opts.dry_run():
91	path.copy_tree(src, dst)
92
93	def report(self, _config, _build, opts, macros, format = None):
94	if len(_build.main_package().name()) > 0 \
95	and not _build.macros.get('%{_disable_reporting}') \
96	and (not _build.opts.get_arg('--no-report') \
97	or _build.opts.get_arg('--mail')):
98	if format is None:
99	format = _build.opts.get_arg('--report-format')
100	if format is not None:
101	if len(format) != 2:
102	raise error.general('invalid report format option: %s' % ('='.join(format)))
103	format = format[1]
104	if format is None:
105	format = 'text'
106	if format == 'text':
107	ext = '.txt'
108	elif format == 'asciidoc':
109	ext = '.txt'
110	elif format == 'html':
111	ext = '.html'
112	elif format == 'xml':
113	ext = '.xml'
114	elif format == 'ini':
115	ext = '.ini'
116	else:
117	raise error.general('invalid report format: %s' % (format))
118	buildroot = _build.config.abspath('%{buildroot}')
119	prefix = _build.macros.expand('%{_prefix}')
120	name = _build.main_package().name() + ext
121	log.notice('reporting: %s -> %s' % (_config, name))
122	if not _build.opts.get_arg('--no-report'):
123	outpath = path.host(path.join(buildroot, prefix, 'share', 'rtems', 'rsb'))
124	if not _build.opts.dry_run():
125	outname = path.host(path.join(outpath, name))
126	else:
127	outname = None
128	r = reports.report(format, self.configs,
129	copy.copy(opts), copy.copy(macros))
130	r.introduction(_build.config.file_name())
131	r.generate(_build.config.file_name())
132	r.epilogue(_build.config.file_name())
133	if not _build.opts.dry_run():
134	_build.mkdir(outpath)
135	r.write(outname)
136	del r
137	if _build.opts.get_arg('--mail'):
138	r = reports.report('text', self.configs,
139	copy.copy(opts), copy.copy(macros))
140	r.introduction(_build.config.file_name())
141	r.generate(_build.config.file_name())
142	r.epilogue(_build.config.file_name())
143	self.write_mail_report(r.out)
144	del r
145
146	def root_copy(self, src, dst):
147	what = '%s -> %s' % \
148	(os.path.relpath(path.host(src)), os.path.relpath(path.host(dst)))
149	log.trace('_bset: %s: collecting: %s' % (self.bset, what))
150	self.copy(src, dst)
151
152	def install(self, name, buildroot, prefix):
153	dst = prefix
154	src = path.join(buildroot, prefix)
155	log.notice('installing: %s -> %s' % (name, path.host(dst)))
156	self.copy(src, dst)
157
158	def canadian_cross(self, _build):
159	# @fixme Switch to using a private macros map.
160	macros_to_save = ['%{_prefix}',
161	'%{_tmproot}',
162	'%{buildroot}',
163	'%{_builddir}',
164	'%{_host}']
165	macros_to_copy = [('%{_host}', '%{_build}'),
166	('%{_tmproot}', '%{_tmpcxcroot}'),
167	('%{buildroot}', '%{buildcxcroot}'),
168	('%{_builddir}', '%{_buildcxcdir}')]
169	orig_macros = {}
170	for m in macros_to_save:
171	orig_macros[m] = _build.config.macro(m)
172	for m in macros_to_copy:
173	_build.config.set_define(m[0], _build.config.macro(m[1]))
174	_build.make()
175	for m in macros_to_save:
176	_build.config.set_define(m, orig_macros[m])
177	if not _build.macros.get('%{_disable_collecting}'):
178	self.root_copy(_build.config.expand('%{buildcxcroot}'),
179	_build.config.expand('%{_tmpcxcroot}'))
180
181	def build_package(self, _config, _build):
182	if not _build.disabled():
183	if _build.canadian_cross():
184	self.canadian_cross(_build)
185	_build.make()
186	if not _build.macros.get('%{_disable_collecting}'):
187	self.root_copy(_build.config.expand('%{buildroot}'),
188	_build.config.expand('%{_tmproot}'))
189
190	def bset_tar(self, _build):
191	tardir = _build.config.expand('%{_tardir}')
192	if self.opts.get_arg('--bset-tar-file') \
193	and not _build.macros.get('%{_disable_packaging}'):
194	path.mkdir(tardir)
195	tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (self.bset_pkg)))
196	log.notice('tarball: %s' % (os.path.relpath(path.host(tar))))
197	if not self.opts.dry_run():
198	tmproot = _build.config.expand('%{_tmproot}')
199	cmd = _build.config.expand("'cd " + tmproot + \
200	" && %{__tar} -cf - . \| %{__bzip2} > " + tar + "'")
201	_build.run(cmd, shell_opts = '-c', cwd = tmproot)
202
203	def parse(self, bset):
204
205	def _clean(line):
206	line = line[0:-1]
207	b = line.find('#')
208	if b >= 0:
209	line = line[1:b]
210	return line.strip()
211
212	bsetname = bset
213
214	if not path.exists(bsetname):
215	for cp in self.macros.expand('%{_configdir}').split(':'):
216	configdir = path.abspath(cp)
217	bsetname = path.join(configdir, bset)
218	if path.exists(bsetname):
219	break
220	bsetname = None
221	if bsetname is None:
222	raise error.general('no build set file found: %s' % (bset))
223	try:
224	log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
225	bset = open(path.host(bsetname), 'r')
226	except IOError, err:
227	raise error.general('error opening bset file: %s' % (bsetname))
228
229	configs = []
230
231	try:
232	lc = 0
233	for l in bset:
234	lc += 1
235	l = _clean(l)
236	if len(l) == 0:
237	continue
238	log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l))
239	ls = l.split()
240	if ls[0][-1] == ':' and ls[0][:-1] == 'package':
241	self.bset_pkg = self.macros.expand(ls[1].strip())
242	self.macros['package'] = self.bset_pkg
243	elif ls[0][0] == '%':
244	def err(msg):
245	raise error.general('%s:%d: %s' % (self.bset, lc, msg))
246	if ls[0] == '%define':
247	if len(ls) > 2:
248	self.macros.define(ls[1].strip(),
249	' '.join([f.strip() for f in ls[2:]]))
250	else:
251	self.macros.define(ls[1].strip())
252	elif ls[0] == '%undefine':
253	if len(ls) > 2:
254	raise error.general('%s:%d: %undefine requires just the name' % \
255	(self.bset, lc))
256	self.macros.undefine(ls[1].strip())
257	elif ls[0] == '%include':
258	configs += self.parse(ls[1].strip())
259	elif ls[0] in ['%patch', '%source']:
260	sources.process(ls[0][1:], ls[1:], self.macros, err)
261	elif ls[0] == '%hash':
262	sources.hash(ls[1:], self.macros, err)
263	else:
264	l = l.strip()
265	c = build.find_config(l, self.configs)
266	if c is None:
267	raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l))
268	configs += [c]
269	except:
270	bset.close()
271	raise
272
273	bset.close()
274
275	return configs
276
277	def load(self):
278
279	#
280	# If the build set file ends with .cfg the user has passed to the
281	# buildset builder a configuration so we just return it.
282	#
283	if self.bset.endswith('.cfg'):
284	configs = [self.bset]
285	else:
286	exbset = self.macros.expand(self.bset)
287	self.macros['_bset'] = exbset
288	root, ext = path.splitext(exbset)
289	if exbset.endswith('.bset'):
290	bset = exbset
291	else:
292	bset = '%s.bset' % (exbset)
293	configs = self.parse(bset)
294	return configs
295
296	def build(self, deps = None, nesting_count = 0):
297
298	build_error = False
299
300	nesting_count += 1
301
302	log.trace('_bset: %s: make' % (self.bset))
303	log.notice('Build Set: %s' % (self.bset))
304
305	if self.opts.get_arg('--mail'):
306	mail_report_subject = '%s %s' % (self.bset, self.macros.expand('%{_host}'))
307
308	current_path = os.environ['PATH']
309
310	start = datetime.datetime.now()
311
312	mail_report = False
313	have_errors = False
314
315	try:
316	configs = self.load()
317
318	log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
319
320	builds = []
321	for s in range(0, len(configs)):
322	b = None
323	try:
324	#
325	# Each section of the build set gets a separate set of
326	# macros so we do not contaminate one configuration with
327	# another.
328	#
329	opts = copy.copy(self.opts)
330	macros = copy.copy(self.macros)
331	if configs[s].endswith('.bset'):
332	log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
333	bs = buildset(configs[s], self.configs, opts, macros)
334	bs.build(deps, nesting_count)
335	del bs
336	elif configs[s].endswith('.cfg'):
337	mail_report = self.opts.get_arg('--mail')
338	log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
339	try:
340	b = build.build(configs[s], self.opts.get_arg('--pkg-tar-files'),
341	opts, macros)
342	except:
343	build_error = True
344	raise
345	if b.macros.get('%{_disable_reporting}'):
346	mail_report = False
347	if deps is None:
348	self.build_package(configs[s], b)
349	self.report(configs[s], b,
350	copy.copy(self.opts),
351	copy.copy(self.macros))
352	# Always product an XML report.
353	self.report(configs[s], b,
354	copy.copy(self.opts),
355	copy.copy(self.macros),
356	format = 'xml')
357	if s == len(configs) - 1 and not have_errors:
358	self.bset_tar(b)
359	else:
360	deps += b.config.includes()
361	builds += [b]
362	else:
363	raise error.general('invalid config type: %s' % (configs[s]))
364	except error.general, gerr:
365	have_errors = True
366	if b is not None:
367	if self.build_failure is None:
368	self.build_failure = b.name()
369	self.write_mail_header('')
370	self.write_mail_header('= ' * 40)
371	self.write_mail_header('Build FAILED: %s' % (b.name()))
372	self.write_mail_header('- ' * 40)
373	self.write_mail_header(str(log.default))
374	self.write_mail_header('- ' * 40)
375	if self.opts.keep_going():
376	log.notice(str(gerr))
377	if self.opts.always_clean():
378	builds += [b]
379	else:
380	raise
381	else:
382	raise
383	if deps is None \
384	and not self.opts.no_install() \
385	and not have_errors:
386	for b in builds:
387	if not b.canadian_cross() \
388	and not b.disabled() \
389	and not b.macros.get('%{_disable_installing}'):
390	self.install(b.name(),
391	b.config.expand('%{buildroot}'),
392	b.config.expand('%{_prefix}'))
393	if deps is None and \
394	(not self.opts.no_clean() or self.opts.always_clean()):
395	for b in builds:
396	if not b.disabled():
397	log.notice('cleaning: %s' % (b.name()))
398	b.cleanup()
399	for b in builds:
400	del b
401	except error.general, gerr:
402	if not build_error:
403	log.stderr(str(gerr))
404	raise
405	except KeyboardInterrupt:
406	mail_report = False
407	raise
408	except:
409	self.build_failure = 'RSB general failure'
410	raise
411	finally:
412	end = datetime.datetime.now()
413	os.environ['PATH'] = current_path
414	build_time = str(end - start)
415	if mail_report:
416	to_addr = self.opts.get_arg('--mail-to')
417	if to_addr is not None:
418	to_addr = to_addr[1]
419	else:
420	to_addr = self.macros.expand('%{_mail_tools_to}')
421	log.notice('Mailing report: %s' % (to_addr))
422	self.write_mail_header('Build Time %s' % (build_time), True)
423	self.write_mail_header('')
424	m = mailer.mail(self.opts)
425	if self.build_failure is not None:
426	mail_report_subject = 'Build: FAILED %s (%s)' %\
427	(mail_report_subject, self.build_failure)
428	pass_fail = 'FAILED'
429	else:
430	mail_report_subject = 'Build: PASSED %s' % (mail_report_subject)
431	if not self.opts.dry_run():
432	m.send(to_addr, mail_report_subject,
433	self.mail_header + self.mail_report)
434	log.notice('Build Set: Time %s' % (build_time))
435
436	def list_bset_cfg_files(opts, configs):
437	if opts.get_arg('--list-configs') or opts.get_arg('--list-bsets'):
438	if opts.get_arg('--list-configs'):
439	ext = '.cfg'
440	else:
441	ext = '.bset'
442	for p in configs['paths']:
443	print 'Examining: %s' % (os.path.relpath(p))
444	for c in configs['files']:
445	if c.endswith(ext):
446	print ' %s' % (c)
447	return True
448	return False
449
450	def run():
451	import sys
452	ec = 0
453	setbuilder_error = False
454	try:
455	optargs = { '--list-configs': 'List available configurations',
456	'--list-bsets': 'List available build sets',
457	'--list-deps': 'List the dependent files.',
458	'--bset-tar-file': 'Create a build set tar file',
459	'--pkg-tar-files': 'Create package tar files',
460	'--no-report': 'Do not create a package report.',
461	'--report-format': 'The report format (text, html, asciidoc).' }
462	mailer.append_options(optargs)
463	opts = options.load(sys.argv, optargs)
464	log.notice('RTEMS Source Builder - Set Builder, v%s' % (version.str()))
465	opts.log_info()
466	if not check.host_setup(opts):
467	raise error.general('host build environment is not set up correctly')
468	configs = build.get_configs(opts)
469	if opts.get_arg('--list-deps'):
470	deps = []
471	else:
472	deps = None
473	if not list_bset_cfg_files(opts, configs):
474	prefix = opts.defaults.expand('%{_prefix}')
475	if not opts.dry_run() and \
476	not opts.canadian_cross() and \
477	not opts.no_install() and \
478	not path.ispathwritable(prefix):
479	raise error.general('prefix is not writable: %s' % (path.host(prefix)))
480	for bset in opts.params():
481	setbuilder_error = True
482	b = buildset(bset, configs, opts)
483	b.build(deps)
484	b = None
485	setbuilder_error = False
486	if deps is not None:
487	c = 0
488	for d in sorted(set(deps)):
489	c += 1
490	print 'dep[%d]: %s' % (c, d)
491	except error.general, gerr:
492	if not setbuilder_error:
493	log.stderr(str(gerr))
494	log.stderr('Build FAILED')
495	ec = 1
496	except error.internal, ierr:
497	if not setbuilder_error:
498	log.stderr(str(ierr))
499	log.stderr('Internal Build FAILED')
500	ec = 1
501	except error.exit, eerr:
502	pass
503	except KeyboardInterrupt:
504	log.notice('abort: user terminated')
505	ec = 1
506	sys.exit(ec)
507
508	if __name__ == "__main__":
509	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: