Context Navigation

source: rtems-source-builder/source-builder/sb/setbuilder.py @ 96c414c

5

Last change on this file since 96c414c was 96c414c, checked in by Chris Johns <chrisj@…>, on 10/22/18 at 00:59:10

windows: Remove BuildRoot? from all configs, add a short tmp path.

Closes #3562.

Property mode set to 100644

File size: 25.8 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Permission to use, copy, modify, and/or distribute this software for any
9	# purpose with or without fee is hereby granted, provided that the above
10	# copyright notice and this permission notice appear in all copies.
11	#
12	# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13	# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14	# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15	# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16	# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17	# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18	# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20	#
21	# This code builds a package compiler tool suite given a tool set. A tool
22	# set lists the various tools. These are specific tool configurations.
23	#
24
25	from __future__ import print_function
26
27	import copy
28	import datetime
29	import glob
30	import operator
31	import os
32	import sys
33	import textwrap
34
35	try:
36	import build
37	import check
38	import error
39	import log
40	import mailer
41	import options
42	import path
43	import reports
44	import sources
45	import version
46	except KeyboardInterrupt:
47	print('abort: user terminated', file = sys.stderr)
48	sys.exit(1)
49	except:
50	print('error: unknown application load error', file = sys.stderr)
51	sys.exit(1)
52
53	class log_capture(object):
54	def __init__(self):
55	self.log = []
56	log.capture = self.capture
57
58	def __str__(self):
59	return os.linesep.join(self.log)
60
61	def capture(self, text):
62	self.log += [l for l in text.replace(chr(13), '').splitlines()]
63
64	def get(self):
65	return self.log
66
67	def clear(self):
68	self.log = []
69
70	class buildset:
71	"""Build a set builds a set of packages."""
72
73	def __init__(self, bset, _configs, opts, macros = None):
74	log.trace('_bset: %s: init' % (bset))
75	self.configs = _configs
76	self.opts = opts
77	if macros is None:
78	self.macros = copy.copy(opts.defaults)
79	else:
80	self.macros = copy.copy(macros)
81	log.trace('_bset: %s: macro defaults' % (bset))
82	log.trace(str(self.macros))
83	self.bset = bset
84	_target = self.macros.expand('%{_target}')
85	if len(_target):
86	pkg_prefix = _target
87	else:
88	pkg_prefix = self.macros.expand('%{_host}')
89	self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset)
90	self.mail_header = ''
91	self.mail_report = ''
92	self.mail_report_0subject = ''
93	self.build_failure = None
94
95	def write_mail_header(self, text = '', prepend = False):
96	if type(text) is list:
97	text = os.linesep.join(text)
98	text = text.replace('\r', '').replace('\n', os.linesep)
99	if len(text) == 0 or text[-1] != os.linesep:
100	text += os.linesep
101	if prepend:
102	self.mail_header = text + self.mail_header
103	else:
104	self.mail_header += text
105
106	def get_mail_header(self):
107	return self.mail_header
108
109	def write_mail_report(self, text, prepend = False):
110	if type(text) is list:
111	text = os.linesep.join(text)
112	text = text.replace('\r', '').replace('\n', os.linesep)
113	if len(text) == 0 or text[-1] != os.linesep:
114	text += os.linesep
115	if prepend:
116	self.mail_report = text + self.mail_report
117	else:
118	self.mail_report += text
119
120	def get_mail_report(self):
121	return self.mail_report
122
123	def copy(self, src, dst):
124	log.output('copy: %s => %s' % (path.host(src), path.host(dst)))
125	if not self.opts.dry_run():
126	path.copy_tree(src, dst)
127
128	def report(self, _config, _build, opts, macros, format = None, mail = None):
129	if len(_build.main_package().name()) > 0 \
130	and not _build.macros.get('%{_disable_reporting}') \
131	and (not _build.opts.get_arg('--no-report') \
132	or _build.opts.get_arg('--mail')):
133	if format is None:
134	format = _build.opts.get_arg('--report-format')
135	if format is not None:
136	if len(format) != 2:
137	raise error.general('invalid report format option: %s' % \
138	('='.join(format)))
139	format = format[1]
140	if format is None:
141	format = 'text'
142	if format == 'text':
143	ext = '.txt'
144	elif format == 'asciidoc':
145	ext = '.txt'
146	elif format == 'html':
147	ext = '.html'
148	elif format == 'xml':
149	ext = '.xml'
150	elif format == 'ini':
151	ext = '.ini'
152	else:
153	raise error.general('invalid report format: %s' % (format))
154	buildroot = _build.config.abspath('%{buildroot}')
155	prefix = _build.macros.expand('%{_prefix}')
156	name = _build.main_package().name() + ext
157	log.notice('reporting: %s -> %s' % (_config, name))
158	if not _build.opts.get_arg('--no-report'):
159	outpath = path.host(path.join(buildroot, prefix, 'share', 'rtems', 'rsb'))
160	if not _build.opts.dry_run():
161	outname = path.host(path.join(outpath, name))
162	else:
163	outname = None
164	r = reports.report(format, self.configs,
165	copy.copy(opts), copy.copy(macros))
166	r.introduction(_build.config.file_name())
167	r.generate(_build.config.file_name())
168	r.epilogue(_build.config.file_name())
169	if not _build.opts.dry_run():
170	_build.mkdir(outpath)
171	r.write(outname)
172	del r
173	if mail:
174	r = reports.report('text', self.configs,
175	copy.copy(opts), copy.copy(macros))
176	r.introduction(_build.config.file_name())
177	r.generate(_build.config.file_name())
178	r.epilogue(_build.config.file_name())
179	self.write_mail_report(r.get_output())
180	del r
181
182	def root_copy(self, src, dst):
183	what = '%s -> %s' % \
184	(os.path.relpath(path.host(src)), os.path.relpath(path.host(dst)))
185	log.trace('_bset: %s: collecting: %s' % (self.bset, what))
186	self.copy(src, dst)
187
188	def install(self, name, buildroot, prefix):
189	dst = prefix
190	src = path.join(buildroot, prefix)
191	log.notice('installing: %s -> %s' % (name, path.host(dst)))
192	self.copy(src, dst)
193
194	def canadian_cross(self, _build):
195	log.trace('_bset: Cxc for build machine: _build => _host')
196	macros_to_copy = [('%{_host}', '%{_build}'),
197	('%{_host_alias}', '%{_build_alias}'),
198	('%{_host_arch}', '%{_build_arch}'),
199	('%{_host_cpu}', '%{_build_cpu}'),
200	('%{_host_os}', '%{_build_os}'),
201	('%{_host_vendor}', '%{_build_vendor}'),
202	('%{_tmproot}', '%{_tmpcxcroot}'),
203	('%{buildroot}', '%{buildcxcroot}'),
204	('%{_builddir}', '%{_buildcxcdir}')]
205	cxc_macros = _build.copy_init_macros()
206	for m in macros_to_copy:
207	log.trace('_bset: Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
208	cxc_macros[m[0]] = cxc_macros[m[1]]
209	_build.set_macros(cxc_macros)
210	_build.reload()
211	_build.make()
212	if not _build.macros.get('%{_disable_collecting}'):
213	self.root_copy(_build.config.expand('%{buildroot}'),
214	_build.config.expand('%{_tmproot}'))
215	_build.set_macros(_build.copy_init_macros())
216	_build.reload()
217
218	def build_package(self, _config, _build):
219	if not _build.disabled():
220	if _build.canadian_cross():
221	self.canadian_cross(_build)
222	_build.make()
223	if not _build.macros.get('%{_disable_collecting}'):
224	self.root_copy(_build.config.expand('%{buildroot}'),
225	_build.config.expand('%{_tmproot}'))
226
227	def bset_tar(self, _build):
228	tardir = _build.config.expand('%{_tardir}')
229	if (self.opts.get_arg('--bset-tar-file') or self.opts.canadian_cross()) \
230	and not _build.macros.get('%{_disable_packaging}'):
231	path.mkdir(tardir)
232	tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (_build.main_package().name())))
233	log.notice('tarball: %s' % (os.path.relpath(path.host(tar))))
234	if not self.opts.dry_run():
235	tmproot = _build.config.expand('%{_tmproot}')
236	cmd = _build.config.expand('"cd ' + tmproot + \
237	' && %{__tar} -cf - . \| %{__bzip2} > ' + tar + '"')
238	_build.run(cmd, shell_opts = '-c', cwd = tmproot)
239
240	def parse(self, bset):
241
242	def _clean(line):
243	line = line[0:-1]
244	b = line.find('#')
245	if b >= 0:
246	line = line[1:b]
247	return line.strip()
248
249	bsetname = bset
250
251	if not path.exists(bsetname):
252	for cp in self.macros.expand('%{_configdir}').split(':'):
253	configdir = path.abspath(cp)
254	bsetname = path.join(configdir, bset)
255	if path.exists(bsetname):
256	break
257	bsetname = None
258	if bsetname is None:
259	raise error.general('no build set file found: %s' % (bset))
260	try:
261	log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
262	bset = open(path.host(bsetname), 'r')
263	except IOError as err:
264	raise error.general('error opening bset file: %s' % (bsetname))
265
266	configs = []
267
268	try:
269	lc = 0
270	for l in bset:
271	lc += 1
272	l = _clean(l)
273	if len(l) == 0:
274	continue
275	log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l))
276	ls = l.split()
277	if ls[0][-1] == ':' and ls[0][:-1] == 'package':
278	self.bset_pkg = ls[1].strip()
279	self.macros['package'] = self.bset_pkg
280	elif ls[0][0] == '%':
281	def err(msg):
282	raise error.general('%s:%d: %s' % (self.bset, lc, msg))
283	if ls[0] == '%define':
284	if len(ls) > 2:
285	self.macros.define(ls[1].strip(),
286	' '.join([f.strip() for f in ls[2:]]))
287	else:
288	self.macros.define(ls[1].strip())
289	elif ls[0] == '%undefine':
290	if len(ls) > 2:
291	raise error.general('%s:%d: %undefine requires just the name' % \
292	(self.bset, lc))
293	self.macros.undefine(ls[1].strip())
294	elif ls[0] == '%include':
295	configs += self.parse(ls[1].strip())
296	elif ls[0] in ['%patch', '%source']:
297	sources.process(ls[0][1:], ls[1:], self.macros, err)
298	elif ls[0] == '%hash':
299	sources.hash(ls[1:], self.macros, err)
300	else:
301	l = l.strip()
302	c = build.find_config(l, self.configs)
303	if c is None:
304	raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l))
305	configs += [c]
306	except:
307	bset.close()
308	raise
309
310	bset.close()
311
312	return configs
313
314	def load(self):
315
316	#
317	# If the build set file ends with .cfg the user has passed to the
318	# buildset builder a configuration so we just return it.
319	#
320	if self.bset.endswith('.cfg'):
321	configs = [self.bset]
322	else:
323	exbset = self.macros.expand(self.bset)
324	self.macros['_bset'] = exbset
325	self.macros['_bset_tmp'] = build.short_name(exbset)
326	root, ext = path.splitext(exbset)
327	if exbset.endswith('.bset'):
328	bset = exbset
329	else:
330	bset = '%s.bset' % (exbset)
331	configs = self.parse(bset)
332	return configs
333
334	def build(self, deps = None, nesting_count = 0, mail = None):
335
336	build_error = False
337
338	nesting_count += 1
339
340	if mail:
341	mail['output'].clear()
342
343	log.trace('_bset: %s: make' % (self.bset))
344	log.notice('Build Set: %s' % (self.bset))
345
346	mail_subject = '%s on %s' % (self.bset,
347	self.macros.expand('%{_host}'))
348
349	current_path = os.environ['PATH']
350
351	start = datetime.datetime.now()
352
353	mail_report = False
354	have_errors = False
355
356	if mail:
357	mail['output'].clear()
358
359	try:
360	configs = self.load()
361
362	log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
363
364	builds = []
365	for s in range(0, len(configs)):
366	b = None
367	try:
368	#
369	# Each section of the build set gets a separate set of
370	# macros so we do not contaminate one configuration with
371	# another.
372	#
373	opts = copy.copy(self.opts)
374	macros = copy.copy(self.macros)
375	if configs[s].endswith('.bset'):
376	log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
377	bs = buildset(configs[s], self.configs, opts, macros)
378	bs.build(deps, nesting_count, mail)
379	del bs
380	elif configs[s].endswith('.cfg'):
381	if mail:
382	mail_report = True
383	log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
384	try:
385	b = build.build(configs[s],
386	self.opts.get_arg('--pkg-tar-files'),
387	opts,
388	macros)
389	except:
390	build_error = True
391	raise
392	if b.macros.get('%{_disable_reporting}'):
393	mail_report = False
394	if deps is None:
395	self.build_package(configs[s], b)
396	self.report(configs[s], b,
397	copy.copy(self.opts),
398	copy.copy(self.macros),
399	mail = mail)
400	# Always produce an XML report.
401	self.report(configs[s], b,
402	copy.copy(self.opts),
403	copy.copy(self.macros),
404	format = 'xml',
405	mail = mail)
406	if s == len(configs) - 1 and not have_errors:
407	self.bset_tar(b)
408	else:
409	deps += b.config.includes()
410	builds += [b]
411	#
412	# Dump post build macros.
413	#
414	log.trace('_bset: macros post-build')
415	log.trace(str(macros))
416	else:
417	raise error.general('invalid config type: %s' % (configs[s]))
418	except error.general as gerr:
419	have_errors = True
420	if b is not None:
421	if self.build_failure is None:
422	self.build_failure = b.name()
423	self.write_mail_header('')
424	self.write_mail_header('= ' * 40)
425	self.write_mail_header('Build FAILED: %s' % (b.name()))
426	self.write_mail_header('- ' * 40)
427	self.write_mail_header(str(log.default))
428	self.write_mail_header('- ' * 40)
429	if self.opts.keep_going():
430	log.notice(str(gerr))
431	if self.opts.always_clean():
432	builds += [b]
433	else:
434	raise
435	else:
436	raise
437	#
438	# Installing ...
439	#
440	log.trace('_bset: installing: deps:%r no-install:%r' % \
441	(deps is None, self.opts.no_install()))
442	if deps is None \
443	and not self.opts.no_install() \
444	and not have_errors:
445	for b in builds:
446	log.trace('_bset: installing: %r' % b.installable())
447	if b.installable():
448	self.install(b.name(),
449	b.config.expand('%{buildroot}'),
450	b.config.expand('%{_prefix}'))
451	#
452	# Sizes ...
453	#
454	if len(builds) > 1:
455	size_build = 0
456	size_installed = 0
457	size_build_max = 0
458	for b in builds:
459	s = b.get_build_size()
460	size_build += s
461	if s > size_build_max:
462	size_build_max = s
463	size_installed += b.get_installed_size()
464	size_sources = 0
465	for p in builds[0].config.expand('%{_sourcedir}').split(':'):
466	size_sources += path.get_size(p)
467	size_patches = 0
468	for p in builds[0].config.expand('%{_patchdir}').split(':'):
469	size_patches += path.get_size(p)
470	size_total = size_sources + size_patches + size_installed
471	build_max_size_human = build.humanize_number(size_build_max + size_installed, 'B')
472	build_total_size_human = build.humanize_number(size_total, 'B')
473	build_sources_size_human = build.humanize_number(size_sources, 'B')
474	build_patches_size_human = build.humanize_number(size_patches, 'B')
475	build_installed_size_human = build.humanize_number(size_installed, 'B')
476	build_size = 'usage: %s' % (build_max_size_human)
477	build_size += ' total: %s' % (build_total_size_human)
478	build_size += ' (sources: %s' % (build_sources_size_human)
479	build_size += ', patches: %s' % (build_patches_size_human)
480	build_size += ', installed %s)' % (build_installed_size_human)
481	#
482	# Cleaning ...
483	#
484	if deps is None and \
485	(not self.opts.no_clean() or self.opts.always_clean()):
486	for b in builds:
487	if not b.disabled():
488	log.notice('cleaning: %s' % (b.name()))
489	b.cleanup()
490	#
491	# Log the build size message
492	#
493	if len(builds) > 1:
494	log.notice('Build Sizes: %s' % (build_size))
495	#
496	# Clear out the builds ...
497	#
498	for b in builds:
499	del b
500	except error.general as gerr:
501	if not build_error:
502	log.stderr(str(gerr))
503	raise
504	except KeyboardInterrupt:
505	mail_report = False
506	raise
507	except:
508	self.build_failure = 'RSB general failure'
509	raise
510	finally:
511	end = datetime.datetime.now()
512	os.environ['PATH'] = current_path
513	build_time = str(end - start)
514	if mail_report and not self.macros.defined('mail_disable'):
515	self.write_mail_header('Build Time: %s' % (build_time), True)
516	self.write_mail_header('', True)
517	if self.build_failure is not None:
518	mail_subject = 'FAILED %s (%s)' % \
519	(mail_subject, self.build_failure)
520	else:
521	mail_subject = 'PASSED %s' % (mail_subject)
522	mail_subject = 'Build %s: %s' % (reports.platform(mode = 'system'),
523	mail_subject)
524	self.write_mail_header(mail['header'], True)
525	self.write_mail_header('')
526	log.notice('Mailing report: %s' % (mail['to']))
527	body = self.get_mail_header()
528	body += 'Sizes' + os.linesep
529	body += '=====' + os.linesep + os.linesep
530	if len(builds) > 1:
531	body += 'Maximum build usage: ' + build_max_size_human + os.linesep
532	body += 'Total size: ' + build_total_size_human + os.linesep
533	body += 'Installed : ' + build_installed_size_human + os.linesep
534	body += 'Sources: ' + build_sources_size_human + os.linesep
535	body += 'Patches: ' + build_patches_size_human + os.linesep
536	else:
537	body += 'No packages built'
538	body += os.linesep
539	body += 'Output' + os.linesep
540	body += '======' + os.linesep + os.linesep
541	body += os.linesep.join(mail['output'].get())
542	body += os.linesep + os.linesep
543	body += 'Report' + os.linesep
544	body += '======' + os.linesep + os.linesep
545	body += self.get_mail_report()
546	if not opts.dry_run():
547	mail['mail'].send(mail['to'], mail_subject, body)
548	log.notice('Build Set: Time %s' % (build_time))
549
550	def list_bset_cfg_files(opts, configs):
551	if opts.get_arg('--list-configs') or opts.get_arg('--list-bsets'):
552	if opts.get_arg('--list-configs'):
553	ext = '.cfg'
554	else:
555	ext = '.bset'
556	for p in configs['paths']:
557	print('Examining: %s' % (os.path.relpath(p)))
558	for c in configs['files']:
559	if c.endswith(ext):
560	print(' %s' % (c))
561	return True
562	return False
563
564	def run():
565	import sys
566	ec = 0
567	setbuilder_error = False
568	mail = None
569	try:
570	optargs = { '--list-configs': 'List available configurations',
571	'--list-bsets': 'List available build sets',
572	'--list-configs': 'List available configuration files.',
573	'--list-deps': 'List the dependent files.',
574	'--bset-tar-file': 'Create a build set tar file',
575	'--pkg-tar-files': 'Create package tar files',
576	'--no-report': 'Do not create a package report.',
577	'--report-format': 'The report format (text, html, asciidoc).' }
578	mailer.append_options(optargs)
579	opts = options.load(sys.argv, optargs)
580	if opts.get_arg('--mail'):
581	mail = { 'mail' : mailer.mail(opts),
582	'output': log_capture() }
583	to_addr = opts.get_arg('--mail-to')
584	if to_addr is not None:
585	mail['to'] = to_addr[1]
586	else:
587	mail['to'] = opts.defaults.expand('%{_mail_tools_to}')
588	mail['from'] = mail['mail'].from_address()
589	log.notice('RTEMS Source Builder - Set Builder, %s' % (version.str()))
590	opts.log_info()
591	if not check.host_setup(opts):
592	raise error.general('host build environment is not set up correctly')
593	if mail:
594	mail['header'] = os.linesep.join(mail['output'].get()) + os.linesep
595	mail['header'] += os.linesep
596	mail['header'] += 'Host: ' + reports.platform('compact') + os.linesep
597	indent = ' '
598	for l in textwrap.wrap(reports.platform('extended'),
599	width = 80 - len(indent)):
600	mail['header'] += indent + l + os.linesep
601	configs = build.get_configs(opts)
602	if opts.get_arg('--list-deps'):
603	deps = []
604	else:
605	deps = None
606	if not list_bset_cfg_files(opts, configs):
607	prefix = opts.defaults.expand('%{_prefix}')
608	if opts.canadian_cross():
609	opts.disable_install()
610
611	if not opts.dry_run() and \
612	not opts.canadian_cross() and \
613	not opts.no_install() and \
614	not path.ispathwritable(prefix):
615	raise error.general('prefix is not writable: %s' % (path.host(prefix)))
616
617	for bset in opts.params():
618	setbuilder_error = True
619	b = buildset(bset, configs, opts)
620	b.build(deps, mail = mail)
621	b = None
622	setbuilder_error = False
623
624	if deps is not None:
625	c = 0
626	for d in sorted(set(deps)):
627	c += 1
628	print('dep[%d]: %s' % (c, d))
629	except error.general as gerr:
630	if not setbuilder_error:
631	log.stderr(str(gerr))
632	log.stderr('Build FAILED')
633	ec = 1
634	except error.internal as ierr:
635	if not setbuilder_error:
636	log.stderr(str(ierr))
637	log.stderr('Internal Build FAILED')
638	ec = 1
639	except error.exit as eerr:
640	pass
641	except KeyboardInterrupt:
642	log.notice('abort: user terminated')
643	ec = 1
644	except:
645	raise
646	log.notice('abort: unknown error')
647	ec = 1
648	sys.exit(ec)
649
650	if __name__ == "__main__":
651	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: