Context Navigation

source: rtems-source-builder/source-builder/sb/setbuilder.py @ c799e04

5

Last change on this file since c799e04 was c799e04, checked in by Chris Johns <chrisj@…>, on 07/06/19 at 09:20:09

5/packages: Add curl and update all packages with RTEMS 5 and LibBSD

Update rtems-bsp support to correctly handle hosts for BSP.
Clean up the options for a BSP.
Check all RTEMS 5 packages and add 5/rtems-packages for all that build.
Update download locations and checksum of all packages.

The work requires the kernel and Libbsd hacve the same prefix.

Property mode set to 100644

File size: 25.8 KB

Line
1	#
2	# RTEMS Tools Project (http://www.rtems.org/)
3	# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
4	# All rights reserved.
5	#
6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
7	#
8	# Permission to use, copy, modify, and/or distribute this software for any
9	# purpose with or without fee is hereby granted, provided that the above
10	# copyright notice and this permission notice appear in all copies.
11	#
12	# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13	# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14	# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15	# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16	# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17	# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18	# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20	#
21	# This code builds a package compiler tool suite given a tool set. A tool
22	# set lists the various tools. These are specific tool configurations.
23	#
24
25	from __future__ import print_function
26
27	import copy
28	import datetime
29	import glob
30	import operator
31	import os
32	import sys
33	import textwrap
34
35	try:
36	import build
37	import check
38	import error
39	import log
40	import mailer
41	import options
42	import path
43	import reports
44	import sources
45	import version
46	except KeyboardInterrupt:
47	print('abort: user terminated', file = sys.stderr)
48	sys.exit(1)
49	except:
50	print('error: unknown application load error', file = sys.stderr)
51	sys.exit(1)
52
53	class log_capture(object):
54	def __init__(self):
55	self.log = []
56	log.capture = self.capture
57
58	def __str__(self):
59	return os.linesep.join(self.log)
60
61	def capture(self, text):
62	self.log += [l for l in text.replace(chr(13), '').splitlines()]
63
64	def get(self):
65	return self.log
66
67	def clear(self):
68	self.log = []
69
70	class buildset:
71	"""Build a set builds a set of packages."""
72
73	def __init__(self, bset, _configs, opts, macros = None):
74	log.trace('_bset: %s: init' % (bset))
75	self.configs = _configs
76	self.opts = opts
77	if macros is None:
78	self.macros = copy.copy(opts.defaults)
79	else:
80	self.macros = copy.copy(macros)
81	log.trace('_bset: %s: macro defaults' % (bset))
82	log.trace(str(self.macros))
83	self.bset = bset
84	_target = self.macros.expand('%{_target}')
85	if len(_target):
86	pkg_prefix = _target
87	else:
88	pkg_prefix = self.macros.expand('%{_host}')
89	self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset)
90	self.mail_header = ''
91	self.mail_report = ''
92	self.mail_report_0subject = ''
93	self.build_failure = None
94
95	def write_mail_header(self, text = '', prepend = False):
96	if type(text) is list:
97	text = os.linesep.join(text)
98	text = text.replace('\r', '').replace('\n', os.linesep)
99	if len(text) == 0 or text[-1] != os.linesep:
100	text += os.linesep
101	if prepend:
102	self.mail_header = text + self.mail_header
103	else:
104	self.mail_header += text
105
106	def get_mail_header(self):
107	return self.mail_header
108
109	def write_mail_report(self, text, prepend = False):
110	if type(text) is list:
111	text = os.linesep.join(text)
112	text = text.replace('\r', '').replace('\n', os.linesep)
113	if len(text) == 0 or text[-1] != os.linesep:
114	text += os.linesep
115	if prepend:
116	self.mail_report = text + self.mail_report
117	else:
118	self.mail_report += text
119
120	def get_mail_report(self):
121	return self.mail_report
122
123	def copy(self, src, dst):
124	log.output('copy: %s => %s' % (path.host(src), path.host(dst)))
125	if not self.opts.dry_run():
126	path.copy_tree(src, dst)
127
128	def report(self, _config, _build, opts, macros, format = None, mail = None):
129	if len(_build.main_package().name()) > 0 \
130	and not _build.macros.get('%{_disable_reporting}') \
131	and (not _build.opts.get_arg('--no-report') \
132	or _build.opts.get_arg('--mail')):
133	if format is None:
134	format = _build.opts.get_arg('--report-format')
135	if format is not None:
136	if len(format) != 2:
137	raise error.general('invalid report format option: %s' % \
138	('='.join(format)))
139	format = format[1]
140	if format is None:
141	format = 'text'
142	if format == 'text':
143	ext = '.txt'
144	elif format == 'asciidoc':
145	ext = '.txt'
146	elif format == 'html':
147	ext = '.html'
148	elif format == 'xml':
149	ext = '.xml'
150	elif format == 'ini':
151	ext = '.ini'
152	else:
153	raise error.general('invalid report format: %s' % (format))
154	buildroot = _build.config.abspath('%{buildroot}')
155	prefix = _build.macros.expand('%{_prefix}')
156	name = _build.main_package().name() + ext
157	log.notice('reporting: %s -> %s' % (_config, name))
158	if not _build.opts.get_arg('--no-report'):
159	outpath = path.host(path.join(buildroot, prefix, 'share', 'rtems', 'rsb'))
160	if not _build.opts.dry_run():
161	outname = path.host(path.join(outpath, name))
162	else:
163	outname = None
164	r = reports.report(format, self.configs,
165	copy.copy(opts), copy.copy(macros))
166	r.introduction(_build.config.file_name())
167	r.generate(_build.config.file_name())
168	r.epilogue(_build.config.file_name())
169	if not _build.opts.dry_run():
170	_build.mkdir(outpath)
171	r.write(outname)
172	del r
173	if mail:
174	r = reports.report('text', self.configs,
175	copy.copy(opts), copy.copy(macros))
176	r.introduction(_build.config.file_name())
177	r.generate(_build.config.file_name())
178	r.epilogue(_build.config.file_name())
179	self.write_mail_report(r.get_output())
180	del r
181
182	def root_copy(self, src, dst):
183	what = '%s -> %s' % \
184	(os.path.relpath(path.host(src)), os.path.relpath(path.host(dst)))
185	log.trace('_bset: %s: collecting: %s' % (self.bset, what))
186	self.copy(src, dst)
187
188	def install(self, name, buildroot, prefix):
189	dst = prefix
190	src = path.join(buildroot, prefix)
191	log.notice('installing: %s -> %s' % (name, path.host(dst)))
192	self.copy(src, dst)
193
194	def canadian_cross(self, _build):
195	log.trace('_bset: Cxc for build machine: _build => _host')
196	macros_to_copy = [('%{_host}', '%{_build}'),
197	('%{_host_alias}', '%{_build_alias}'),
198	('%{_host_arch}', '%{_build_arch}'),
199	('%{_host_cpu}', '%{_build_cpu}'),
200	('%{_host_os}', '%{_build_os}'),
201	('%{_host_vendor}', '%{_build_vendor}'),
202	('%{_tmproot}', '%{_tmpcxcroot}'),
203	('%{buildroot}', '%{buildcxcroot}'),
204	('%{_builddir}', '%{_buildcxcdir}')]
205	cxc_macros = _build.copy_init_macros()
206	for m in macros_to_copy:
207	log.trace('_bset: Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
208	cxc_macros[m[0]] = cxc_macros[m[1]]
209	_build.set_macros(cxc_macros)
210	_build.reload()
211	_build.make()
212	if not _build.macros.get('%{_disable_collecting}'):
213	self.root_copy(_build.config.expand('%{buildroot}'),
214	_build.config.expand('%{_tmproot}'))
215	_build.set_macros(_build.copy_init_macros())
216	_build.reload()
217
218	def build_package(self, _config, _build):
219	if not _build.disabled():
220	if _build.canadian_cross():
221	self.canadian_cross(_build)
222	_build.make()
223	if not _build.macros.get('%{_disable_collecting}'):
224	self.root_copy(_build.config.expand('%{buildroot}'),
225	_build.config.expand('%{_tmproot}'))
226
227	def bset_tar(self, _build):
228	tardir = _build.config.expand('%{_tardir}')
229	if (self.opts.get_arg('--bset-tar-file') or self.opts.canadian_cross()) \
230	and not _build.macros.get('%{_disable_packaging}'):
231	path.mkdir(tardir)
232	tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (_build.main_package().name())))
233	log.notice('tarball: %s' % (os.path.relpath(path.host(tar))))
234	if not self.opts.dry_run():
235	tmproot = _build.config.expand('%{_tmproot}')
236	cmd = _build.config.expand('"cd ' + tmproot + \
237	' && %{__tar} -cf - . \| %{__bzip2} > ' + tar + '"')
238	_build.run(cmd, shell_opts = '-c', cwd = tmproot)
239
240	def parse(self, bset):
241
242	def _clean(line):
243	line = line[0:-1]
244	b = line.find('#')
245	if b >= 0:
246	line = line[1:b]
247	return line.strip()
248
249	bsetname = bset
250
251	if not path.exists(bsetname):
252	for cp in self.macros.expand('%{_configdir}').split(':'):
253	configdir = path.abspath(cp)
254	bsetname = path.join(configdir, bset)
255	if path.exists(bsetname):
256	break
257	bsetname = None
258	if bsetname is None:
259	raise error.general('no build set file found: %s' % (bset))
260	try:
261	log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
262	bset = open(path.host(bsetname), 'r')
263	except IOError as err:
264	raise error.general('error opening bset file: %s' % (bsetname))
265
266	configs = []
267
268	try:
269	lc = 0
270	for l in bset:
271	lc += 1
272	l = _clean(l)
273	if len(l) == 0:
274	continue
275	log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l))
276	ls = l.split()
277	if ls[0][-1] == ':' and ls[0][:-1] == 'package':
278	self.bset_pkg = ls[1].strip()
279	self.macros['package'] = self.bset_pkg
280	elif ls[0][0] == '%':
281	def err(msg):
282	raise error.general('%s:%d: %s' % (self.bset, lc, msg))
283	if ls[0] == '%define':
284	if len(ls) > 2:
285	self.macros.define(ls[1].strip(),
286	' '.join([f.strip() for f in ls[2:]]))
287	else:
288	self.macros.define(ls[1].strip())
289	elif ls[0] == '%undefine':
290	if len(ls) > 2:
291	raise error.general('%s:%d: %undefine requires just the name' % \
292	(self.bset, lc))
293	self.macros.undefine(ls[1].strip())
294	elif ls[0] == '%include':
295	configs += self.parse(ls[1].strip())
296	elif ls[0] in ['%patch', '%source']:
297	sources.process(ls[0][1:], ls[1:], self.macros, err)
298	elif ls[0] == '%hash':
299	sources.hash(ls[1:], self.macros, err)
300	else:
301	l = l.strip()
302	c = build.find_config(l, self.configs)
303	if c is None:
304	raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l))
305	configs += [c]
306	except:
307	bset.close()
308	raise
309
310	bset.close()
311
312	return configs
313
314	def load(self):
315
316	#
317	# If the build set file ends with .cfg the user has passed to the
318	# buildset builder a configuration so we just return it.
319	#
320	if self.bset.endswith('.cfg'):
321	configs = [self.bset]
322	else:
323	exbset = self.macros.expand(self.bset)
324	self.macros['_bset'] = exbset
325	self.macros['_bset_tmp'] = build.short_name(exbset)
326	root, ext = path.splitext(exbset)
327	if exbset.endswith('.bset'):
328	bset = exbset
329	else:
330	bset = '%s.bset' % (exbset)
331	configs = self.parse(bset)
332	return configs
333
334	def build(self, deps = None, nesting_count = 0, mail = None):
335
336	build_error = False
337
338	nesting_count += 1
339
340	if mail:
341	mail['output'].clear()
342
343	log.trace('_bset: %s: make' % (self.bset))
344	log.notice('Build Set: %s' % (self.bset))
345
346	mail_subject = '%s on %s' % (self.bset,
347	self.macros.expand('%{_host}'))
348
349	current_path = os.environ['PATH']
350
351	start = datetime.datetime.now()
352
353	mail_report = False
354	have_errors = False
355
356	if mail:
357	mail['output'].clear()
358
359	try:
360	configs = self.load()
361
362	log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
363
364	sizes_valid = False
365	builds = []
366	for s in range(0, len(configs)):
367	b = None
368	try:
369	#
370	# Each section of the build set gets a separate set of
371	# macros so we do not contaminate one configuration with
372	# another.
373	#
374	opts = copy.copy(self.opts)
375	macros = copy.copy(self.macros)
376	if configs[s].endswith('.bset'):
377	log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
378	bs = buildset(configs[s], self.configs, opts, macros)
379	bs.build(deps, nesting_count, mail)
380	del bs
381	elif configs[s].endswith('.cfg'):
382	if mail:
383	mail_report = True
384	log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
385	try:
386	b = build.build(configs[s],
387	self.opts.get_arg('--pkg-tar-files'),
388	opts,
389	macros)
390	except:
391	build_error = True
392	raise
393	if b.macros.get('%{_disable_reporting}'):
394	mail_report = False
395	if deps is None:
396	self.build_package(configs[s], b)
397	self.report(configs[s], b,
398	copy.copy(self.opts),
399	copy.copy(self.macros),
400	mail = mail)
401	# Always produce an XML report.
402	self.report(configs[s], b,
403	copy.copy(self.opts),
404	copy.copy(self.macros),
405	format = 'xml',
406	mail = mail)
407	if s == len(configs) - 1 and not have_errors:
408	self.bset_tar(b)
409	else:
410	deps += b.config.includes()
411	builds += [b]
412	#
413	# Dump post build macros.
414	#
415	log.trace('_bset: macros post-build')
416	log.trace(str(b.macros))
417	else:
418	raise error.general('invalid config type: %s' % (configs[s]))
419	except error.general as gerr:
420	have_errors = True
421	if b is not None:
422	if self.build_failure is None:
423	self.build_failure = b.name()
424	self.write_mail_header('')
425	self.write_mail_header('= ' * 40)
426	self.write_mail_header('Build FAILED: %s' % (b.name()))
427	self.write_mail_header('- ' * 40)
428	self.write_mail_header(str(log.default))
429	self.write_mail_header('- ' * 40)
430	if self.opts.keep_going():
431	log.notice(str(gerr))
432	if self.opts.always_clean():
433	builds += [b]
434	else:
435	raise
436	else:
437	raise
438	#
439	# Installing ...
440	#
441	log.trace('_bset: installing: deps:%r no-install:%r' % \
442	(deps is None, self.opts.no_install()))
443	if deps is None \
444	and not self.opts.no_install() \
445	and not have_errors:
446	for b in builds:
447	log.trace('_bset: installing: %r' % b.installable())
448	if b.installable():
449	self.install(b.name(),
450	b.config.expand('%{buildroot}'),
451	b.config.expand('%{_prefix}'))
452	#
453	# Sizes ...
454	#
455	if len(builds) > 1:
456	size_build = 0
457	size_installed = 0
458	size_build_max = 0
459	for b in builds:
460	s = b.get_build_size()
461	size_build += s
462	if s > size_build_max:
463	size_build_max = s
464	size_installed += b.get_installed_size()
465	size_sources = 0
466	for p in builds[0].config.expand('%{_sourcedir}').split(':'):
467	size_sources += path.get_size(p)
468	size_patches = 0
469	for p in builds[0].config.expand('%{_patchdir}').split(':'):
470	size_patches += path.get_size(p)
471	size_total = size_sources + size_patches + size_installed
472	build_max_size_human = build.humanize_number(size_build_max + size_installed, 'B')
473	build_total_size_human = build.humanize_number(size_total, 'B')
474	build_sources_size_human = build.humanize_number(size_sources, 'B')
475	build_patches_size_human = build.humanize_number(size_patches, 'B')
476	build_installed_size_human = build.humanize_number(size_installed, 'B')
477	build_size = 'usage: %s' % (build_max_size_human)
478	build_size += ' total: %s' % (build_total_size_human)
479	build_size += ' (sources: %s' % (build_sources_size_human)
480	build_size += ', patches: %s' % (build_patches_size_human)
481	build_size += ', installed %s)' % (build_installed_size_human)
482	sizes_valid = True
483	#
484	# Cleaning ...
485	#
486	if deps is None and \
487	(not self.opts.no_clean() or self.opts.always_clean()):
488	for b in builds:
489	if not b.disabled():
490	log.notice('cleaning: %s' % (b.name()))
491	b.cleanup()
492	#
493	# Log the build size message
494	#
495	if len(builds) > 1:
496	log.notice('Build Sizes: %s' % (build_size))
497	#
498	# Clear out the builds ...
499	#
500	for b in builds:
501	del b
502	except error.general as gerr:
503	if not build_error:
504	log.stderr(str(gerr))
505	raise
506	except KeyboardInterrupt:
507	mail_report = False
508	raise
509	except:
510	self.build_failure = 'RSB general failure'
511	raise
512	finally:
513	end = datetime.datetime.now()
514	os.environ['PATH'] = current_path
515	build_time = str(end - start)
516	if mail_report and not self.macros.defined('mail_disable'):
517	self.write_mail_header('Build Time: %s' % (build_time), True)
518	self.write_mail_header('', True)
519	if self.build_failure is not None:
520	mail_subject = 'FAILED %s (%s)' % \
521	(mail_subject, self.build_failure)
522	else:
523	mail_subject = 'PASSED %s' % (mail_subject)
524	mail_subject = 'Build %s: %s' % (reports.platform(mode = 'system'),
525	mail_subject)
526	self.write_mail_header(mail['header'], True)
527	self.write_mail_header('')
528	log.notice('Mailing report: %s' % (mail['to']))
529	body = self.get_mail_header()
530	body += 'Sizes' + os.linesep
531	body += '=====' + os.linesep + os.linesep
532	if sizes_valid:
533	body += 'Maximum build usage: ' + build_max_size_human + os.linesep
534	body += 'Total size: ' + build_total_size_human + os.linesep
535	body += 'Installed : ' + build_installed_size_human + os.linesep
536	body += 'Sources: ' + build_sources_size_human + os.linesep
537	body += 'Patches: ' + build_patches_size_human + os.linesep
538	else:
539	body += 'No packages built'
540	body += os.linesep
541	body += 'Output' + os.linesep
542	body += '======' + os.linesep + os.linesep
543	body += os.linesep.join(mail['output'].get())
544	body += os.linesep + os.linesep
545	body += 'Report' + os.linesep
546	body += '======' + os.linesep + os.linesep
547	body += self.get_mail_report()
548	if not opts.dry_run():
549	mail['mail'].send(mail['to'], mail_subject, body)
550	log.notice('Build Set: Time %s' % (build_time))
551
552	def list_bset_cfg_files(opts, configs):
553	if opts.get_arg('--list-configs') or opts.get_arg('--list-bsets'):
554	if opts.get_arg('--list-configs'):
555	ext = '.cfg'
556	else:
557	ext = '.bset'
558	for p in configs['paths']:
559	print('Examining: %s' % (os.path.relpath(p)))
560	for c in configs['files']:
561	if c.endswith(ext):
562	print(' %s' % (c))
563	return True
564	return False
565
566	def run():
567	import sys
568	ec = 0
569	setbuilder_error = False
570	mail = None
571	try:
572	optargs = { '--list-configs': 'List available configurations',
573	'--list-bsets': 'List available build sets',
574	'--list-configs': 'List available configuration files.',
575	'--list-deps': 'List the dependent files.',
576	'--bset-tar-file': 'Create a build set tar file',
577	'--pkg-tar-files': 'Create package tar files',
578	'--no-report': 'Do not create a package report.',
579	'--report-format': 'The report format (text, html, asciidoc).' }
580	mailer.append_options(optargs)
581	opts = options.load(sys.argv, optargs)
582	if opts.get_arg('--mail'):
583	mail = { 'mail' : mailer.mail(opts),
584	'output': log_capture() }
585	to_addr = opts.get_arg('--mail-to')
586	if to_addr is not None:
587	mail['to'] = to_addr[1]
588	else:
589	mail['to'] = opts.defaults.expand('%{_mail_tools_to}')
590	mail['from'] = mail['mail'].from_address()
591	log.notice('RTEMS Source Builder - Set Builder, %s' % (version.str()))
592	opts.log_info()
593	if not check.host_setup(opts):
594	raise error.general('host build environment is not set up correctly')
595	if mail:
596	mail['header'] = os.linesep.join(mail['output'].get()) + os.linesep
597	mail['header'] += os.linesep
598	mail['header'] += 'Host: ' + reports.platform('compact') + os.linesep
599	indent = ' '
600	for l in textwrap.wrap(reports.platform('extended'),
601	width = 80 - len(indent)):
602	mail['header'] += indent + l + os.linesep
603	configs = build.get_configs(opts)
604	if opts.get_arg('--list-deps'):
605	deps = []
606	else:
607	deps = None
608	if not list_bset_cfg_files(opts, configs):
609	prefix = opts.defaults.expand('%{_prefix}')
610	if opts.canadian_cross():
611	opts.disable_install()
612
613	if not opts.dry_run() and \
614	not opts.canadian_cross() and \
615	not opts.no_install() and \
616	not path.ispathwritable(prefix):
617	raise error.general('prefix is not writable: %s' % (path.host(prefix)))
618
619	for bset in opts.params():
620	setbuilder_error = True
621	b = buildset(bset, configs, opts)
622	b.build(deps, mail = mail)
623	b = None
624	setbuilder_error = False
625
626	if deps is not None:
627	c = 0
628	for d in sorted(set(deps)):
629	c += 1
630	print('dep[%d]: %s' % (c, d))
631	except error.general as gerr:
632	if not setbuilder_error:
633	log.stderr(str(gerr))
634	log.stderr('Build FAILED')
635	ec = 1
636	except error.internal as ierr:
637	if not setbuilder_error:
638	log.stderr(str(ierr))
639	log.stderr('Internal Build FAILED')
640	ec = 1
641	except error.exit as eerr:
642	pass
643	except KeyboardInterrupt:
644	log.notice('abort: user terminated')
645	ec = 1
646	except:
647	raise
648	log.notice('abort: unknown error')
649	ec = 1
650	sys.exit(ec)
651
652	if __name__ == "__main__":
653	run()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: