Context Navigation

build.py @ 65d9457

4.104.114.95

Last change on this file since 65d9457 was 65d9457, checked in by Chris Johns <chrisj@…>, on 05/14/14 at 22:49:34

sb: Move error report generation to the build phase.

Generate a separate report of each package being built in a build set.
This creates a better list of faults in the case of nesting build sets
such as */rtems-all.

Property mode set to 100644

File size: 18.7 KB

Rev	Line
[bf13d27]	1	#
	2	# RTEMS Tools Project (http://www.rtems.org/)
[649a64c]	3	# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
[bf13d27]	4	# All rights reserved.
	5	#
	6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
	7	#
	8	# Permission to use, copy, modify, and/or distribute this software for any
	9	# purpose with or without fee is hereby granted, provided that the above
	10	# copyright notice and this permission notice appear in all copies.
	11	#
	12	# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
	13	# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
	14	# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
	15	# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	16	# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
	17	# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
	18	# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	19
	20	#
	21	# This code builds a package given a config file. It only builds to be
	22	# installed not to be package unless you run a packager around this.
	23	#
	24
	25	import getopt
[015fa1b]	26	import glob
[bf13d27]	27	import os
	28	import shutil
	29	import stat
	30	import sys
	31	import urllib2
	32	import urlparse
	33
[efb6688]	34	try:
	35	import check
	36	import config
[649a64c]	37	import download
[efb6688]	38	import error
[74da24c]	39	import ereport
[efb6688]	40	import execute
	41	import log
[cb12e48]	42	import options
[efb6688]	43	import path
[9a15c40]	44	import sources
[efb6688]	45	import version
	46	except KeyboardInterrupt:
	47	print 'abort: user terminated'
	48	sys.exit(1)
	49	except:
	50	print 'error: unknown application load error'
	51	sys.exit(1)
[bf13d27]	52
	53	class script:
	54	"""Create and manage a shell script."""
	55
[5142bec]	56	def __init__(self):
[bf13d27]	57	self.reset()
	58
	59	def reset(self):
	60	self.body = []
	61	self.lc = 0
	62
	63	def append(self, text):
	64	if type(text) is str:
	65	text = text.splitlines()
[5142bec]	66	if not log.quiet:
[bf13d27]	67	i = 0
	68	for l in text:
	69	i += 1
[cafbcc6]	70	log.output('script:%3d: %s' % (self.lc + i, l))
[bf13d27]	71	self.lc += len(text)
	72	self.body.extend(text)
	73
	74	def write(self, name, check_for_errors = False):
	75	s = None
	76	try:
[ab8319a]	77	s = open(path.host(name), 'w')
[bf13d27]	78	s.write('\n'.join(self.body))
	79	s.close()
[ab8319a]	80	os.chmod(path.host(name), stat.S_IRWXU \| \
[bf13d27]	81	stat.S_IRGRP \| stat.S_IXGRP \| \
	82	stat.S_IROTH \| stat.S_IXOTH)
	83	except IOError, err:
	84	raise error.general('creating script: ' + name)
	85	except:
	86	if s is not None:
	87	s.close()
	88	raise
	89	if s is not None:
	90	s.close()
	91
	92	class build:
	93	"""Build a package given a config file."""
	94
[5237f1cc]	95	def _name_(self, name):
	96	#
	97	# If on Windows use shorter names to keep the build paths.
	98	#
	99	if options.host_windows:
	100	buildname = ''
	101	add = True
	102	for c in name:
	103	if c == '-':
	104	add = True
	105	elif add:
	106	buildname += c
	107	add = False
	108	return buildname
	109	else:
	110	return name
	111
[65d9457]	112	def _generate_report_(self, header):
	113	ereport.generate('rsb-report-%s.txt' % self.macros['name'], self.opts, header)
	114
[cb12e48]	115	def __init__(self, name, create_tar_files, opts, macros = None):
[bf13d27]	116	self.opts = opts
[cb12e48]	117	if macros is None:
	118	self.macros = opts.defaults
	119	else:
	120	self.macros = macros
[cafbcc6]	121	self.create_tar_files = create_tar_files
[5142bec]	122	log.notice('config: ' + name)
[cb12e48]	123	self.config = config.file(name, opts, self.macros)
[5142bec]	124	self.script = script()
[5237f1cc]	125	self.macros['buildname'] = self._name_(self.macros['name'])
[bf13d27]	126
[ab8319a]	127	def rmdir(self, rmpath):
[5142bec]	128	log.output('removing: %s' % (path.host(rmpath)))
[bf13d27]	129	if not self.opts.dry_run():
[ab8319a]	130	if path.exists(rmpath):
[ee47d72]	131	path.removeall(rmpath)
[bf13d27]	132
[ab8319a]	133	def mkdir(self, mkpath):
[5142bec]	134	log.output('making dir: %s' % (path.host(mkpath)))
[bf13d27]	135	if not self.opts.dry_run():
[ee47d72]	136	path.mkdir(mkpath)
[bf13d27]	137
[4f26bdb]	138	def canadian_cross(self):
	139	_host = self.config.expand('%{_host}')
	140	_build = self.config.expand('%{_build}')
	141	_target = self.config.expand('%{_target}')
	142	return self.config.defined('%{allow_cxc}') and \
	143	_host != _build and _host != _target
	144
[9a15c40]	145	def source(self, name):
	146	#
	147	# Return the list of sources. Merge in any macro defined sources as
	148	# these may be overridden by user loaded macros.
	149	#
	150	_map = 'source-%s' % (name)
	151	src_keys = self.macros.map_keys(_map)
	152	if len(src_keys) == 0:
	153	raise error.general('no source set: %s (%s)' % (name, _map))
	154	srcs = []
	155	for s in src_keys:
	156	sm = self.macros.get(s, globals = False, maps = _map)
	157	if sm is None:
	158	raise error.internal('source macro not found: %s in %s (%s)' % \
	159	(s, name, _map))
	160	url = self.config.expand(sm[2])
	161	src = download.parse_url(url, '_sourcedir', self.config, self.opts)
	162	download.get_file(src['url'], src['local'], self.opts, self.config)
	163	if 'symlink' in src:
	164	src['script'] = '%%{__ln_s} %s ${source_dir_%s}' % (src['symlink'], name)
	165	elif 'compressed' in src:
	166	#
	167	# Zip files unpack as well so do not use tar.
	168	#
	169	src['script'] = '%s %s' % (src['compressed'], src['local'])
	170	if src['compressed-type'] != 'zip':
	171	src['script'] += ' \| %{__tar_extract} -'
	172	else:
[83586f7]	173	src['script'] = '%%{__tar_extract} %s' % (src['local'])
[9a15c40]	174	srcs += [src]
	175	return srcs
	176
	177	def source_setup(self, package, args):
	178	log.output('source setup: %s: %s' % (package.name(), ' '.join(args)))
	179	setup_name = args[1]
	180	args = args[1:]
	181	try:
	182	opts, args = getopt.getopt(args[1:], 'qDcn:b:a:')
	183	except getopt.GetoptError, ge:
	184	raise error.general('source setup error: %s' % str(ge))
[bf13d27]	185	quiet = False
[7618a74c]	186	unpack_before_chdir = True
[bf13d27]	187	delete_before_unpack = True
	188	create_dir = False
[9a15c40]	189	deleted_dir = False
	190	created_dir = False
	191	changed_dir = False
	192	opt_name = None
[bf13d27]	193	for o in opts:
	194	if o[0] == '-q':
	195	quiet = True
	196	elif o[0] == '-D':
	197	delete_before_unpack = False
	198	elif o[0] == '-c':
	199	create_dir = True
	200	elif o[0] == '-n':
[9a15c40]	201	opt_name = o[1]
[bf13d27]	202	elif o[0] == '-b':
	203	unpack_before_chdir = True
	204	elif o[0] == '-a':
	205	unpack_before_chdir = False
[9a15c40]	206	name = None
	207	for source in self.source(setup_name):
	208	if name is None:
	209	if opt_name is None:
	210	if source:
	211	opt_name = source['name']
	212	else:
	213	raise error.general('setup source tag not found: %d' % (source_tag))
	214	else:
	215	name = opt_name
	216	name = self._name_(name)
	217	self.script.append(self.config.expand('cd %{_builddir}'))
	218	if not deleted_dir and delete_before_unpack:
	219	self.script.append(self.config.expand('%{__rm} -rf ' + name))
	220	deleted_dir = True
	221	if not created_dir and create_dir:
	222	self.script.append(self.config.expand('%{__mkdir_p} ' + name))
	223	created_dir = True
	224	if not changed_dir and (not unpack_before_chdir or create_dir):
	225	self.script.append(self.config.expand('cd ' + name))
	226	changed_dir = True
	227	self.script.append(self.config.expand(source['script']))
	228	if not changed_dir and (unpack_before_chdir and not create_dir):
[bf13d27]	229	self.script.append(self.config.expand('cd ' + name))
[9a15c40]	230	changed_dir = True
[bf13d27]	231	self.script.append(self.config.expand('%{__setup_post}'))
	232
[9a15c40]	233	def patch_setup(self, package, args):
	234	name = args[1]
	235	args = args[2:]
	236	_map = 'patch-%s' % (name)
	237	default_opts = ' '.join(args)
	238	patch_keys = self.macros.map_keys(_map)
	239	patches = []
	240	for p in patch_keys:
	241	pm = self.macros.get(p, globals = False, maps = _map)
	242	if pm is None:
	243	raise error.internal('patch macro not found: %s in %s (%s)' % \
	244	(p, name, _map))
	245	opts = []
	246	url = []
	247	for pp in pm[2].split():
	248	if len(url) == 0 and pp[0] == '-':
	249	opts += [pp]
	250	else:
	251	url += [pp]
	252	if len(url) == 0:
	253	raise error.general('patch URL not found: %s' % (' '.join(args)))
	254	if len(opts) == 0:
	255	opts = default_opts
	256	else:
	257	opts = ' '.join(opts)
	258	opts = self.config.expand(opts)
	259	url = self.config.expand(' '.join(url))
	260	#
	261	# Parse the URL first in the source builder's patch directory.
	262	#
	263	patch = download.parse_url(url, '_patchdir', self.config, self.opts)
	264	#
	265	# If not in the source builder package check the source directory.
	266	#
	267	if not path.exists(patch['local']):
	268	patch = download.parse_url(url, '_patchdir', self.config, self.opts)
	269	download.get_file(patch['url'], patch['local'], self.opts, self.config)
	270	if 'compressed' in patch:
	271	patch['script'] = patch['compressed'] + ' ' + patch['local']
	272	else:
	273	patch['script'] = '%{__cat} ' + patch['local']
	274	patch['script'] += ' \| %%{__patch} %s' % (opts)
	275	self.script.append(self.config.expand(patch['script']))
	276
[bf13d27]	277	def run(self, command, shell_opts = '', cwd = None):
	278	e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
	279	cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
[5142bec]	280	log.output('run: ' + cmd)
[ab8319a]	281	exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
[bf13d27]	282	if exit_code != 0:
[74da24c]	283	log.output('shell cmd failed: %s' % (cmd))
	284	raise error.general('building %s' % (self.macros['buildname']))
[bf13d27]	285
	286	def builddir(self):
	287	builddir = self.config.abspath('_builddir')
	288	self.rmdir(builddir)
	289	if not self.opts.dry_run():
	290	self.mkdir(builddir)
	291
	292	def prep(self, package):
	293	self.script.append('echo "==> %prep:"')
	294	_prep = package.prep()
[ebf8a1f]	295	if _prep:
	296	for l in _prep:
	297	args = l.split()
[9a15c40]	298	if len(args):
	299	if args[0] == '%setup':
	300	if len(args) == 1:
	301	raise error.general('invalid %%setup directive: %s' % (' '.join(args)))
	302	if args[1] == 'source':
	303	self.source_setup(package, args[1:])
	304	elif args[1] == 'patch':
	305	self.patch_setup(package, args[1:])
	306	elif args[0].startswith('%patch'):
	307	self.patch(package, args)
	308	else:
	309	self.script.append(' '.join(args))
[bf13d27]	310
	311	def build(self, package):
[4f26bdb]	312	self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
	313	self.script.append('%s ${SB_BUILD_ROOT}' %
	314	(self.config.expand('%{__rmdir}')))
	315	self.script.append('%s ${SB_BUILD_ROOT}' %
	316	(self.config.expand('%{__mkdir_p}')))
[bf13d27]	317	self.script.append('echo "==> %build:"')
	318	_build = package.build()
[ebf8a1f]	319	if _build:
	320	for l in _build:
	321	self.script.append(l)
[bf13d27]	322
	323	def install(self, package):
	324	self.script.append('echo "==> %install:"')
	325	_install = package.install()
[ebf8a1f]	326	if _install:
	327	for l in _install:
	328	args = l.split()
	329	self.script.append(' '.join(args))
[bf13d27]	330
	331	def files(self, package):
[ebf8a1f]	332	if self.create_tar_files \
	333	and not self.macros.get('%{_disable_packaging'):
[cafbcc6]	334	self.script.append('echo "==> %files:"')
[0add2ea]	335	inpath = path.abspath(self.config.expand('%{buildroot}'))
[cafbcc6]	336	tardir = path.abspath(self.config.expand('%{_tardir}'))
	337	self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
[4f26bdb]	338	self.script.append(self.config.expand(' %%{__mkdir_p} %s' % tardir))
[cafbcc6]	339	self.script.append(self.config.expand(' cd ' + inpath))
	340	tar = path.join(tardir, package.long_name() + '.tar.bz2')
	341	cmd = self.config.expand(' %{__tar} -cf - . ' + '\| %{__bzip2} > ' + tar)
	342	self.script.append(cmd)
	343	self.script.append(self.config.expand(' cd %{_builddir}'))
	344	self.script.append('fi')
[bf13d27]	345
	346	def clean(self, package):
	347	self.script.append('echo "==> %clean:"')
	348	_clean = package.clean()
	349	if _clean is not None:
	350	for l in _clean:
	351	args = l.split()
	352	self.script.append(' '.join(args))
	353
[4f26bdb]	354	def build_package(self, package):
	355	if self.canadian_cross():
	356	self.script.append('echo "==> Candian-cross build/target:"')
	357	self.script.append('SB_CXC="yes"')
	358	else:
	359	self.script.append('SB_CXC="no"')
	360	self.build(package)
	361	self.install(package)
	362	self.files(package)
	363	if not self.opts.no_clean():
	364	self.clean(package)
	365
[bf13d27]	366	def cleanup(self):
[ebf8a1f]	367	package = self.main_package()
	368	if not package.disabled() and not self.opts.no_clean():
[bf13d27]	369	buildroot = self.config.abspath('buildroot')
	370	builddir = self.config.abspath('_builddir')
[4f26bdb]	371	buildcxcdir = self.config.abspath('_buildcxcdir')
[8d7624e]	372	tmproot = self.config.abspath('_tmproot')
[5142bec]	373	log.trace('cleanup: %s' % (buildroot))
[bf13d27]	374	self.rmdir(buildroot)
[5142bec]	375	log.trace('cleanup: %s' % (builddir))
[bf13d27]	376	self.rmdir(builddir)
[4f26bdb]	377	if self.canadian_cross():
[5142bec]	378	log.trace('cleanup: %s' % (buildcxcdir))
[4f26bdb]	379	self.rmdir(buildcxcdir)
[5142bec]	380	log.trace('cleanup: %s' % (tmproot))
[8d7624e]	381	self.rmdir(tmproot)
[bf13d27]	382
[54d76bb]	383	def main_package(self):
[bf13d27]	384	packages = self.config.packages()
[54d76bb]	385	return packages['main']
	386
	387	def make(self):
	388	package = self.main_package()
[ebf8a1f]	389	if package.disabled():
	390	log.notice('package: nothing to build')
[4f26bdb]	391	else:
[65d9457]	392	try:
	393	name = package.name()
[ebf8a1f]	394	if self.canadian_cross():
[65d9457]	395	log.notice('package: (Cxc) %s' % (name))
[ebf8a1f]	396	else:
[65d9457]	397	log.notice('package: %s' % (name))
	398	log.trace('---- macro maps %s' % ('-' * 55))
	399	log.trace('%s' % (str(self.config.macros)))
	400	log.trace('-' * 70)
	401	self.script.reset()
	402	self.script.append(self.config.expand('%{___build_template}'))
	403	self.script.append('echo "=> ' + name + ':"')
	404	self.prep(package)
	405	self.build_package(package)
	406	if not self.opts.dry_run():
	407	self.builddir()
	408	sn = path.join(self.config.expand('%{_builddir}'), 'doit')
	409	log.output('write script: ' + sn)
	410	self.script.write(sn)
	411	if self.canadian_cross():
	412	log.notice('building: (Cxc) %s' % (name))
	413	else:
	414	log.notice('building: %s' % (name))
	415	self.run(sn)
	416	except error.general, gerr:
	417	log.notice(str(gerr))
	418	log.stderr('Build FAILED')
	419	self._generate_report_('Build: %s' % (gerr))
	420	raise
	421	except error.internal, ierr:
	422	log.notice(str(ierr))
	423	log.stderr('Internal Build FAILED')
	424	self._generate_report_('Build: %s' % (ierr))
	425	raise
	426	except:
	427	raise
	428	if self.opts.dry_run():
	429	self._generate_report_('Build: dry run')
[bf13d27]	430
	431	def name(self):
	432	packages = self.config.packages()
	433	package = packages['main']
	434	return package.name()
	435
[ebf8a1f]	436	def disabled(self):
	437	packages = self.config.packages()
	438	package = packages['main']
	439	return package.disabled()
	440
[cb12e48]	441	def get_configs(opts):
[e908afb]	442
	443	def _scan(_path, ext):
	444	configs = []
	445	for root, dirs, files in os.walk(_path):
	446	prefix = root[len(_path) + 1:]
	447	for file in files:
[fba1136]	448	for e in ext:
	449	if file.endswith(e):
	450	configs += [path.join(prefix, file)]
[e908afb]	451	return configs
	452
[fba1136]	453	configs = { 'paths': [], 'files': [] }
[cb12e48]	454	for cp in opts.defaults.expand('%{_configdir}').split(':'):
[bc71066]	455	hcp = path.host(path.abspath(cp))
	456	configs['paths'] += [hcp]
	457	configs['files'] += _scan(hcp, ['.cfg', '.bset'])
[fba1136]	458	configs['files'] = sorted(configs['files'])
	459	return configs
[71b8893]	460
[0759d98]	461	def find_config(config, configs):
	462	config_root, config_ext = path.splitext(config)
	463	if config_ext not in ['', '.bset', '.cfg']:
	464	config_root = config
	465	config_ext = ''
	466	for c in configs['files']:
	467	r, e = path.splitext(c)
	468	if config_root == r:
	469	if config_ext == '' or config_ext == e:
	470	return c
	471	return None
	472
[bf13d27]	473	def run(args):
[74da24c]	474	ec = 0
[bf13d27]	475	try:
[71b8893]	476	optargs = { '--list-configs': 'List available configurations' }
[cb12e48]	477	opts = options.load(args, optargs)
[5142bec]	478	log.notice('RTEMS Source Builder, Package Builder v%s' % (version.str()))
[cb12e48]	479	if not check.host_setup(opts):
[71b8893]	480	if not opts.force():
[0add2ea]	481	raise error.general('host build environment is not set up' +
[4f26bdb]	482	' correctly (use --force to proceed)')
[5142bec]	483	log.notice('warning: forcing build with known host setup problems')
[71b8893]	484	if opts.get_arg('--list-configs'):
[cb12e48]	485	configs = get_configs(opts)
[cafbcc6]	486	for p in configs['paths']:
	487	print 'Examining: %s' % (os.path.relpath(p))
	488	for c in configs['files']:
	489	if c.endswith('.cfg'):
	490	print ' %s' % (c)
[71b8893]	491	else:
	492	for config_file in opts.config_files():
[cb12e48]	493	b = build(config_file, True, opts)
[71b8893]	494	b.make()
[74da24c]	495	b = None
[bf13d27]	496	except error.general, gerr:
[74da24c]	497	log.stderr('Build FAILED')
	498	ec = 1
[bf13d27]	499	except error.internal, ierr:
[74da24c]	500	log.stderr('Internal Build FAILED')
	501	ec = 1
[bf13d27]	502	except error.exit, eerr:
	503	pass
	504	except KeyboardInterrupt:
[5142bec]	505	log.notice('abort: user terminated')
[74da24c]	506	ec = 1
	507	sys.exit(ec)
[bf13d27]	508
	509	if __name__ == "__main__":
	510	run(sys.argv)

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: rtems-source-builder/source-builder/sb/build.py @ 65d9457

Download in other formats: