source: rtems-source-builder/source-builder/sb/build.py @ 74da24c

4.104.114.95
Last change on this file since 74da24c was 74da24c, checked in by Chris Johns <chrisj@…>, on 05/08/14 at 02:58:14

sb: Generate an error report on an error.

Generate an error report users can send to the mailing list with
error details.
  • Property mode set to 100644
File size: 17.3 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import getopt
26import glob
27import os
28import shutil
29import stat
30import sys
31import urllib2
32import urlparse
33
34try:
35    import check
36    import config
37    import download
38    import error
39    import ereport
40    import execute
41    import log
42    import options
43    import path
44    import version
45except KeyboardInterrupt:
46    print 'abort: user terminated'
47    sys.exit(1)
48except:
49    print 'error: unknown application load error'
50    sys.exit(1)
51
52class script:
53    """Create and manage a shell script."""
54
55    def __init__(self):
56        self.reset()
57
58    def reset(self):
59        self.body = []
60        self.lc = 0
61
62    def append(self, text):
63        if type(text) is str:
64            text = text.splitlines()
65        if not log.quiet:
66            i = 0
67            for l in text:
68                i += 1
69                log.output('script:%3d: %s' % (self.lc + i, l))
70        self.lc += len(text)
71        self.body.extend(text)
72
73    def write(self, name, check_for_errors = False):
74        s = None
75        try:
76            s = open(path.host(name), 'w')
77            s.write('\n'.join(self.body))
78            s.close()
79            os.chmod(path.host(name), stat.S_IRWXU | \
80                         stat.S_IRGRP | stat.S_IXGRP | \
81                         stat.S_IROTH | stat.S_IXOTH)
82        except IOError, err:
83            raise error.general('creating script: ' + name)
84        except:
85            if s is not None:
86                s.close()
87            raise
88        if s is not None:
89            s.close()
90
91class build:
92    """Build a package given a config file."""
93
94    def _name_(self, name):
95        #
96        # If on Windows use shorter names to keep the build paths.
97        #
98        if options.host_windows:
99            buildname = ''
100            add = True
101            for c in name:
102                if c == '-':
103                    add = True
104                elif add:
105                    buildname += c
106                    add = False
107            return buildname
108        else:
109            return name
110
111    def __init__(self, name, create_tar_files, opts, macros = None):
112        self.opts = opts
113        if macros is None:
114            self.macros = opts.defaults
115        else:
116            self.macros = macros
117        self.create_tar_files = create_tar_files
118        log.notice('config: ' + name)
119        self.config = config.file(name, opts, self.macros)
120        self.script = script()
121        self.macros['buildname'] = self._name_(self.macros['name'])
122
123    def rmdir(self, rmpath):
124        log.output('removing: %s' % (path.host(rmpath)))
125        if not self.opts.dry_run():
126            if path.exists(rmpath):
127                path.removeall(rmpath)
128
129    def mkdir(self, mkpath):
130        log.output('making dir: %s' % (path.host(mkpath)))
131        if not self.opts.dry_run():
132            path.mkdir(mkpath)
133
134    def source(self, package, source_tag):
135        #
136        # Scan the sources found in the config file for the one we are
137        # after. Infos or tags are lists. Merge in any macro defined
138        # sources as these may be overridden by user loaded macros.
139        #
140        sources = package.sources()
141        url = None
142        for s in sources:
143            tag = s[len('source'):]
144            if tag.isdigit():
145                if int(tag) == source_tag:
146                    url = sources[s][0]
147                    break
148        if url is None:
149            raise error.general('source tag not found: source%d' % (source_tag))
150        source = download.parse_url(url, '_sourcedir', self.config, self.opts)
151        download.get_file(source['url'], source['local'], self.opts, self.config)
152        if 'symlink' in source:
153            source['script'] = '%%{__ln_s} %s ${source_dir_%d}' % (source['symlink'], source_tag)
154        elif 'compressed' in source:
155            source['script'] = source['compressed'] + ' ' + \
156                source['local'] + ' | %{__tar_extract} -'
157        else:
158            source['script'] = '%{__tar_extract} ' + source['local']
159        return source
160
161    def patch(self, package, args):
162        #
163        # Scan the patches found in the config file for the one we are
164        # after. Infos or tags are lists.
165        #
166        patches = package.patches()
167        url = None
168        for p in patches:
169            if args[0][1:].lower() == p:
170                url = patches[p][0]
171                break
172        if url is None:
173            raise error.general('patch tag not found: %s' % (args[0]))
174        #
175        # Parse the URL first in the source builder's patch directory.
176        #
177        patch = download.parse_url(url, '_patchdir', self.config, self.opts)
178        #
179        # If not in the source builder package check the source directory.
180        #
181        if not path.exists(patch['local']):
182            patch = download.parse_url(url, '_patchdir', self.config, self.opts)
183        download.get_file(patch['url'], patch['local'], self.opts, self.config)
184        if 'compressed' in patch:
185            patch['script'] = patch['compressed'] + ' ' +  patch['local']
186        else:
187            patch['script'] = '%{__cat} ' + patch['local']
188        patch['script'] += ' | %{__patch} ' + ' '.join(args[1:])
189        self.script.append(self.config.expand(patch['script']))
190
191    def canadian_cross(self):
192        _host = self.config.expand('%{_host}')
193        _build = self.config.expand('%{_build}')
194        _target = self.config.expand('%{_target}')
195        return self.config.defined('%{allow_cxc}') and \
196            _host != _build and _host != _target
197
198    def setup(self, package, args):
199        log.output('prep: %s: %s' % (package.name(), ' '.join(args)))
200        opts, args = getopt.getopt(args[1:], 'qDcTn:b:a:')
201        source_tag = 0
202        quiet = False
203        unpack_default_source = True
204        unpack_before_chdir = True
205        delete_before_unpack = True
206        create_dir = False
207        name = None
208        for o in opts:
209            if o[0] == '-q':
210                quiet = True
211            elif o[0] == '-D':
212                delete_before_unpack = False
213            elif o[0] == '-c':
214                create_dir = True
215            elif o[0] == '-T':
216                unpack_default_source = False
217            elif o[0] == '-n':
218                name = o[1]
219            elif o[0] == '-b':
220                unpack_before_chdir = True
221                if not o[1].isdigit():
222                    raise error.general('setup -b source tag is not a number: %s' % (o[1]))
223                source_tag = int(o[1])
224            elif o[0] == '-a':
225                unpack_before_chdir = False
226                if not o[1].isdigit():
227                    raise error.general('setup -a source tag is not a number: %s' % (o[1]))
228                source_tag = int(o[1])
229        source0 = None
230        source = self.source(package, source_tag)
231        if name is None:
232            if source:
233                name = source['name']
234            else:
235                raise error.general('setup source tag not found: %d' % (source_tag))
236        name = self._name_(name)
237        self.script.append(self.config.expand('cd %{_builddir}'))
238        if delete_before_unpack:
239            self.script.append(self.config.expand('%{__rm} -rf ' + name))
240        if create_dir:
241            self.script.append(self.config.expand('%{__mkdir_p} ' + name))
242        #
243        # If -a? then change directory before unpacking.
244        #
245        if not unpack_before_chdir or create_dir:
246            self.script.append(self.config.expand('cd ' + name))
247        #
248        # Unpacking the source. Note, treated the same as -a0.
249        #
250        if unpack_default_source and source_tag != 0:
251            source0 = self.source(package, 0)
252            if source0 is None:
253                raise error.general('no setup source0 tag found')
254            self.script.append(self.config.expand(source0['script']))
255        self.script.append(self.config.expand(source['script']))
256        if unpack_before_chdir and not create_dir:
257            self.script.append(self.config.expand('cd ' + name))
258        self.script.append(self.config.expand('%{__setup_post}'))
259
260    def run(self, command, shell_opts = '', cwd = None):
261        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
262        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
263        log.output('run: ' + cmd)
264        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
265        if exit_code != 0:
266            log.output('shell cmd failed: %s' % (cmd))
267            raise error.general('building %s' % (self.macros['buildname']))
268
269    def builddir(self):
270        builddir = self.config.abspath('_builddir')
271        self.rmdir(builddir)
272        if not self.opts.dry_run():
273            self.mkdir(builddir)
274
275    def prep(self, package):
276        self.script.append('echo "==> %prep:"')
277        _prep = package.prep()
278        if _prep:
279            for l in _prep:
280                args = l.split()
281                if args[0] == '%setup':
282                    self.setup(package, args)
283                elif args[0].startswith('%patch'):
284                    self.patch(package, args)
285                else:
286                    self.script.append(' '.join(args))
287
288    def build(self, package):
289        self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
290        self.script.append('%s ${SB_BUILD_ROOT}' %
291                           (self.config.expand('%{__rmdir}')))
292        self.script.append('%s ${SB_BUILD_ROOT}' %
293                           (self.config.expand('%{__mkdir_p}')))
294        self.script.append('echo "==> %build:"')
295        _build = package.build()
296        if _build:
297            for l in _build:
298                self.script.append(l)
299
300    def install(self, package):
301        self.script.append('echo "==> %install:"')
302        _install = package.install()
303        if _install:
304            for l in _install:
305                args = l.split()
306                self.script.append(' '.join(args))
307
308    def files(self, package):
309        if self.create_tar_files \
310           and not self.macros.get('%{_disable_packaging'):
311            self.script.append('echo "==> %files:"')
312            inpath = path.abspath(self.config.expand('%{buildroot}'))
313            tardir = path.abspath(self.config.expand('%{_tardir}'))
314            self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
315            self.script.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))
316            self.script.append(self.config.expand('  cd ' + inpath))
317            tar = path.join(tardir, package.long_name() + '.tar.bz2')
318            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
319            self.script.append(cmd)
320            self.script.append(self.config.expand('  cd %{_builddir}'))
321            self.script.append('fi')
322
323    def clean(self, package):
324        self.script.append('echo "==> %clean:"')
325        _clean = package.clean()
326        if _clean is not None:
327            for l in _clean:
328                args = l.split()
329                self.script.append(' '.join(args))
330
331    def build_package(self, package):
332        if self.canadian_cross():
333            self.script.append('echo "==> Candian-cross build/target:"')
334            self.script.append('SB_CXC="yes"')
335        else:
336            self.script.append('SB_CXC="no"')
337        self.build(package)
338        self.install(package)
339        self.files(package)
340        if not self.opts.no_clean():
341            self.clean(package)
342
343    def cleanup(self):
344        package = self.main_package()
345        if not package.disabled() and not self.opts.no_clean():
346            buildroot = self.config.abspath('buildroot')
347            builddir = self.config.abspath('_builddir')
348            buildcxcdir = self.config.abspath('_buildcxcdir')
349            tmproot = self.config.abspath('_tmproot')
350            log.trace('cleanup: %s' % (buildroot))
351            self.rmdir(buildroot)
352            log.trace('cleanup: %s' % (builddir))
353            self.rmdir(builddir)
354            if self.canadian_cross():
355                log.trace('cleanup: %s' % (buildcxcdir))
356                self.rmdir(buildcxcdir)
357            log.trace('cleanup: %s' % (tmproot))
358            self.rmdir(tmproot)
359
360    def main_package(self):
361        packages = self.config.packages()
362        return packages['main']
363
364    def make(self):
365        package = self.main_package()
366        if package.disabled():
367            log.notice('package: nothing to build')
368        else:
369            name = package.name()
370            if self.canadian_cross():
371                log.notice('package: (Cxc) %s' % (name))
372            else:
373                log.notice('package: %s' % (name))
374                log.trace('---- macro maps %s' % ('-' * 55))
375                log.trace('%s' % (str(self.config.macros)))
376                log.trace('-' * 70)
377            self.script.reset()
378            self.script.append(self.config.expand('%{___build_template}'))
379            self.script.append('echo "=> ' + name + ':"')
380            self.prep(package)
381            self.build_package(package)
382            if not self.opts.dry_run():
383                self.builddir()
384                sn = path.join(self.config.expand('%{_builddir}'), 'doit')
385                log.output('write script: ' + sn)
386                self.script.write(sn)
387                if self.canadian_cross():
388                    log.notice('building: (Cxc) %s' % (name))
389                else:
390                    log.notice('building: %s' % (name))
391                self.run(sn)
392
393    def name(self):
394        packages = self.config.packages()
395        package = packages['main']
396        return package.name()
397
398    def disabled(self):
399        packages = self.config.packages()
400        package = packages['main']
401        return package.disabled()
402
403def get_configs(opts):
404
405    def _scan(_path, ext):
406        configs = []
407        for root, dirs, files in os.walk(_path):
408            prefix = root[len(_path) + 1:]
409            for file in files:
410                for e in ext:
411                    if file.endswith(e):
412                        configs += [path.join(prefix, file)]
413        return configs
414
415    configs = { 'paths': [], 'files': [] }
416    for cp in opts.defaults.expand('%{_configdir}').split(':'):
417        hcp = path.host(path.abspath(cp))
418        configs['paths'] += [hcp]
419        configs['files'] += _scan(hcp, ['.cfg', '.bset'])
420    configs['files'] = sorted(configs['files'])
421    return configs
422
423def find_config(config, configs):
424    config_root, config_ext = path.splitext(config)
425    if config_ext not in ['', '.bset', '.cfg']:
426        config_root = config
427        config_ext = ''
428    for c in configs['files']:
429        r, e = path.splitext(c)
430        if config_root == r:
431            if config_ext == '' or config_ext == e:
432                return c
433    return None
434
435def run(args):
436    ec = 0
437    opts = None
438    b = None
439    erheader = None
440    try:
441        optargs = { '--list-configs': 'List available configurations' }
442        opts = options.load(args, optargs)
443        log.notice('RTEMS Source Builder, Package Builder v%s' % (version.str()))
444        if not check.host_setup(opts):
445            if not opts.force():
446                raise error.general('host build environment is not set up' +
447                                    ' correctly (use --force to proceed)')
448            log.notice('warning: forcing build with known host setup problems')
449        if opts.get_arg('--list-configs'):
450            configs = get_configs(opts)
451            for p in configs['paths']:
452                print 'Examining: %s' % (os.path.relpath(p))
453                for c in configs['files']:
454                    if c.endswith('.cfg'):
455                        print '    %s' % (c)
456        else:
457            for config_file in opts.config_files():
458                b = build(config_file, True, opts)
459                b.make()
460                b = None
461    except error.general, gerr:
462        erheader = 'Build: %s' % (gerr)
463        log.notice(str(gerr))
464        log.stderr('Build FAILED')
465        ec = 1
466    except error.internal, ierr:
467        erheader = 'Build: %s' % (ierr)
468        log.notice(str(ierr))
469        log.stderr('Internal Build FAILED')
470        ec = 1
471    except error.exit, eerr:
472        pass
473    except KeyboardInterrupt:
474        log.notice('abort: user terminated')
475        ec = 1
476    if (ec != 0 and erheader and opts and b) or (opts and opts.dry_run()):
477        if opts.dry_run():
478            bname = 'dry-run'
479        else:
480            bname = b.name()
481        ereport.generate('rsb-report-%s.txt' % (bname), opts, erheader)
482    sys.exit(ec)
483
484if __name__ == "__main__":
485    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.