source: rtems-source-builder/source-builder/sb/build.py @ 467cace

4.104.114.95
Last change on this file since 467cace was 65d9457, checked in by Chris Johns <chrisj@…>, on 05/14/14 at 22:49:34

sb: Move error report generation to the build phase.

Generate a separate report of each package being built in a build set.
This creates a better list of faults in the case of nesting build sets
such as */rtems-all.
  • Property mode set to 100644
File size: 18.7 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import getopt
26import glob
27import os
28import shutil
29import stat
30import sys
31import urllib2
32import urlparse
33
34try:
35    import check
36    import config
37    import download
38    import error
39    import ereport
40    import execute
41    import log
42    import options
43    import path
44    import sources
45    import version
46except KeyboardInterrupt:
47    print 'abort: user terminated'
48    sys.exit(1)
49except:
50    print 'error: unknown application load error'
51    sys.exit(1)
52
53class script:
54    """Create and manage a shell script."""
55
56    def __init__(self):
57        self.reset()
58
59    def reset(self):
60        self.body = []
61        self.lc = 0
62
63    def append(self, text):
64        if type(text) is str:
65            text = text.splitlines()
66        if not log.quiet:
67            i = 0
68            for l in text:
69                i += 1
70                log.output('script:%3d: %s' % (self.lc + i, l))
71        self.lc += len(text)
72        self.body.extend(text)
73
74    def write(self, name, check_for_errors = False):
75        s = None
76        try:
77            s = open(path.host(name), 'w')
78            s.write('\n'.join(self.body))
79            s.close()
80            os.chmod(path.host(name), stat.S_IRWXU | \
81                         stat.S_IRGRP | stat.S_IXGRP | \
82                         stat.S_IROTH | stat.S_IXOTH)
83        except IOError, err:
84            raise error.general('creating script: ' + name)
85        except:
86            if s is not None:
87                s.close()
88            raise
89        if s is not None:
90            s.close()
91
92class build:
93    """Build a package given a config file."""
94
95    def _name_(self, name):
96        #
97        # If on Windows use shorter names to keep the build paths.
98        #
99        if options.host_windows:
100            buildname = ''
101            add = True
102            for c in name:
103                if c == '-':
104                    add = True
105                elif add:
106                    buildname += c
107                    add = False
108            return buildname
109        else:
110            return name
111
112    def _generate_report_(self, header):
113        ereport.generate('rsb-report-%s.txt' % self.macros['name'], self.opts, header)
114
115    def __init__(self, name, create_tar_files, opts, macros = None):
116        self.opts = opts
117        if macros is None:
118            self.macros = opts.defaults
119        else:
120            self.macros = macros
121        self.create_tar_files = create_tar_files
122        log.notice('config: ' + name)
123        self.config = config.file(name, opts, self.macros)
124        self.script = script()
125        self.macros['buildname'] = self._name_(self.macros['name'])
126
127    def rmdir(self, rmpath):
128        log.output('removing: %s' % (path.host(rmpath)))
129        if not self.opts.dry_run():
130            if path.exists(rmpath):
131                path.removeall(rmpath)
132
133    def mkdir(self, mkpath):
134        log.output('making dir: %s' % (path.host(mkpath)))
135        if not self.opts.dry_run():
136            path.mkdir(mkpath)
137
138    def canadian_cross(self):
139        _host = self.config.expand('%{_host}')
140        _build = self.config.expand('%{_build}')
141        _target = self.config.expand('%{_target}')
142        return self.config.defined('%{allow_cxc}') and \
143            _host != _build and _host != _target
144
145    def source(self, name):
146        #
147        # Return the list of sources. Merge in any macro defined sources as
148        # these may be overridden by user loaded macros.
149        #
150        _map = 'source-%s' % (name)
151        src_keys = self.macros.map_keys(_map)
152        if len(src_keys) == 0:
153            raise error.general('no source set: %s (%s)' % (name, _map))
154        srcs = []
155        for s in src_keys:
156            sm = self.macros.get(s, globals = False, maps = _map)
157            if sm is None:
158                raise error.internal('source macro not found: %s in %s (%s)' % \
159                                         (s, name, _map))
160            url = self.config.expand(sm[2])
161            src = download.parse_url(url, '_sourcedir', self.config, self.opts)
162            download.get_file(src['url'], src['local'], self.opts, self.config)
163            if 'symlink' in src:
164                src['script'] = '%%{__ln_s} %s ${source_dir_%s}' % (src['symlink'], name)
165            elif 'compressed' in src:
166                #
167                # Zip files unpack as well so do not use tar.
168                #
169                src['script'] = '%s %s' % (src['compressed'], src['local'])
170                if src['compressed-type'] != 'zip':
171                    src['script'] += ' | %{__tar_extract} -'
172            else:
173                src['script'] = '%%{__tar_extract} %s' % (src['local'])
174            srcs += [src]
175        return srcs
176
177    def source_setup(self, package, args):
178        log.output('source setup: %s: %s' % (package.name(), ' '.join(args)))
179        setup_name = args[1]
180        args = args[1:]
181        try:
182            opts, args = getopt.getopt(args[1:], 'qDcn:b:a:')
183        except getopt.GetoptError, ge:
184            raise error.general('source setup error: %s' % str(ge))
185        quiet = False
186        unpack_before_chdir = True
187        delete_before_unpack = True
188        create_dir = False
189        deleted_dir = False
190        created_dir = False
191        changed_dir = False
192        opt_name = None
193        for o in opts:
194            if o[0] == '-q':
195                quiet = True
196            elif o[0] == '-D':
197                delete_before_unpack = False
198            elif o[0] == '-c':
199                create_dir = True
200            elif o[0] == '-n':
201                opt_name = o[1]
202            elif o[0] == '-b':
203                unpack_before_chdir = True
204            elif o[0] == '-a':
205                unpack_before_chdir = False
206        name = None
207        for source in self.source(setup_name):
208            if name is None:
209                if opt_name is None:
210                    if source:
211                        opt_name = source['name']
212                    else:
213                        raise error.general('setup source tag not found: %d' % (source_tag))
214                else:
215                    name = opt_name
216                name = self._name_(name)
217            self.script.append(self.config.expand('cd %{_builddir}'))
218            if not deleted_dir and  delete_before_unpack:
219                self.script.append(self.config.expand('%{__rm} -rf ' + name))
220                deleted_dir = True
221            if not created_dir and create_dir:
222                self.script.append(self.config.expand('%{__mkdir_p} ' + name))
223                created_dir = True
224            if not changed_dir and (not unpack_before_chdir or create_dir):
225                self.script.append(self.config.expand('cd ' + name))
226                changed_dir = True
227            self.script.append(self.config.expand(source['script']))
228        if not changed_dir and (unpack_before_chdir and not create_dir):
229            self.script.append(self.config.expand('cd ' + name))
230            changed_dir = True
231        self.script.append(self.config.expand('%{__setup_post}'))
232
233    def patch_setup(self, package, args):
234        name = args[1]
235        args = args[2:]
236        _map = 'patch-%s' % (name)
237        default_opts = ' '.join(args)
238        patch_keys = self.macros.map_keys(_map)
239        patches = []
240        for p in patch_keys:
241            pm = self.macros.get(p, globals = False, maps = _map)
242            if pm is None:
243                raise error.internal('patch macro not found: %s in %s (%s)' % \
244                                         (p, name, _map))
245            opts = []
246            url = []
247            for pp in pm[2].split():
248                if len(url) == 0 and pp[0] == '-':
249                    opts += [pp]
250                else:
251                    url += [pp]
252            if len(url) == 0:
253                raise error.general('patch URL not found: %s' % (' '.join(args)))
254            if len(opts) == 0:
255                opts = default_opts
256            else:
257                opts = ' '.join(opts)
258            opts = self.config.expand(opts)
259            url = self.config.expand(' '.join(url))
260            #
261            # Parse the URL first in the source builder's patch directory.
262            #
263            patch = download.parse_url(url, '_patchdir', self.config, self.opts)
264            #
265            # If not in the source builder package check the source directory.
266            #
267            if not path.exists(patch['local']):
268                patch = download.parse_url(url, '_patchdir', self.config, self.opts)
269            download.get_file(patch['url'], patch['local'], self.opts, self.config)
270            if 'compressed' in patch:
271                patch['script'] = patch['compressed'] + ' ' +  patch['local']
272            else:
273                patch['script'] = '%{__cat} ' + patch['local']
274            patch['script'] += ' | %%{__patch} %s' % (opts)
275            self.script.append(self.config.expand(patch['script']))
276
277    def run(self, command, shell_opts = '', cwd = None):
278        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
279        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
280        log.output('run: ' + cmd)
281        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
282        if exit_code != 0:
283            log.output('shell cmd failed: %s' % (cmd))
284            raise error.general('building %s' % (self.macros['buildname']))
285
286    def builddir(self):
287        builddir = self.config.abspath('_builddir')
288        self.rmdir(builddir)
289        if not self.opts.dry_run():
290            self.mkdir(builddir)
291
292    def prep(self, package):
293        self.script.append('echo "==> %prep:"')
294        _prep = package.prep()
295        if _prep:
296            for l in _prep:
297                args = l.split()
298                if len(args):
299                    if args[0] == '%setup':
300                        if len(args) == 1:
301                            raise error.general('invalid %%setup directive: %s' % (' '.join(args)))
302                        if args[1] == 'source':
303                            self.source_setup(package, args[1:])
304                        elif args[1] == 'patch':
305                            self.patch_setup(package, args[1:])
306                    elif args[0].startswith('%patch'):
307                        self.patch(package, args)
308                    else:
309                        self.script.append(' '.join(args))
310
311    def build(self, package):
312        self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
313        self.script.append('%s ${SB_BUILD_ROOT}' %
314                           (self.config.expand('%{__rmdir}')))
315        self.script.append('%s ${SB_BUILD_ROOT}' %
316                           (self.config.expand('%{__mkdir_p}')))
317        self.script.append('echo "==> %build:"')
318        _build = package.build()
319        if _build:
320            for l in _build:
321                self.script.append(l)
322
323    def install(self, package):
324        self.script.append('echo "==> %install:"')
325        _install = package.install()
326        if _install:
327            for l in _install:
328                args = l.split()
329                self.script.append(' '.join(args))
330
331    def files(self, package):
332        if self.create_tar_files \
333           and not self.macros.get('%{_disable_packaging'):
334            self.script.append('echo "==> %files:"')
335            inpath = path.abspath(self.config.expand('%{buildroot}'))
336            tardir = path.abspath(self.config.expand('%{_tardir}'))
337            self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
338            self.script.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))
339            self.script.append(self.config.expand('  cd ' + inpath))
340            tar = path.join(tardir, package.long_name() + '.tar.bz2')
341            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
342            self.script.append(cmd)
343            self.script.append(self.config.expand('  cd %{_builddir}'))
344            self.script.append('fi')
345
346    def clean(self, package):
347        self.script.append('echo "==> %clean:"')
348        _clean = package.clean()
349        if _clean is not None:
350            for l in _clean:
351                args = l.split()
352                self.script.append(' '.join(args))
353
354    def build_package(self, package):
355        if self.canadian_cross():
356            self.script.append('echo "==> Candian-cross build/target:"')
357            self.script.append('SB_CXC="yes"')
358        else:
359            self.script.append('SB_CXC="no"')
360        self.build(package)
361        self.install(package)
362        self.files(package)
363        if not self.opts.no_clean():
364            self.clean(package)
365
366    def cleanup(self):
367        package = self.main_package()
368        if not package.disabled() and not self.opts.no_clean():
369            buildroot = self.config.abspath('buildroot')
370            builddir = self.config.abspath('_builddir')
371            buildcxcdir = self.config.abspath('_buildcxcdir')
372            tmproot = self.config.abspath('_tmproot')
373            log.trace('cleanup: %s' % (buildroot))
374            self.rmdir(buildroot)
375            log.trace('cleanup: %s' % (builddir))
376            self.rmdir(builddir)
377            if self.canadian_cross():
378                log.trace('cleanup: %s' % (buildcxcdir))
379                self.rmdir(buildcxcdir)
380            log.trace('cleanup: %s' % (tmproot))
381            self.rmdir(tmproot)
382
383    def main_package(self):
384        packages = self.config.packages()
385        return packages['main']
386
387    def make(self):
388        package = self.main_package()
389        if package.disabled():
390            log.notice('package: nothing to build')
391        else:
392            try:
393                name = package.name()
394                if self.canadian_cross():
395                    log.notice('package: (Cxc) %s' % (name))
396                else:
397                    log.notice('package: %s' % (name))
398                    log.trace('---- macro maps %s' % ('-' * 55))
399                    log.trace('%s' % (str(self.config.macros)))
400                    log.trace('-' * 70)
401                self.script.reset()
402                self.script.append(self.config.expand('%{___build_template}'))
403                self.script.append('echo "=> ' + name + ':"')
404                self.prep(package)
405                self.build_package(package)
406                if not self.opts.dry_run():
407                    self.builddir()
408                    sn = path.join(self.config.expand('%{_builddir}'), 'doit')
409                    log.output('write script: ' + sn)
410                    self.script.write(sn)
411                    if self.canadian_cross():
412                        log.notice('building: (Cxc) %s' % (name))
413                    else:
414                        log.notice('building: %s' % (name))
415                    self.run(sn)
416            except error.general, gerr:
417                log.notice(str(gerr))
418                log.stderr('Build FAILED')
419                self._generate_report_('Build: %s' % (gerr))
420                raise
421            except error.internal, ierr:
422                log.notice(str(ierr))
423                log.stderr('Internal Build FAILED')
424                self._generate_report_('Build: %s' % (ierr))
425                raise
426            except:
427                raise
428            if self.opts.dry_run():
429                self._generate_report_('Build: dry run')
430
431    def name(self):
432        packages = self.config.packages()
433        package = packages['main']
434        return package.name()
435
436    def disabled(self):
437        packages = self.config.packages()
438        package = packages['main']
439        return package.disabled()
440
441def get_configs(opts):
442
443    def _scan(_path, ext):
444        configs = []
445        for root, dirs, files in os.walk(_path):
446            prefix = root[len(_path) + 1:]
447            for file in files:
448                for e in ext:
449                    if file.endswith(e):
450                        configs += [path.join(prefix, file)]
451        return configs
452
453    configs = { 'paths': [], 'files': [] }
454    for cp in opts.defaults.expand('%{_configdir}').split(':'):
455        hcp = path.host(path.abspath(cp))
456        configs['paths'] += [hcp]
457        configs['files'] += _scan(hcp, ['.cfg', '.bset'])
458    configs['files'] = sorted(configs['files'])
459    return configs
460
461def find_config(config, configs):
462    config_root, config_ext = path.splitext(config)
463    if config_ext not in ['', '.bset', '.cfg']:
464        config_root = config
465        config_ext = ''
466    for c in configs['files']:
467        r, e = path.splitext(c)
468        if config_root == r:
469            if config_ext == '' or config_ext == e:
470                return c
471    return None
472
473def run(args):
474    ec = 0
475    try:
476        optargs = { '--list-configs': 'List available configurations' }
477        opts = options.load(args, optargs)
478        log.notice('RTEMS Source Builder, Package Builder v%s' % (version.str()))
479        if not check.host_setup(opts):
480            if not opts.force():
481                raise error.general('host build environment is not set up' +
482                                    ' correctly (use --force to proceed)')
483            log.notice('warning: forcing build with known host setup problems')
484        if opts.get_arg('--list-configs'):
485            configs = get_configs(opts)
486            for p in configs['paths']:
487                print 'Examining: %s' % (os.path.relpath(p))
488                for c in configs['files']:
489                    if c.endswith('.cfg'):
490                        print '    %s' % (c)
491        else:
492            for config_file in opts.config_files():
493                b = build(config_file, True, opts)
494                b.make()
495                b = None
496    except error.general, gerr:
497        log.stderr('Build FAILED')
498        ec = 1
499    except error.internal, ierr:
500        log.stderr('Internal Build FAILED')
501        ec = 1
502    except error.exit, eerr:
503        pass
504    except KeyboardInterrupt:
505        log.notice('abort: user terminated')
506        ec = 1
507    sys.exit(ec)
508
509if __name__ == "__main__":
510    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.