source: rtems-source-builder/source-builder/sb/build.py @ 858e43b

4.11
Last change on this file since 858e43b was 858e43b, checked in by Chris Johns <chrisj@…>, on 03/14/16 at 04:19:29

sb: Fix build's --with-error-report handling.

This reflects changes in the way the --with/--without options are handled.

Updates #2526.

  • Property mode set to 100644
File size: 21.2 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25from __future__ import print_function
26
27import copy
28import getopt
29import glob
30import os
31import shutil
32import stat
33import sys
34
35try:
36    import check
37    import config
38    import download
39    import error
40    import ereport
41    import execute
42    import log
43    import options
44    import path
45    import sources
46    import version
47except KeyboardInterrupt:
48    print('abort: user terminated')
49    sys.exit(1)
50except:
51    print('error: unknown application load error')
52    sys.exit(1)
53
54class script:
55    """Create and manage a shell script."""
56
57    def __init__(self):
58        self.reset()
59
60    def reset(self):
61        self.body = []
62        self.lc = 0
63
64    def append(self, text):
65        if type(text) is str:
66            text = text.splitlines()
67        if not log.quiet:
68            i = 0
69            for l in text:
70                i += 1
71                log.output('script:%3d: %s' % (self.lc + i, l))
72        self.lc += len(text)
73        self.body.extend(text)
74
75    def write(self, name, check_for_errors = False):
76        s = None
77        try:
78            s = open(path.host(name), 'w')
79            s.write('\n'.join(self.body))
80            s.close()
81            os.chmod(path.host(name), stat.S_IRWXU | \
82                         stat.S_IRGRP | stat.S_IXGRP | \
83                         stat.S_IROTH | stat.S_IXOTH)
84        except IOError as err:
85            raise error.general('creating script: ' + name)
86        except:
87            if s is not None:
88                s.close()
89            raise
90        if s is not None:
91            s.close()
92
93class build:
94    """Build a package given a config file."""
95
96    def _name_(self, name):
97        #
98        # If on Windows use shorter names to keep the build paths.
99        #
100        if options.host_windows:
101            buildname = ''
102            add = True
103            for c in name:
104                if c == '-':
105                    add = True
106                elif add:
107                    buildname += c
108                    add = False
109            return buildname
110        else:
111            return name
112
113    def _generate_report_(self, header, footer = None):
114        label, result = self.opts.with_arg('error-report')
115        if (label.startswith('without') and result != 'yes') or \
116           (label.startswith('with') and result != 'no'):
117            ereport.generate('rsb-report-%s.txt' % self.macros['name'],
118                             self.opts, header, footer)
119
120    def __init__(self, name, create_tar_files, opts, macros = None):
121        try:
122            self.opts = opts
123            self.init_name = name
124            self.init_macros = macros
125            self.config = None
126            self.create_tar_files = create_tar_files
127            log.notice('config: ' + name)
128            self.set_macros(macros)
129            self.config = config.file(name, opts, self.macros)
130            self.script = script()
131            self.macros['buildname'] = self._name_(self.macros['name'])
132        except error.general as gerr:
133            log.notice(str(gerr))
134            log.stderr('Build FAILED')
135            raise
136        except error.internal as ierr:
137            log.notice(str(ierr))
138            log.stderr('Internal Build FAILED')
139            raise
140        except:
141            raise
142
143    def copy_init_macros(self):
144        return copy.copy(self.init_macros)
145
146    def copy_macros(self):
147        return copy.copy(self.macros)
148
149    def set_macros(self, macros):
150        if macros is None:
151            self.macros = copy.copy(self.opts.defaults)
152        else:
153            self.macros = copy.copy(macros)
154        if self.config:
155            self.config.set_macros(self.macros)
156
157    def rmdir(self, rmpath):
158        log.output('removing: %s' % (path.host(rmpath)))
159        if not self.opts.dry_run():
160            if path.exists(rmpath):
161                path.removeall(rmpath)
162
163    def mkdir(self, mkpath):
164        log.output('making dir: %s' % (path.host(mkpath)))
165        if not self.opts.dry_run():
166            path.mkdir(mkpath)
167
168    def canadian_cross(self):
169        _host = self.config.expand('%{_host}')
170        _build = self.config.expand('%{_build}')
171        _target = self.config.expand('%{_target}')
172        _allowed = self.config.defined('%{allow_cxc}')
173        if len(_host) and len(_build) and (_target) and \
174           _allowed and _host != _build and _host != _target:
175            return True
176        return False
177
178    def installable(self):
179        _host = self.config.expand('%{_host}')
180        _build = self.config.expand('%{_build}')
181        _canadian_cross = self.canadian_cross()
182        if self.macros.get('_disable_installing') and \
183           self.config.expand('%{_disable_installing}') == 'yes':
184            _disable_installing = True
185        else:
186            _disable_installing = False
187        _no_install = self.opts.no_install()
188        log.trace('_build: installable: host=%s build=%s ' \
189                  'no-install=%r Cxc=%r disable_installing=%r disabled=%r' % \
190                  (_host, _build, _no_install, _canadian_cross, _disable_installing, \
191                   self.disabled()))
192        return len(_host) and len(_build) and \
193            not self.disabled() and \
194            not _disable_installing and \
195            not _canadian_cross
196
197    def source(self, name):
198        #
199        # Return the list of sources. Merge in any macro defined sources as
200        # these may be overridden by user loaded macros.
201        #
202        _map = 'source-%s' % (name)
203        src_keys = [s for s in self.macros.map_keys(_map) if s != 'setup']
204        if len(src_keys) == 0:
205            raise error.general('no source set: %s (%s)' % (name, _map))
206        srcs = []
207        for s in src_keys:
208            sm = self.macros.get(s, globals = False, maps = _map)
209            if sm is None:
210                raise error.internal('source macro not found: %s in %s (%s)' % \
211                                         (s, name, _map))
212            url = self.config.expand(sm[2])
213            src = download.parse_url(url, '_sourcedir', self.config, self.opts)
214            download.get_file(src['url'], src['local'], self.opts, self.config)
215            if 'symlink' in src:
216                sname = name.replace('-', '_')
217                src['script'] = '%%{__ln_s} %s ${source_dir_%s}' % (src['symlink'], sname)
218            elif 'compressed' in src:
219                #
220                # Zip files unpack as well so do not use tar.
221                #
222                src['script'] = '%s %s' % (src['compressed'], src['local'])
223                if src['compressed-type'] != 'zip':
224                    src['script'] += ' | %{__tar_extract} -'
225            else:
226                src['script'] = '%%{__tar_extract} %s' % (src['local'])
227            srcs += [src]
228        return srcs
229
230    def source_setup(self, package, args):
231        log.output('source setup: %s: %s' % (package.name(), ' '.join(args)))
232        setup_name = args[1]
233        args = args[1:]
234        try:
235            opts, args = getopt.getopt(args[1:], 'qDcn:ba')
236        except getopt.GetoptError as ge:
237            raise error.general('source setup error: %s' % str(ge))
238        quiet = False
239        unpack_before_chdir = True
240        delete_before_unpack = True
241        create_dir = False
242        deleted_dir = False
243        created_dir = False
244        changed_dir = False
245        opt_name = None
246        for o in opts:
247            if o[0] == '-q':
248                quiet = True
249            elif o[0] == '-D':
250                delete_before_unpack = False
251            elif o[0] == '-c':
252                create_dir = True
253            elif o[0] == '-n':
254                opt_name = o[1]
255            elif o[0] == '-b':
256                unpack_before_chdir = True
257            elif o[0] == '-a':
258                unpack_before_chdir = False
259        name = None
260        for source in self.source(setup_name):
261            if name is None:
262                if opt_name is None:
263                    if source:
264                        opt_name = source['name']
265                    else:
266                        raise error.general('setup source tag not found: %d' % (source_tag))
267                else:
268                    name = opt_name
269            self.script.append(self.config.expand('cd %{_builddir}'))
270            if not deleted_dir and  delete_before_unpack:
271                self.script.append(self.config.expand('%{__rm} -rf ' + name))
272                deleted_dir = True
273            if not created_dir and create_dir:
274                self.script.append(self.config.expand('%{__mkdir_p} ' + name))
275                created_dir = True
276            if not changed_dir and (not unpack_before_chdir or create_dir):
277                self.script.append(self.config.expand('cd ' + name))
278                changed_dir = True
279            self.script.append(self.config.expand(source['script']))
280        if not changed_dir and (unpack_before_chdir and not create_dir):
281            self.script.append(self.config.expand('cd ' + name))
282            changed_dir = True
283        self.script.append(self.config.expand('%{__setup_post}'))
284
285    def patch_setup(self, package, args):
286        name = args[1]
287        args = args[2:]
288        _map = 'patch-%s' % (name)
289        default_opts = ' '.join(args)
290        patch_keys = [p for p in self.macros.map_keys(_map) if p != 'setup']
291        patches = []
292        for p in patch_keys:
293            pm = self.macros.get(p, globals = False, maps = _map)
294            if pm is None:
295                raise error.internal('patch macro not found: %s in %s (%s)' % \
296                                         (p, name, _map))
297            opts = []
298            url = []
299            for pp in pm[2].split():
300                if len(url) == 0 and pp[0] == '-':
301                    opts += [pp]
302                else:
303                    url += [pp]
304            if len(url) == 0:
305                raise error.general('patch URL not found: %s' % (' '.join(args)))
306            if len(opts) == 0:
307                opts = default_opts
308            else:
309                opts = ' '.join(opts)
310            opts = self.config.expand(opts)
311            url = self.config.expand(' '.join(url))
312            #
313            # Parse the URL first in the source builder's patch directory.
314            #
315            patch = download.parse_url(url, '_patchdir', self.config, self.opts)
316            #
317            # If not in the source builder package check the source directory.
318            #
319            if not path.exists(patch['local']):
320                patch = download.parse_url(url, '_patchdir', self.config, self.opts)
321            download.get_file(patch['url'], patch['local'], self.opts, self.config)
322            if 'compressed' in patch:
323                patch['script'] = patch['compressed'] + ' ' +  patch['local']
324            else:
325                patch['script'] = '%{__cat} ' + patch['local']
326            patch['script'] += ' | %%{__patch} %s' % (opts)
327            self.script.append(self.config.expand(patch['script']))
328
329    def run(self, command, shell_opts = '', cwd = None):
330        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
331        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
332        log.output('run: ' + cmd)
333        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
334        if exit_code != 0:
335            log.output('shell cmd failed: %s' % (cmd))
336            raise error.general('building %s' % (self.macros['buildname']))
337
338    def builddir(self):
339        builddir = self.config.abspath('_builddir')
340        self.rmdir(builddir)
341        if not self.opts.dry_run():
342            self.mkdir(builddir)
343
344    def prep(self, package):
345        self.script.append('echo "==> %prep:"')
346        _prep = package.prep()
347        if _prep:
348            for l in _prep:
349                args = l.split()
350                if len(args):
351                    def err(msg):
352                        raise error.general('%s: %s' % (package, msg))
353                    if args[0] == '%setup':
354                        if len(args) == 1:
355                            raise error.general('invalid %%setup directive: %s' % \
356                                                (' '.join(args)))
357                        if args[1] == 'source':
358                            self.source_setup(package, args[1:])
359                        elif args[1] == 'patch':
360                            self.patch_setup(package, args[1:])
361                    elif args[0] in ['%patch', '%source']:
362                        sources.process(args[0][1:], args[1:], self.macros, err)
363                    elif args[0] == '%hash':
364                        sources.hash(args[1:], self.macros, err)
365                        self.hash(package, args)
366                    else:
367                        self.script.append(' '.join(args))
368
369    def build(self, package):
370        self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
371        self.script.append('%s ${SB_BUILD_ROOT}' %
372                           (self.config.expand('%{__rmdir}')))
373        self.script.append('%s ${SB_BUILD_ROOT}' %
374                           (self.config.expand('%{__mkdir_p}')))
375        self.script.append('echo "==> %build:"')
376        _build = package.build()
377        if _build:
378            for l in _build:
379                self.script.append(l)
380
381    def install(self, package):
382        self.script.append('echo "==> %install:"')
383        _install = package.install()
384        if _install:
385            for l in _install:
386                args = l.split()
387                self.script.append(' '.join(args))
388
389    def files(self, package):
390        if self.create_tar_files \
391           and not self.macros.get('%{_disable_packaging'):
392            self.script.append('echo "==> %files:"')
393            inpath = path.abspath(self.config.expand('%{buildroot}'))
394            tardir = path.abspath(self.config.expand('%{_tardir}'))
395            self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
396            self.script.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))
397            self.script.append(self.config.expand('  cd ' + inpath))
398            tar = path.join(tardir, package.long_name() + '.tar.bz2')
399            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
400            self.script.append(cmd)
401            self.script.append(self.config.expand('  cd %{_builddir}'))
402            self.script.append('fi')
403
404    def clean(self, package):
405        self.script.append('echo "==> %clean:"')
406        _clean = package.clean()
407        if _clean is not None:
408            for l in _clean:
409                args = l.split()
410                self.script.append(' '.join(args))
411
412    def build_package(self, package):
413        if self.canadian_cross():
414            if not self.config.defined('%{allow_cxc}'):
415                raise error.general('Canadian Cross is not allowed')
416            self.script.append('echo "==> Candian-cross build/target:"')
417            self.script.append('SB_CXC="yes"')
418        else:
419            self.script.append('SB_CXC="no"')
420        self.build(package)
421        self.install(package)
422        self.files(package)
423        if not self.opts.no_clean():
424            self.clean(package)
425
426    def cleanup(self):
427        package = self.main_package()
428        if not package.disabled() and not self.opts.no_clean():
429            buildroot = self.config.abspath('buildroot')
430            builddir = self.config.abspath('_builddir')
431            buildcxcdir = self.config.abspath('_buildcxcdir')
432            tmproot = self.config.abspath('_tmproot')
433            log.trace('cleanup: %s' % (buildroot))
434            self.rmdir(buildroot)
435            log.trace('cleanup: %s' % (builddir))
436            self.rmdir(builddir)
437            if self.canadian_cross():
438                log.trace('cleanup: %s' % (buildcxcdir))
439                self.rmdir(buildcxcdir)
440            log.trace('cleanup: %s' % (tmproot))
441            self.rmdir(tmproot)
442
443    def main_package(self):
444        packages = self.config.packages()
445        return packages['main']
446
447    def reload(self):
448        self.config.load(self.init_name)
449
450    def make(self):
451        package = self.main_package()
452        if package.disabled():
453            log.notice('package: nothing to build')
454        else:
455            try:
456                name = package.name()
457                if self.canadian_cross():
458                    cxc_label = '(Cxc) '
459                else:
460                    cxc_label = ''
461                log.notice('package: %s%s' % (cxc_label, name))
462                log.trace('---- macro maps %s' % ('-' * 55))
463                log.trace('%s' % (str(self.config.macros)))
464                log.trace('-' * 70)
465                self.script.reset()
466                self.script.append(self.config.expand('%{___build_template}'))
467                self.script.append('echo "=> ' + name + ':"')
468                self.prep(package)
469                self.build_package(package)
470                if not self.opts.dry_run():
471                    self.builddir()
472                    sn = path.join(self.config.expand('%{_builddir}'), 'doit')
473                    log.output('write script: ' + sn)
474                    self.script.write(sn)
475                    log.notice('building: %s%s' % (cxc_label, name))
476                    self.run(sn)
477            except error.general as gerr:
478                log.notice(str(gerr))
479                log.stderr('Build FAILED')
480                self._generate_report_('Build: %s' % (gerr))
481                raise
482            except error.internal as ierr:
483                log.notice(str(ierr))
484                log.stderr('Internal Build FAILED')
485                self._generate_report_('Build: %s' % (ierr))
486                raise
487            except:
488                raise
489            if self.opts.dry_run():
490                self._generate_report_('Build: dry run (no actual error)',
491                                       'Build: dry run (no actual error)')
492
493    def name(self):
494        packages = self.config.packages()
495        package = packages['main']
496        return package.name()
497
498    def disabled(self):
499        packages = self.config.packages()
500        package = packages['main']
501        return package.disabled()
502
503def get_configs(opts):
504
505    def _scan(_path, ext):
506        configs = []
507        for root, dirs, files in os.walk(_path):
508            prefix = root[len(_path) + 1:]
509            for file in files:
510                for e in ext:
511                    if file.endswith(e):
512                        configs += [path.join(prefix, file)]
513        return configs
514
515    configs = { 'paths': [], 'files': [] }
516    for cp in opts.defaults.expand('%{_configdir}').split(':'):
517        hcp = path.host(path.abspath(cp))
518        configs['paths'] += [hcp]
519        configs['files'] += _scan(hcp, ['.cfg', '.bset'])
520    configs['files'] = sorted(set(configs['files']))
521    return configs
522
523def find_config(config, configs):
524    config_root, config_ext = path.splitext(config)
525    if config_ext not in ['', '.bset', '.cfg']:
526        config_root = config
527        config_ext = ''
528    for c in configs['files']:
529        r, e = path.splitext(c)
530        if config_root == r:
531            if config_ext == '' or config_ext == e:
532                return c
533    return None
534
535def run(args):
536    ec = 0
537    try:
538        optargs = { '--list-configs': 'List available configurations' }
539        opts = options.load(args, optargs)
540        log.notice('RTEMS Source Builder, Package Builder, %s' % (version.str()))
541        opts.log_info()
542        if not check.host_setup(opts):
543            if not opts.force():
544                raise error.general('host build environment is not set up' +
545                                    ' correctly (use --force to proceed)')
546            log.notice('warning: forcing build with known host setup problems')
547        if opts.get_arg('--list-configs'):
548            configs = get_configs(opts)
549            for p in configs['paths']:
550                print('Examining: %s' % (os.path.relpath(p)))
551                for c in configs['files']:
552                    if c.endswith('.cfg'):
553                        print('    %s' % (c))
554        else:
555            for config_file in opts.config_files():
556                b = build(config_file, True, opts)
557                b.make()
558                b = None
559    except error.general as gerr:
560        log.stderr('Build FAILED')
561        ec = 1
562    except error.internal as ierr:
563        log.stderr('Internal Build FAILED')
564        ec = 1
565    except error.exit as eerr:
566        pass
567    except KeyboardInterrupt:
568        log.notice('abort: user terminated')
569        ec = 1
570    sys.exit(ec)
571
572if __name__ == "__main__":
573    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.