source: rtems-source-builder/source-builder/sb/build.py @ 6444d58

4.104.95
Last change on this file since 6444d58 was 6444d58, checked in by Chris Johns <chrisj@…>, on 07/20/15 at 03:49:42

Canandian Cross Compiling and RTEMS 3rd party package building Fixes.

The change fixes installing for RTEMS 3rd Party packages where the
RSB considered them Canadian Cross Compiling (Cxc). Fixing the
Cxc issue broke real Cxc builds. The change corrects the issue of
macros being changed in the Cxc and the prep data not being udpated.
The configuration is loaded again after the updated macros. The
macros are also copied and restored to ensure a clean stable base.

The change also introduces --rtems-tools and --rtems-bsp to align
the command line with the waf configure process or RTEMS application.
  • Property mode set to 100644
File size: 21.0 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import copy
26import getopt
27import glob
28import os
29import shutil
30import stat
31import sys
32import urllib2
33import urlparse
34
35try:
36    import check
37    import config
38    import download
39    import error
40    import ereport
41    import execute
42    import log
43    import options
44    import path
45    import sources
46    import version
47except KeyboardInterrupt:
48    print 'abort: user terminated'
49    sys.exit(1)
50except:
51    print 'error: unknown application load error'
52    sys.exit(1)
53
54class script:
55    """Create and manage a shell script."""
56
57    def __init__(self):
58        self.reset()
59
60    def reset(self):
61        self.body = []
62        self.lc = 0
63
64    def append(self, text):
65        if type(text) is str:
66            text = text.splitlines()
67        if not log.quiet:
68            i = 0
69            for l in text:
70                i += 1
71                log.output('script:%3d: %s' % (self.lc + i, l))
72        self.lc += len(text)
73        self.body.extend(text)
74
75    def write(self, name, check_for_errors = False):
76        s = None
77        try:
78            s = open(path.host(name), 'w')
79            s.write('\n'.join(self.body))
80            s.close()
81            os.chmod(path.host(name), stat.S_IRWXU | \
82                         stat.S_IRGRP | stat.S_IXGRP | \
83                         stat.S_IROTH | stat.S_IXOTH)
84        except IOError, err:
85            raise error.general('creating script: ' + name)
86        except:
87            if s is not None:
88                s.close()
89            raise
90        if s is not None:
91            s.close()
92
93class build:
94    """Build a package given a config file."""
95
96    def _name_(self, name):
97        #
98        # If on Windows use shorter names to keep the build paths.
99        #
100        if options.host_windows:
101            buildname = ''
102            add = True
103            for c in name:
104                if c == '-':
105                    add = True
106                elif add:
107                    buildname += c
108                    add = False
109            return buildname
110        else:
111            return name
112
113    def _generate_report_(self, header, footer = None):
114        ereport.generate('rsb-report-%s.txt' % self.macros['name'],
115                         self.opts, header, footer)
116
117    def __init__(self, name, create_tar_files, opts, macros = None):
118        try:
119            self.opts = opts
120            self.init_name = name
121            self.init_macros = macros
122            self.config = None
123            self.create_tar_files = create_tar_files
124            log.notice('config: ' + name)
125            self.set_macros(macros)
126            self.config = config.file(name, opts, self.macros)
127            self.script = script()
128            self.macros['buildname'] = self._name_(self.macros['name'])
129        except error.general, gerr:
130            log.notice(str(gerr))
131            log.stderr('Build FAILED')
132            raise
133        except error.internal, ierr:
134            log.notice(str(ierr))
135            log.stderr('Internal Build FAILED')
136            raise
137        except:
138            raise
139
140    def copy_init_macros(self):
141        return copy.copy(self.init_macros)
142
143    def copy_macros(self):
144        return copy.copy(self.macros)
145
146    def set_macros(self, macros):
147        if macros is None:
148            self.macros = copy.copy(opts.defaults)
149        else:
150            self.macros = copy.copy(macros)
151        if self.config:
152            self.config.set_macros(self.macros)
153
154    def rmdir(self, rmpath):
155        log.output('removing: %s' % (path.host(rmpath)))
156        if not self.opts.dry_run():
157            if path.exists(rmpath):
158                path.removeall(rmpath)
159
160    def mkdir(self, mkpath):
161        log.output('making dir: %s' % (path.host(mkpath)))
162        if not self.opts.dry_run():
163            path.mkdir(mkpath)
164
165    def canadian_cross(self):
166        _host = self.config.expand('%{_host}')
167        _build = self.config.expand('%{_build}')
168        _target = self.config.expand('%{_target}')
169        _allowed = self.config.defined('%{allow_cxc}')
170        if len(_host) and len(_build) and (_target) and \
171           _allowed and _host != _build and _host != _target:
172            return True
173        return False
174
175    def installable(self):
176        _host = self.config.expand('%{_host}')
177        _build = self.config.expand('%{_build}')
178        _canadian_cross = self.canadian_cross()
179        if self.macros.get('_disable_installing') and \
180           self.config.expand('%{_disable_installing}') == 'yes':
181            _disable_installing = True
182        else:
183            _disable_installing = False
184        _no_install = self.opts.no_install()
185        log.trace('_build: installable: host=%s build=%s ' \
186                  'no-install=%r Cxc=%r disable_installing=%r disabled=%r' % \
187                  (_host, _build, _no_install, _canadian_cross, _disable_installing, \
188                   self.disabled()))
189        return len(_host) and len(_build) and \
190            not self.disabled() and \
191            not _disable_installing and \
192            not _canadian_cross
193
194    def source(self, name):
195        #
196        # Return the list of sources. Merge in any macro defined sources as
197        # these may be overridden by user loaded macros.
198        #
199        _map = 'source-%s' % (name)
200        src_keys = [s for s in self.macros.map_keys(_map) if s != 'setup']
201        if len(src_keys) == 0:
202            raise error.general('no source set: %s (%s)' % (name, _map))
203        srcs = []
204        for s in src_keys:
205            sm = self.macros.get(s, globals = False, maps = _map)
206            if sm is None:
207                raise error.internal('source macro not found: %s in %s (%s)' % \
208                                         (s, name, _map))
209            url = self.config.expand(sm[2])
210            src = download.parse_url(url, '_sourcedir', self.config, self.opts)
211            download.get_file(src['url'], src['local'], self.opts, self.config)
212            if 'symlink' in src:
213                sname = name.replace('-', '_')
214                src['script'] = '%%{__ln_s} %s ${source_dir_%s}' % (src['symlink'], sname)
215            elif 'compressed' in src:
216                #
217                # Zip files unpack as well so do not use tar.
218                #
219                src['script'] = '%s %s' % (src['compressed'], src['local'])
220                if src['compressed-type'] != 'zip':
221                    src['script'] += ' | %{__tar_extract} -'
222            else:
223                src['script'] = '%%{__tar_extract} %s' % (src['local'])
224            srcs += [src]
225        return srcs
226
227    def source_setup(self, package, args):
228        log.output('source setup: %s: %s' % (package.name(), ' '.join(args)))
229        setup_name = args[1]
230        args = args[1:]
231        try:
232            opts, args = getopt.getopt(args[1:], 'qDcn:ba')
233        except getopt.GetoptError, ge:
234            raise error.general('source setup error: %s' % str(ge))
235        quiet = False
236        unpack_before_chdir = True
237        delete_before_unpack = True
238        create_dir = False
239        deleted_dir = False
240        created_dir = False
241        changed_dir = False
242        opt_name = None
243        for o in opts:
244            if o[0] == '-q':
245                quiet = True
246            elif o[0] == '-D':
247                delete_before_unpack = False
248            elif o[0] == '-c':
249                create_dir = True
250            elif o[0] == '-n':
251                opt_name = o[1]
252            elif o[0] == '-b':
253                unpack_before_chdir = True
254            elif o[0] == '-a':
255                unpack_before_chdir = False
256        name = None
257        for source in self.source(setup_name):
258            if name is None:
259                if opt_name is None:
260                    if source:
261                        opt_name = source['name']
262                    else:
263                        raise error.general('setup source tag not found: %d' % (source_tag))
264                else:
265                    name = opt_name
266            self.script.append(self.config.expand('cd %{_builddir}'))
267            if not deleted_dir and  delete_before_unpack:
268                self.script.append(self.config.expand('%{__rm} -rf ' + name))
269                deleted_dir = True
270            if not created_dir and create_dir:
271                self.script.append(self.config.expand('%{__mkdir_p} ' + name))
272                created_dir = True
273            if not changed_dir and (not unpack_before_chdir or create_dir):
274                self.script.append(self.config.expand('cd ' + name))
275                changed_dir = True
276            self.script.append(self.config.expand(source['script']))
277        if not changed_dir and (unpack_before_chdir and not create_dir):
278            self.script.append(self.config.expand('cd ' + name))
279            changed_dir = True
280        self.script.append(self.config.expand('%{__setup_post}'))
281
282    def patch_setup(self, package, args):
283        name = args[1]
284        args = args[2:]
285        _map = 'patch-%s' % (name)
286        default_opts = ' '.join(args)
287        patch_keys = [p for p in self.macros.map_keys(_map) if p != 'setup']
288        patches = []
289        for p in patch_keys:
290            pm = self.macros.get(p, globals = False, maps = _map)
291            if pm is None:
292                raise error.internal('patch macro not found: %s in %s (%s)' % \
293                                         (p, name, _map))
294            opts = []
295            url = []
296            for pp in pm[2].split():
297                if len(url) == 0 and pp[0] == '-':
298                    opts += [pp]
299                else:
300                    url += [pp]
301            if len(url) == 0:
302                raise error.general('patch URL not found: %s' % (' '.join(args)))
303            if len(opts) == 0:
304                opts = default_opts
305            else:
306                opts = ' '.join(opts)
307            opts = self.config.expand(opts)
308            url = self.config.expand(' '.join(url))
309            #
310            # Parse the URL first in the source builder's patch directory.
311            #
312            patch = download.parse_url(url, '_patchdir', self.config, self.opts)
313            #
314            # If not in the source builder package check the source directory.
315            #
316            if not path.exists(patch['local']):
317                patch = download.parse_url(url, '_patchdir', self.config, self.opts)
318            download.get_file(patch['url'], patch['local'], self.opts, self.config)
319            if 'compressed' in patch:
320                patch['script'] = patch['compressed'] + ' ' +  patch['local']
321            else:
322                patch['script'] = '%{__cat} ' + patch['local']
323            patch['script'] += ' | %%{__patch} %s' % (opts)
324            self.script.append(self.config.expand(patch['script']))
325
326    def run(self, command, shell_opts = '', cwd = None):
327        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
328        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
329        log.output('run: ' + cmd)
330        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
331        if exit_code != 0:
332            log.output('shell cmd failed: %s' % (cmd))
333            raise error.general('building %s' % (self.macros['buildname']))
334
335    def builddir(self):
336        builddir = self.config.abspath('_builddir')
337        self.rmdir(builddir)
338        if not self.opts.dry_run():
339            self.mkdir(builddir)
340
341    def prep(self, package):
342        self.script.append('echo "==> %prep:"')
343        _prep = package.prep()
344        if _prep:
345            for l in _prep:
346                args = l.split()
347                if len(args):
348                    def err(msg):
349                        raise error.general('%s: %s' % (package, msg))
350                    if args[0] == '%setup':
351                        if len(args) == 1:
352                            raise error.general('invalid %%setup directive: %s' % \
353                                                (' '.join(args)))
354                        if args[1] == 'source':
355                            self.source_setup(package, args[1:])
356                        elif args[1] == 'patch':
357                            self.patch_setup(package, args[1:])
358                    elif args[0] in ['%patch', '%source']:
359                        sources.process(args[0][1:], args[1:], self.macros, err)
360                    elif args[0] == '%hash':
361                        sources.hash(args[1:], self.macros, err)
362                        self.hash(package, args)
363                    else:
364                        self.script.append(' '.join(args))
365
366    def build(self, package):
367        self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
368        self.script.append('%s ${SB_BUILD_ROOT}' %
369                           (self.config.expand('%{__rmdir}')))
370        self.script.append('%s ${SB_BUILD_ROOT}' %
371                           (self.config.expand('%{__mkdir_p}')))
372        self.script.append('echo "==> %build:"')
373        _build = package.build()
374        if _build:
375            for l in _build:
376                self.script.append(l)
377
378    def install(self, package):
379        self.script.append('echo "==> %install:"')
380        _install = package.install()
381        if _install:
382            for l in _install:
383                args = l.split()
384                self.script.append(' '.join(args))
385
386    def files(self, package):
387        if self.create_tar_files \
388           and not self.macros.get('%{_disable_packaging'):
389            self.script.append('echo "==> %files:"')
390            inpath = path.abspath(self.config.expand('%{buildroot}'))
391            tardir = path.abspath(self.config.expand('%{_tardir}'))
392            self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
393            self.script.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))
394            self.script.append(self.config.expand('  cd ' + inpath))
395            tar = path.join(tardir, package.long_name() + '.tar.bz2')
396            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
397            self.script.append(cmd)
398            self.script.append(self.config.expand('  cd %{_builddir}'))
399            self.script.append('fi')
400
401    def clean(self, package):
402        self.script.append('echo "==> %clean:"')
403        _clean = package.clean()
404        if _clean is not None:
405            for l in _clean:
406                args = l.split()
407                self.script.append(' '.join(args))
408
409    def build_package(self, package):
410        if self.canadian_cross():
411            if not self.config.defined('%{allow_cxc}'):
412                raise error.general('Canadian Cross is not allowed')
413            self.script.append('echo "==> Candian-cross build/target:"')
414            self.script.append('SB_CXC="yes"')
415        else:
416            self.script.append('SB_CXC="no"')
417        self.build(package)
418        self.install(package)
419        self.files(package)
420        if not self.opts.no_clean():
421            self.clean(package)
422
423    def cleanup(self):
424        package = self.main_package()
425        if not package.disabled() and not self.opts.no_clean():
426            buildroot = self.config.abspath('buildroot')
427            builddir = self.config.abspath('_builddir')
428            buildcxcdir = self.config.abspath('_buildcxcdir')
429            tmproot = self.config.abspath('_tmproot')
430            log.trace('cleanup: %s' % (buildroot))
431            self.rmdir(buildroot)
432            log.trace('cleanup: %s' % (builddir))
433            self.rmdir(builddir)
434            if self.canadian_cross():
435                log.trace('cleanup: %s' % (buildcxcdir))
436                self.rmdir(buildcxcdir)
437            log.trace('cleanup: %s' % (tmproot))
438            self.rmdir(tmproot)
439
440    def main_package(self):
441        packages = self.config.packages()
442        return packages['main']
443
444    def reload(self):
445        self.config.load(self.init_name)
446
447    def make(self):
448        package = self.main_package()
449        if package.disabled():
450            log.notice('package: nothing to build')
451        else:
452            try:
453                name = package.name()
454                if self.canadian_cross():
455                    cxc_label = '(Cxc) '
456                else:
457                    cxc_label = ''
458                log.notice('package: %s%s' % (cxc_label, name))
459                log.trace('---- macro maps %s' % ('-' * 55))
460                log.trace('%s' % (str(self.config.macros)))
461                log.trace('-' * 70)
462                self.script.reset()
463                self.script.append(self.config.expand('%{___build_template}'))
464                self.script.append('echo "=> ' + name + ':"')
465                self.prep(package)
466                self.build_package(package)
467                if not self.opts.dry_run():
468                    self.builddir()
469                    sn = path.join(self.config.expand('%{_builddir}'), 'doit')
470                    log.output('write script: ' + sn)
471                    self.script.write(sn)
472                    log.notice('building: %s%s' % (cxc_label, name))
473                    self.run(sn)
474            except error.general, gerr:
475                log.notice(str(gerr))
476                log.stderr('Build FAILED')
477                self._generate_report_('Build: %s' % (gerr))
478                raise
479            except error.internal, ierr:
480                log.notice(str(ierr))
481                log.stderr('Internal Build FAILED')
482                self._generate_report_('Build: %s' % (ierr))
483                raise
484            except:
485                raise
486            if self.opts.dry_run():
487                self._generate_report_('Build: dry run (no actual error)',
488                                       'Build: dry run (no actual error)')
489
490    def name(self):
491        packages = self.config.packages()
492        package = packages['main']
493        return package.name()
494
495    def disabled(self):
496        packages = self.config.packages()
497        package = packages['main']
498        return package.disabled()
499
500def get_configs(opts):
501
502    def _scan(_path, ext):
503        configs = []
504        for root, dirs, files in os.walk(_path):
505            prefix = root[len(_path) + 1:]
506            for file in files:
507                for e in ext:
508                    if file.endswith(e):
509                        configs += [path.join(prefix, file)]
510        return configs
511
512    configs = { 'paths': [], 'files': [] }
513    for cp in opts.defaults.expand('%{_configdir}').split(':'):
514        hcp = path.host(path.abspath(cp))
515        configs['paths'] += [hcp]
516        configs['files'] += _scan(hcp, ['.cfg', '.bset'])
517    configs['files'] = sorted(set(configs['files']))
518    return configs
519
520def find_config(config, configs):
521    config_root, config_ext = path.splitext(config)
522    if config_ext not in ['', '.bset', '.cfg']:
523        config_root = config
524        config_ext = ''
525    for c in configs['files']:
526        r, e = path.splitext(c)
527        if config_root == r:
528            if config_ext == '' or config_ext == e:
529                return c
530    return None
531
532def run(args):
533    ec = 0
534    try:
535        optargs = { '--list-configs': 'List available configurations' }
536        opts = options.load(args, optargs)
537        log.notice('RTEMS Source Builder, Package Builder v%s' % (version.str()))
538        if not check.host_setup(opts):
539            if not opts.force():
540                raise error.general('host build environment is not set up' +
541                                    ' correctly (use --force to proceed)')
542            log.notice('warning: forcing build with known host setup problems')
543        if opts.get_arg('--list-configs'):
544            configs = get_configs(opts)
545            for p in configs['paths']:
546                print 'Examining: %s' % (os.path.relpath(p))
547                for c in configs['files']:
548                    if c.endswith('.cfg'):
549                        print '    %s' % (c)
550        else:
551            for config_file in opts.config_files():
552                b = build(config_file, True, opts)
553                b.make()
554                b = None
555    except error.general, gerr:
556        log.stderr('Build FAILED')
557        ec = 1
558    except error.internal, ierr:
559        log.stderr('Internal Build FAILED')
560        ec = 1
561    except error.exit, eerr:
562        pass
563    except KeyboardInterrupt:
564        log.notice('abort: user terminated')
565        ec = 1
566    sys.exit(ec)
567
568if __name__ == "__main__":
569    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.