source: rtems-source-builder/source-builder/sb/build.py @ 4f26bdb

4.104.114.95
Last change on this file since 4f26bdb was 4f26bdb, checked in by Chris Johns <chrisj@…>, on 04/01/13 at 04:19:56

Add Canadian Cross support.

Add support to build MinGW tools using Cygwin. This is a Canadian cross
build.

Do not expand the directives when parsing a configuration file. Hold
in the package object the text as read from the configuration file. Still
parse the logic but leave the macros. This allows a configuration to be
varied when the build happens. The Canadian cross uses this to build a
build compiler used to build a Cxc runtime.

Add Cxc support to the build module. In the defaults add rm and rmfile
macros, add Cxc paths and pre-build script code.

In the setbuilder check for a Cxc build and if so and the package
allow Cxc build the build host version then the host target
version.

Add cygiwn support to the defaults processing and to the Windows module.
  • Property mode set to 100644
File size: 20.3 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2012 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import getopt
26import glob
27import os
28import shutil
29import stat
30import sys
31import urllib2
32import urlparse
33
34import check
35import config
36import defaults
37import error
38import execute
39import log
40import path
41
42#
43# Version of Sourcer Builder Build.
44#
45version = '0.1'
46
47def _notice(opts, text):
48    if not opts.quiet() and not log.default.has_stdout():
49        print text
50    log.output(text)
51    log.flush()
52
53class script:
54    """Create and manage a shell script."""
55
56    def __init__(self, quiet = True, trace = False):
57        self.quiet = quiet
58        self.trace = trace
59        self.reset()
60
61    def reset(self):
62        self.body = []
63        self.lc = 0
64
65    def append(self, text):
66        if type(text) is str:
67            text = text.splitlines()
68        if not self.quiet:
69            i = 0
70            for l in text:
71                i += 1
72                log.output('script:%3d: %s' % (self.lc + i, l))
73                if self.trace:
74                    print '%3d: S %s' % (self.lc + i,  l)
75        self.lc += len(text)
76        self.body.extend(text)
77
78    def write(self, name, check_for_errors = False):
79        s = None
80        try:
81            s = open(path.host(name), 'w')
82            s.write('\n'.join(self.body))
83            s.close()
84            os.chmod(path.host(name), stat.S_IRWXU | \
85                         stat.S_IRGRP | stat.S_IXGRP | \
86                         stat.S_IROTH | stat.S_IXOTH)
87        except IOError, err:
88            raise error.general('creating script: ' + name)
89        except:
90            if s is not None:
91                s.close()
92            raise
93        if s is not None:
94            s.close()
95
96class build:
97    """Build a package given a config file."""
98
99    def __init__(self, name, create_tar_files, _defaults, opts):
100        self.opts = opts
101        self.create_tar_files = create_tar_files
102        _notice(opts, 'config: ' + name)
103        self.config = config.file(name, _defaults = _defaults, opts = opts)
104        self.script = script(quiet = opts.quiet(), trace = opts.trace())
105
106    def _output(self, text):
107        if not self.opts.quiet():
108            log.output(text)
109
110    def rmdir(self, rmpath):
111        self._output('removing: %s' % (path.host(rmpath)))
112        if not self.opts.dry_run():
113            if path.exists(rmpath):
114                path.removeall(rmpath)
115
116    def mkdir(self, mkpath):
117        self._output('making dir: %s' % (path.host(mkpath)))
118        if not self.opts.dry_run():
119            path.mkdir(mkpath)
120
121    def get_file(self, url, local):
122        if local is None:
123            raise error.general('source/patch path invalid')
124        if not path.isdir(path.dirname(local)):
125            _notice(self.opts,
126                    'Creating source directory: %s' % (os.path.relpath(path.host(path.dirname(local)))))
127            self.mkdir(path.host(path.dirname(local)))
128        if not path.exists(local):
129            #
130            # Not localy found so we need to download it. Check if a URL has
131            # been provided on the command line.
132            #
133            url_bases = self.opts.urls()
134            urls = []
135            if url_bases is not None:
136                for base in url_bases:
137                    if base[-1:] != '/':
138                        base += '/'
139                    url_path = urlparse.urlsplit(url)[2]
140                    slash = url_path.rfind('/')
141                    if slash < 0:
142                        url_file = url_path
143                    else:
144                        url_file = url_path[slash + 1:]
145                    urls.append(urlparse.urljoin(base, url_file))
146            urls.append(url)
147            if self.opts.trace():
148                print '_url:', ','.join(urls), '->', local
149            for url in urls:
150                #
151                # Hack for GitHub.
152                #
153                if url.startswith('https://api.github.com'):
154                    url = urlparse.urljoin(url, self.config.expand('tarball/%{version}'))
155                _notice(self.opts, 'download: %s -> %s' % (url, os.path.relpath(path.host(local))))
156                if not self.opts.dry_run():
157                    failed = False
158                    _in = None
159                    _out = None
160                    try:
161                        _in = urllib2.urlopen(url)
162                        _out = open(path.host(local), 'wb')
163                        _out.write(_in.read())
164                    except IOError, err:
165                        msg = 'download: %s: error: %s' % (url, str(err))
166                        _notice(self.opts, msg)
167                        if path.exists(local):
168                            os.remove(path.host(local))
169                        failed = True
170                    except ValueError, err:
171                        msg = 'download: %s: error: %s' % (url, str(err))
172                        _notice(self.opts, msg)
173                        if path.exists(local):
174                            os.remove(path.host(local))
175                        failed = True
176                    except:
177                        msg = 'download: %s: error' % (url)
178                        print >> sys.stderr, msg
179                        if _out is not None:
180                            _out.close()
181                        raise
182                    if _out is not None:
183                        _out.close()
184                    if _in is not None:
185                        del _in
186                    if not failed:
187                        if not path.isfile(local):
188                            raise error.general('source is not a file: %s' % (path.host(local)))
189                        return
190            if not self.opts.dry_run():
191                raise error.general('downloading %s: all paths have failed, giving up' % (url))
192
193    def parse_url(self, url, pathkey):
194        #
195        # Split the source up into the parts we need.
196        #
197        source = {}
198        source['url'] = url
199        source['path'] = path.dirname(url)
200        source['file'] = path.basename(url)
201        source['name'], source['ext'] = path.splitext(source['file'])
202        #
203        # Get the file. Checks the local source directory first.
204        #
205        source['local'] = None
206        for p in self.config.define(pathkey).split(':'):
207            local = path.join(path.abspath(p), source['file'])
208            if source['local'] is None:
209                source['local'] = local
210            if path.exists(local):
211                source['local'] = local
212                break
213        #
214        # Is the file compressed ?
215        #
216        esl = source['ext'].split('.')
217        if esl[-1:][0] == 'gz':
218            source['compressed'] = '%{__gzip} -dc'
219        elif esl[-1:][0] == 'bz2':
220            source['compressed'] = '%{__bzip2} -dc'
221        elif esl[-1:][0] == 'bz2':
222            source['compressed'] = '%{__zip} -u'
223        elif esl[-1:][0] == 'xz':
224            source['compressed'] = '%{__xz} -dc'
225        source['script'] = ''
226        return source
227
228    def source(self, package, source_tag):
229        #
230        # Scan the sources found in the config file for the one we are
231        # after. Infos or tags are lists.
232        #
233        sources = package.sources()
234        url = None
235        for s in sources:
236            tag = s[len('source'):]
237            if tag.isdigit():
238                if int(tag) == source_tag:
239                    url = sources[s][0]
240                    break
241        if url is None:
242            raise error.general('source tag not found: source%d' % (source_tag))
243        source = self.parse_url(url, '_sourcedir')
244        self.get_file(source['url'], source['local'])
245        if 'compressed' in source:
246            source['script'] = source['compressed'] + ' ' + \
247                source['local'] + ' | %{__tar_extract} -'
248        else:
249            source['script'] = '%{__tar_extract} ' + source['local']
250        return source
251
252    def patch(self, package, args):
253        #
254        # Scan the patches found in the config file for the one we are
255        # after. Infos or tags are lists.
256        #
257        patches = package.patches()
258        url = None
259        for p in patches:
260            if args[0][1:].lower() == p:
261                url = patches[p][0]
262                break
263        if url is None:
264            raise error.general('patch tag not found: %s' % (args[0]))
265        #
266        # Parse the URL first in the source builder's patch directory.
267        #
268        patch = self.parse_url(url, '_patchdir')
269        #
270        # If not in the source builder package check the source directory.
271        #
272        if not path.exists(patch['local']):
273            patch = self.parse_url(url, '_patchdir')
274        self.get_file(patch['url'], patch['local'])
275        if 'compressed' in patch:
276            patch['script'] = patch['compressed'] + ' ' +  patch['local']
277        else:
278            patch['script'] = '%{__cat} ' + patch['local']
279        patch['script'] += ' | %{__patch} ' + ' '.join(args[1:])
280        self.script.append(self.config.expand(patch['script']))
281
282    def canadian_cross(self):
283        _host = self.config.expand('%{_host}')
284        _build = self.config.expand('%{_build}')
285        _target = self.config.expand('%{_target}')
286        return self.config.defined('%{allow_cxc}') and \
287            _host != _build and _host != _target
288
289    def setup(self, package, args):
290        self._output('prep: %s: %s' % (package.name(), ' '.join(args)))
291        opts, args = getopt.getopt(args[1:], 'qDcTn:b:a:')
292        source_tag = 0
293        quiet = False
294        unpack_default_source = True
295        unpack_before_chdir = True
296        delete_before_unpack = True
297        create_dir = False
298        name = None
299        for o in opts:
300            if o[0] == '-q':
301                quiet = True
302            elif o[0] == '-D':
303                delete_before_unpack = False
304            elif o[0] == '-c':
305                create_dir = True
306            elif o[0] == '-T':
307                unpack_default_source = False
308            elif o[0] == '-n':
309                name = o[1]
310            elif o[0] == '-b':
311                unpack_before_chdir = True
312                if not o[1].isdigit():
313                    raise error.general('setup -b source tag is not a number: %s' % (o[1]))
314                source_tag = int(o[1])
315            elif o[0] == '-a':
316                unpack_before_chdir = False
317                if not o[1].isdigit():
318                    raise error.general('setup -a source tag is not a number: %s' % (o[1]))
319                source_tag = int(o[1])
320        source0 = None
321        source = self.source(package, source_tag)
322        if name is None:
323            if source:
324                name = source['name']
325            else:
326                raise error.general('setup source tag not found: %d' % (source_tag))
327        self.script.append(self.config.expand('cd %{_builddir}'))
328        if delete_before_unpack:
329            self.script.append(self.config.expand('%{__rm} -rf ' + name))
330        if create_dir:
331            self.script.append(self.config.expand('%{__mkdir_p} ' + name))
332        #
333        # If -a? then change directory before unpacking.
334        #
335        if not unpack_before_chdir or create_dir:
336            self.script.append(self.config.expand('cd ' + name))
337        #
338        # Unpacking the source. Note, treated the same as -a0.
339        #
340        if unpack_default_source and source_tag != 0:
341            source0 = self.source(package, 0)
342            if source0 is None:
343                raise error.general('no setup source0 tag found')
344            self.script.append(self.config.expand(source0['script']))
345        self.script.append(self.config.expand(source['script']))
346        if unpack_before_chdir and not create_dir:
347            self.script.append(self.config.expand('cd ' + name))
348        self.script.append(self.config.expand('%{__setup_post}'))
349
350    def run(self, command, shell_opts = '', cwd = None):
351        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
352        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
353        self._output('run: ' + cmd)
354        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
355        if exit_code != 0:
356            raise error.general('shell cmd failed: %s' % (cmd))
357
358    def builddir(self):
359        builddir = self.config.abspath('_builddir')
360        self.rmdir(builddir)
361        if not self.opts.dry_run():
362            self.mkdir(builddir)
363
364    def prep(self, package):
365        self.script.append('echo "==> %prep:"')
366        _prep = package.prep()
367        for l in _prep:
368            args = l.split()
369            if args[0] == '%setup':
370                self.setup(package, args)
371            elif args[0].startswith('%patch'):
372                self.patch(package, args)
373            else:
374                self.script.append(' '.join(args))
375
376    def build(self, package):
377        self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
378        self.script.append('%s ${SB_BUILD_ROOT}' %
379                           (self.config.expand('%{__rmdir}')))
380        self.script.append('%s ${SB_BUILD_ROOT}' %
381                           (self.config.expand('%{__mkdir_p}')))
382        self.script.append('echo "==> %build:"')
383        _build = package.build()
384        for l in _build:
385            args = l.split()
386            self.script.append(' '.join(args))
387
388    def install(self, package):
389        self.script.append('echo "==> %install:"')
390        _install = package.install()
391        for l in _install:
392            args = l.split()
393            self.script.append(' '.join(args))
394
395    def files(self, package):
396        if self.create_tar_files:
397            self.script.append('echo "==> %files:"')
398            prefixbase = self.opts.prefixbase()
399            if prefixbase is None:
400                prefixbase = ''
401            inpath = path.join('%{buildroot}', prefixbase)
402            tardir = path.abspath(self.config.expand('%{_tardir}'))
403            self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
404            self.script.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))
405            self.script.append(self.config.expand('  cd ' + inpath))
406            tar = path.join(tardir, package.long_name() + '.tar.bz2')
407            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
408            self.script.append(cmd)
409            self.script.append(self.config.expand('  cd %{_builddir}'))
410            self.script.append('fi')
411
412    def clean(self, package):
413        self.script.append('echo "==> %clean:"')
414        _clean = package.clean()
415        if _clean is not None:
416            for l in _clean:
417                args = l.split()
418                self.script.append(' '.join(args))
419
420    def build_package(self, package):
421        if self.canadian_cross():
422            self.script.append('echo "==> Candian-cross build/target:"')
423            self.script.append('SB_CXC="yes"')
424        else:
425            self.script.append('SB_CXC="no"')
426        self.build(package)
427        self.install(package)
428        self.files(package)
429        if not self.opts.no_clean():
430            self.clean(package)
431
432    def cleanup(self):
433        if not self.opts.no_clean():
434            buildroot = self.config.abspath('buildroot')
435            builddir = self.config.abspath('_builddir')
436            buildcxcdir = self.config.abspath('_buildcxcdir')
437            tmproot = self.config.abspath('_tmproot')
438            if self.opts.trace():
439                _notice(self.opts, 'cleanup: %s' % (buildroot))
440            self.rmdir(buildroot)
441            if self.opts.trace():
442                _notice(self.opts, 'cleanup: %s' % (builddir))
443            self.rmdir(builddir)
444            if self.canadian_cross():
445                if self.opts.trace():
446                    _notice(self.opts, 'cleanup: %s' % (buildcxcdir))
447                self.rmdir(buildcxcdir)
448            if self.opts.trace():
449                _notice(self.opts, 'cleanup: %s' % (tmproot))
450            self.rmdir(tmproot)
451
452    def main_package(self):
453        packages = self.config.packages()
454        return packages['main']
455
456    def make(self):
457        package = self.main_package()
458        name = package.name()
459        if self.canadian_cross():
460            _notice(self.opts, 'package: (Cxc) %s' % (name))
461        else:
462            _notice(self.opts, 'package: %s' % (name))
463        self.script.reset()
464        self.script.append(self.config.expand('%{___build_template}'))
465        self.script.append('echo "=> ' + name + ':"')
466        self.prep(package)
467        self.build_package(package)
468        if not self.opts.dry_run():
469            self.builddir()
470            sn = path.join(self.config.expand('%{_builddir}'), 'doit')
471            self._output('write script: ' + sn)
472            self.script.write(sn)
473            if self.canadian_cross():
474                _notice(self.opts, 'building: (Cxc) %s' % (name))
475            else:
476                _notice(self.opts, 'building: %s' % (name))
477            self.run(sn)
478
479    def name(self):
480        packages = self.config.packages()
481        package = packages['main']
482        return package.name()
483
484def get_configs(opts, _defaults):
485
486    def _scan(_path, ext):
487        configs = []
488        for root, dirs, files in os.walk(_path):
489            prefix = root[len(_path) + 1:]
490            for file in files:
491                for e in ext:
492                    if file.endswith(e):
493                        configs += [path.join(prefix, file)]
494        return configs
495
496    configs = { 'paths': [], 'files': [] }
497    for cp in opts.expand('%{_configdir}', _defaults).split(':'):
498        hcp = path.host(path.abspath(cp))
499        configs['paths'] += [hcp]
500        configs['files'] += _scan(hcp, ['.cfg', '.bset'])
501    configs['files'] = sorted(configs['files'])
502    return configs
503
504def find_config(config, configs):
505    config_root, config_ext = path.splitext(config)
506    if config_ext not in ['', '.bset', '.cfg']:
507        config_root = config
508        config_ext = ''
509    for c in configs['files']:
510        r, e = path.splitext(c)
511        if config_root == r:
512            if config_ext == '' or config_ext == e:
513                return c
514    return None
515
516def run(args):
517    try:
518        optargs = { '--list-configs': 'List available configurations' }
519        opts, _defaults = defaults.load(args, optargs)
520        log.default = log.log(opts.logfiles())
521        _notice(opts, 'RTEMS Source Builder, Package Builder v%s' % (version))
522        if not check.host_setup(opts, _defaults):
523            if not opts.force():
524                raise error.general('host build environment is not set up' +
525                                    ' correctly (use --force to proceed)')
526            _notice(opts, 'warning: forcing build with known host setup problems')
527        if opts.get_arg('--list-configs'):
528            configs = get_configs(opts, _defaults)
529            for p in configs['paths']:
530                print 'Examining: %s' % (os.path.relpath(p))
531                for c in configs['files']:
532                    if c.endswith('.cfg'):
533                        print '    %s' % (c)
534        else:
535            for config_file in opts.config_files():
536                b = build(config_file, True, _defaults = _defaults, opts = opts)
537                b.make()
538                del b
539    except error.general, gerr:
540        print gerr
541        print >> sys.stderr, 'Build FAILED'
542        sys.exit(1)
543    except error.internal, ierr:
544        print ierr
545        print >> sys.stderr, 'Build FAILED'
546        sys.exit(1)
547    except error.exit, eerr:
548        pass
549    except KeyboardInterrupt:
550        _notice(opts, 'abort: user terminated')
551        sys.exit(1)
552    sys.exit(0)
553
554if __name__ == "__main__":
555    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.