source: rtems-source-builder/source-builder/sb/build.py @ 649a64c

4.104.114.95
Last change on this file since 649a64c was 649a64c, checked in by Chris Johns <chrisj@…>, on 04/16/13 at 04:25:34

Add download git support.

Add support to use a git cloned repo as the source. Move the download
code out of the build module and into a separate module. Add to this
module support for git.

Update the GCC common configuration to support using a symlinked
git repo.

Add checks for all languages.
  • Property mode set to 100644
File size: 16.5 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import getopt
26import glob
27import os
28import shutil
29import stat
30import sys
31import urllib2
32import urlparse
33
34try:
35    import check
36    import config
37    import download
38    import error
39    import execute
40    import log
41    import options
42    import path
43    import version
44except KeyboardInterrupt:
45    print 'abort: user terminated'
46    sys.exit(1)
47except:
48    print 'error: unknown application load error'
49    sys.exit(1)
50
51def _notice(opts, text):
52    if not opts.quiet() and not log.default.has_stdout():
53        print text
54    log.output(text)
55    log.flush()
56
57class script:
58    """Create and manage a shell script."""
59
60    def __init__(self, quiet = True, trace = False):
61        self.quiet = quiet
62        self.trace = trace
63        self.reset()
64
65    def reset(self):
66        self.body = []
67        self.lc = 0
68
69    def append(self, text):
70        if type(text) is str:
71            text = text.splitlines()
72        if not self.quiet:
73            i = 0
74            for l in text:
75                i += 1
76                log.output('script:%3d: %s' % (self.lc + i, l))
77                if self.trace:
78                    print '%3d: S %s' % (self.lc + i,  l)
79        self.lc += len(text)
80        self.body.extend(text)
81
82    def write(self, name, check_for_errors = False):
83        s = None
84        try:
85            s = open(path.host(name), 'w')
86            s.write('\n'.join(self.body))
87            s.close()
88            os.chmod(path.host(name), stat.S_IRWXU | \
89                         stat.S_IRGRP | stat.S_IXGRP | \
90                         stat.S_IROTH | stat.S_IXOTH)
91        except IOError, err:
92            raise error.general('creating script: ' + name)
93        except:
94            if s is not None:
95                s.close()
96            raise
97        if s is not None:
98            s.close()
99
100class build:
101    """Build a package given a config file."""
102
103    def __init__(self, name, create_tar_files, opts, macros = None):
104        self.opts = opts
105        if macros is None:
106            self.macros = opts.defaults
107        else:
108            self.macros = macros
109        self.create_tar_files = create_tar_files
110        _notice(opts, 'config: ' + name)
111        self.config = config.file(name, opts, self.macros)
112        self.script = script(quiet = opts.quiet(), trace = opts.trace())
113
114    def _output(self, text):
115        if not self.opts.quiet():
116            log.output(text)
117
118    def rmdir(self, rmpath):
119        self._output('removing: %s' % (path.host(rmpath)))
120        if not self.opts.dry_run():
121            if path.exists(rmpath):
122                path.removeall(rmpath)
123
124    def mkdir(self, mkpath):
125        self._output('making dir: %s' % (path.host(mkpath)))
126        if not self.opts.dry_run():
127            path.mkdir(mkpath)
128
129    def source(self, package, source_tag):
130        #
131        # Scan the sources found in the config file for the one we are
132        # after. Infos or tags are lists. Merge in any macro defined
133        # sources as these may be overridden by user loaded macros.
134        #
135        sources = package.sources()
136        url = None
137        for s in sources:
138            tag = s[len('source'):]
139            if tag.isdigit():
140                if int(tag) == source_tag:
141                    url = sources[s][0]
142                    break
143        if url is None:
144            raise error.general('source tag not found: source%d' % (source_tag))
145        source = download.parse_url(url, '_sourcedir', self.config, self.opts)
146        download.get_file(source['url'], source['local'], self.opts, self.config)
147        if 'symlink' in source:
148            source['script'] = '%%{__ln_s} %s ${source_dir_%d}' % (source['local'], source_tag)
149        elif 'compressed' in source:
150            source['script'] = source['compressed'] + ' ' + \
151                source['local'] + ' | %{__tar_extract} -'
152        else:
153            source['script'] = '%{__tar_extract} ' + source['local']
154        return source
155
156    def patch(self, package, args):
157        #
158        # Scan the patches found in the config file for the one we are
159        # after. Infos or tags are lists.
160        #
161        patches = package.patches()
162        url = None
163        for p in patches:
164            if args[0][1:].lower() == p:
165                url = patches[p][0]
166                break
167        if url is None:
168            raise error.general('patch tag not found: %s' % (args[0]))
169        #
170        # Parse the URL first in the source builder's patch directory.
171        #
172        patch = download.parse_url(url, '_patchdir', self.config, self.opts)
173        #
174        # If not in the source builder package check the source directory.
175        #
176        if not path.exists(patch['local']):
177            patch = download.parse_url(url, '_patchdir', self.config, self.opts)
178        download.get_file(patch['url'], patch['local'], self.opts, self.config)
179        if 'compressed' in patch:
180            patch['script'] = patch['compressed'] + ' ' +  patch['local']
181        else:
182            patch['script'] = '%{__cat} ' + patch['local']
183        patch['script'] += ' | %{__patch} ' + ' '.join(args[1:])
184        self.script.append(self.config.expand(patch['script']))
185
186    def canadian_cross(self):
187        _host = self.config.expand('%{_host}')
188        _build = self.config.expand('%{_build}')
189        _target = self.config.expand('%{_target}')
190        return self.config.defined('%{allow_cxc}') and \
191            _host != _build and _host != _target
192
193    def setup(self, package, args):
194        self._output('prep: %s: %s' % (package.name(), ' '.join(args)))
195        opts, args = getopt.getopt(args[1:], 'qDcTn:b:a:')
196        source_tag = 0
197        quiet = False
198        unpack_default_source = True
199        unpack_before_chdir = True
200        delete_before_unpack = True
201        create_dir = False
202        name = None
203        for o in opts:
204            if o[0] == '-q':
205                quiet = True
206            elif o[0] == '-D':
207                delete_before_unpack = False
208            elif o[0] == '-c':
209                create_dir = True
210            elif o[0] == '-T':
211                unpack_default_source = False
212            elif o[0] == '-n':
213                name = o[1]
214            elif o[0] == '-b':
215                unpack_before_chdir = True
216                if not o[1].isdigit():
217                    raise error.general('setup -b source tag is not a number: %s' % (o[1]))
218                source_tag = int(o[1])
219            elif o[0] == '-a':
220                unpack_before_chdir = False
221                if not o[1].isdigit():
222                    raise error.general('setup -a source tag is not a number: %s' % (o[1]))
223                source_tag = int(o[1])
224        source0 = None
225        source = self.source(package, source_tag)
226        if name is None:
227            if source:
228                name = source['name']
229            else:
230                raise error.general('setup source tag not found: %d' % (source_tag))
231        self.script.append(self.config.expand('cd %{_builddir}'))
232        if delete_before_unpack:
233            self.script.append(self.config.expand('%{__rm} -rf ' + name))
234        if create_dir:
235            self.script.append(self.config.expand('%{__mkdir_p} ' + name))
236        #
237        # If -a? then change directory before unpacking.
238        #
239        if not unpack_before_chdir or create_dir:
240            self.script.append(self.config.expand('cd ' + name))
241        #
242        # Unpacking the source. Note, treated the same as -a0.
243        #
244        if unpack_default_source and source_tag != 0:
245            source0 = self.source(package, 0)
246            if source0 is None:
247                raise error.general('no setup source0 tag found')
248            self.script.append(self.config.expand(source0['script']))
249        self.script.append(self.config.expand(source['script']))
250        if unpack_before_chdir and not create_dir:
251            self.script.append(self.config.expand('cd ' + name))
252        self.script.append(self.config.expand('%{__setup_post}'))
253
254    def run(self, command, shell_opts = '', cwd = None):
255        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
256        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
257        self._output('run: ' + cmd)
258        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
259        if exit_code != 0:
260            raise error.general('shell cmd failed: %s' % (cmd))
261
262    def builddir(self):
263        builddir = self.config.abspath('_builddir')
264        self.rmdir(builddir)
265        if not self.opts.dry_run():
266            self.mkdir(builddir)
267
268    def prep(self, package):
269        self.script.append('echo "==> %prep:"')
270        _prep = package.prep()
271        for l in _prep:
272            args = l.split()
273            if args[0] == '%setup':
274                self.setup(package, args)
275            elif args[0].startswith('%patch'):
276                self.patch(package, args)
277            else:
278                self.script.append(' '.join(args))
279
280    def build(self, package):
281        self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
282        self.script.append('%s ${SB_BUILD_ROOT}' %
283                           (self.config.expand('%{__rmdir}')))
284        self.script.append('%s ${SB_BUILD_ROOT}' %
285                           (self.config.expand('%{__mkdir_p}')))
286        self.script.append('echo "==> %build:"')
287        _build = package.build()
288        for l in _build:
289            args = l.split()
290            self.script.append(' '.join(args))
291
292    def install(self, package):
293        self.script.append('echo "==> %install:"')
294        _install = package.install()
295        for l in _install:
296            args = l.split()
297            self.script.append(' '.join(args))
298
299    def files(self, package):
300        if self.create_tar_files:
301            self.script.append('echo "==> %files:"')
302            inpath = path.abspath(self.config.expand('%{buildroot}'))
303            tardir = path.abspath(self.config.expand('%{_tardir}'))
304            self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
305            self.script.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))
306            self.script.append(self.config.expand('  cd ' + inpath))
307            tar = path.join(tardir, package.long_name() + '.tar.bz2')
308            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
309            self.script.append(cmd)
310            self.script.append(self.config.expand('  cd %{_builddir}'))
311            self.script.append('fi')
312
313    def clean(self, package):
314        self.script.append('echo "==> %clean:"')
315        _clean = package.clean()
316        if _clean is not None:
317            for l in _clean:
318                args = l.split()
319                self.script.append(' '.join(args))
320
321    def build_package(self, package):
322        if self.canadian_cross():
323            self.script.append('echo "==> Candian-cross build/target:"')
324            self.script.append('SB_CXC="yes"')
325        else:
326            self.script.append('SB_CXC="no"')
327        self.build(package)
328        self.install(package)
329        self.files(package)
330        if not self.opts.no_clean():
331            self.clean(package)
332
333    def cleanup(self):
334        if not self.opts.no_clean():
335            buildroot = self.config.abspath('buildroot')
336            builddir = self.config.abspath('_builddir')
337            buildcxcdir = self.config.abspath('_buildcxcdir')
338            tmproot = self.config.abspath('_tmproot')
339            if self.opts.trace():
340                _notice(self.opts, 'cleanup: %s' % (buildroot))
341            self.rmdir(buildroot)
342            if self.opts.trace():
343                _notice(self.opts, 'cleanup: %s' % (builddir))
344            self.rmdir(builddir)
345            if self.canadian_cross():
346                if self.opts.trace():
347                    _notice(self.opts, 'cleanup: %s' % (buildcxcdir))
348                self.rmdir(buildcxcdir)
349            if self.opts.trace():
350                _notice(self.opts, 'cleanup: %s' % (tmproot))
351            self.rmdir(tmproot)
352
353    def main_package(self):
354        packages = self.config.packages()
355        return packages['main']
356
357    def make(self):
358        package = self.main_package()
359        name = package.name()
360        if self.canadian_cross():
361            _notice(self.opts, 'package: (Cxc) %s' % (name))
362        else:
363            _notice(self.opts, 'package: %s' % (name))
364        if self.opts.trace():
365            print '---- macro maps', '-' * 55
366            print self.config.macros
367            print '-' * 70
368        self.script.reset()
369        self.script.append(self.config.expand('%{___build_template}'))
370        self.script.append('echo "=> ' + name + ':"')
371        self.prep(package)
372        self.build_package(package)
373        if not self.opts.dry_run():
374            self.builddir()
375            sn = path.join(self.config.expand('%{_builddir}'), 'doit')
376            self._output('write script: ' + sn)
377            self.script.write(sn)
378            if self.canadian_cross():
379                _notice(self.opts, 'building: (Cxc) %s' % (name))
380            else:
381                _notice(self.opts, 'building: %s' % (name))
382            self.run(sn)
383
384    def name(self):
385        packages = self.config.packages()
386        package = packages['main']
387        return package.name()
388
389def get_configs(opts):
390
391    def _scan(_path, ext):
392        configs = []
393        for root, dirs, files in os.walk(_path):
394            prefix = root[len(_path) + 1:]
395            for file in files:
396                for e in ext:
397                    if file.endswith(e):
398                        configs += [path.join(prefix, file)]
399        return configs
400
401    configs = { 'paths': [], 'files': [] }
402    for cp in opts.defaults.expand('%{_configdir}').split(':'):
403        hcp = path.host(path.abspath(cp))
404        configs['paths'] += [hcp]
405        configs['files'] += _scan(hcp, ['.cfg', '.bset'])
406    configs['files'] = sorted(configs['files'])
407    return configs
408
409def find_config(config, configs):
410    config_root, config_ext = path.splitext(config)
411    if config_ext not in ['', '.bset', '.cfg']:
412        config_root = config
413        config_ext = ''
414    for c in configs['files']:
415        r, e = path.splitext(c)
416        if config_root == r:
417            if config_ext == '' or config_ext == e:
418                return c
419    return None
420
421def run(args):
422    try:
423        optargs = { '--list-configs': 'List available configurations' }
424        opts = options.load(args, optargs)
425        log.default = log.log(opts.logfiles())
426        _notice(opts, 'RTEMS Source Builder, Package Builder v%s' % (version.str()))
427        if not check.host_setup(opts):
428            if not opts.force():
429                raise error.general('host build environment is not set up' +
430                                    ' correctly (use --force to proceed)')
431            _notice(opts, 'warning: forcing build with known host setup problems')
432        if opts.get_arg('--list-configs'):
433            configs = get_configs(opts)
434            for p in configs['paths']:
435                print 'Examining: %s' % (os.path.relpath(p))
436                for c in configs['files']:
437                    if c.endswith('.cfg'):
438                        print '    %s' % (c)
439        else:
440            for config_file in opts.config_files():
441                b = build(config_file, True, opts)
442                b.make()
443                del b
444    except error.general, gerr:
445        print gerr
446        print >> sys.stderr, 'Build FAILED'
447        sys.exit(1)
448    except error.internal, ierr:
449        print ierr
450        print >> sys.stderr, 'Build FAILED'
451        sys.exit(1)
452    except error.exit, eerr:
453        pass
454    except KeyboardInterrupt:
455        _notice(opts, 'abort: user terminated')
456        sys.exit(1)
457    sys.exit(0)
458
459if __name__ == "__main__":
460    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.